From 09acde1e384d0cf350731e7d7eda9584abbae4ec Mon Sep 17 00:00:00 2001 From: dayfine Date: Mon, 9 Nov 2026 10:34:17 -0800 Subject: [PATCH 1/6] BDSP AutoStory: pick the language and the player's look in the new-game intro The intro pressed A on every screen, so it took the language the cursor starts on and the first look (Lucas); run 3 (French, Dawn) needed both picked by hand. NintendoSwitch/Programs/MenuCursor: the D-pad presses from one item of a list or grid to another, none across an edge, so they hold whether or not the menu wraps; and a D-pad probe (press, opposite press, measure) that tells a screen with a cursor from one without, for menus no detector knows. Nothing in it is BDSP's, for PLZA's new game too. BDSP: the language select's order and the appearance grid's shape are data. Until a pick is made, screens that would take A get a probe first: a list that moves up/down but not sideways takes the language pick, a grid that moves sideways takes the look pick, then A. The player's keyboard with a pick still to make fails. New options: Switch language (where the list's cursor starts) and Player's look (models 1-8, PlayerModelOption's icons). Defaults keep the old behavior. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../MenuCursor/NintendoSwitch_MenuCursor.cpp | 97 +++++++++ .../MenuCursor/NintendoSwitch_MenuCursor.h | 73 +++++++ .../NintendoSwitch_MenuCursor_Tests.cpp | 189 ++++++++++++++++++ .../NintendoSwitch_MenuCursor_Tests.h | 23 +++ .../Options/PokemonBDSP_PlayerModelOption.cpp | 10 +- .../Options/PokemonBDSP_PlayerModelOption.h | 3 + .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 101 ++++++++++ .../Actions/PokemonBDSP_AutoStory_NewGame.cpp | 183 ++++++++++++++++- .../Actions/PokemonBDSP_AutoStory_NewGame.h | 53 ++++- .../AutoStory/PokemonBDSP_AutoStory.cpp | 30 ++- .../AutoStory/PokemonBDSP_AutoStory.h | 3 + .../PokemonBDSP_AutoStory_StoryExecutor.cpp | 2 +- .../PokemonBDSP_AutoStory_StoryExecutor.h | 3 + SerialPrograms/cmake/SourceFiles.cmake | 4 + 14 files changed, 757 insertions(+), 17 deletions(-) create mode 100644 SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp create mode 100644 SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h create mode 100644 SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp create mode 100644 SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp new file mode 100644 index 0000000000..17dc468105 --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp @@ -0,0 +1,97 @@ +/* Menu Cursor + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTools/ImageDiff.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "NintendoSwitch_MenuCursor.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + +using namespace std::chrono_literals; + + +namespace{ + +void add_presses(std::vector& moves, DpadPosition direction, size_t count){ + moves.insert(moves.end(), count, direction); +} +void add_sideways(std::vector& moves, size_t from_column, size_t to_column){ + if (to_column > from_column){ + add_presses(moves, DPAD_RIGHT, to_column - from_column); + }else{ + add_presses(moves, DPAD_LEFT, from_column - to_column); + } +} + +// 0 when either frame is missing: a probe without frames sees nothing move. +double screen_change(const VideoSnapshot& before, const VideoSnapshot& after){ + if (!before || !after){ + return 0; + } + return ImageMatch::pixel_RMSD(*before.frame, *after.frame); +} + +} // namespace + + +std::optional> menu_cursor_moves(const MenuGrid& grid, size_t from, size_t to){ + if (grid.columns == 0 || from >= grid.count || to >= grid.count){ + return std::nullopt; + } + const size_t from_row = from / grid.columns; + const size_t from_column = from % grid.columns; + const size_t to_row = to / grid.columns; + const size_t to_column = to % grid.columns; + std::vector moves; + if (to_row > from_row){ + // The start's row is above the target's, so it is full: sideways first. + add_sideways(moves, from_column, to_column); + add_presses(moves, DPAD_DOWN, to_row - from_row); + }else{ + // Up to the target's row first, which is full unless it is the start's. + add_presses(moves, DPAD_UP, from_row - to_row); + add_sideways(moves, from_column, to_column); + } + return moves; +} + + +bool cursor_probe_moved(const CursorProbe& probe){ + return probe.press_change >= CURSOR_PRESS_CHANGE && + probe.press_change >= CURSOR_NOISE_RATIO * probe.still_change && + 2 * probe.back_change <= probe.press_change; +} + +CursorProbe probe_cursor(VideoStream& stream, ProControllerContext& context, DpadPosition there, DpadPosition back){ + context.wait_for_all_requests(); + const VideoSnapshot first = stream.video().snapshot(); + context.wait_for(CURSOR_PROBE_WAIT); + const VideoSnapshot before = stream.video().snapshot(); + pbf_press_dpad(context, there, 80ms, CURSOR_PROBE_WAIT); + context.wait_for_all_requests(); + const VideoSnapshot pressed = stream.video().snapshot(); + pbf_press_dpad(context, back, 80ms, CURSOR_PROBE_WAIT); + context.wait_for_all_requests(); + const VideoSnapshot returned = stream.video().snapshot(); + return CursorProbe{ + screen_change(first, before), + screen_change(before, pressed), + screen_change(before, returned), + }; +} + +void press_cursor_moves(ProControllerContext& context, const std::vector& moves){ + for (DpadPosition move : moves){ + pbf_press_dpad(context, move, 80ms, 320ms); + } +} + + + +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h new file mode 100644 index 0000000000..fd3ad96de6 --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h @@ -0,0 +1,73 @@ +/* Menu Cursor + * + * From: https://github.com/PokemonAutomation/ + * + * Moves the cursor of a fixed list or grid (a game's language select, the + * player's look) to an item by its index. The menu's shape is data; the + * presses follow from it. A D-pad probe tells whether a screen has a + * cursor at all, for menus no detector knows. + * + */ + +#ifndef PokemonAutomation_NintendoSwitch_MenuCursor_H +#define PokemonAutomation_NintendoSwitch_MenuCursor_H + +#include +#include +#include +#include "CommonFramework/Tools/VideoStream.h" +#include "NintendoSwitch/Controllers/NintendoSwitch_ControllerButtons.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + + +// A menu of `count` items in rows of `columns`, numbered left to right, then +// top to bottom; a list has one column. Only the last row may be short. +struct MenuGrid{ + size_t columns = 1; + size_t count = 0; +}; + +// The D-pad presses that take the cursor from item `from` to item `to`. +// None crosses an edge, so they hold whether or not the menu wraps, and +// sideways presses happen on a full row. nullopt when either item is outside +// the grid. +std::optional> menu_cursor_moves(const MenuGrid& grid, size_t from, size_t to); + + +// How much the screen changed (pixel RMSD) around a D-pad probe, each against +// the frame just before the press: over a wait with no input, after the press, +// and after the opposite press. +struct CursorProbe{ + double still_change; + double press_change; + double back_change; +}; +// The least change a press that moved a cursor makes. +inline constexpr double CURSOR_PRESS_CHANGE = 6; +// How many times what the screen changes on its own (an animated preview, a +// blinking arrow) a press that moved a cursor changes it. +inline constexpr double CURSOR_NOISE_RATIO = 3; +// Whether `probe` saw a cursor move and come back: the press changed the +// screen by CURSOR_PRESS_CHANGE and CURSOR_NOISE_RATIO times the still +// change, and the opposite press put back at least half of it. +bool cursor_probe_moved(const CursorProbe& probe); + +// The wait before a probe's first press, and after each press. +inline constexpr Milliseconds CURSOR_PROBE_WAIT = Milliseconds(400); +// Waits with no input, presses `there`, waits, presses `back`, waits, and +// measures the screen at each step. On a screen with no cursor `there` and +// `back` do nothing; on one with a cursor they leave it where it was, unless +// `there` meets an edge that does not wrap. +CursorProbe probe_cursor(VideoStream& stream, ProControllerContext& context, DpadPosition there, DpadPosition back); + +// Presses `moves` in order, one D-pad tap each. +void press_cursor_moves(ProControllerContext& context, const std::vector& moves); + + + +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp new file mode 100644 index 0000000000..77bc1152e2 --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp @@ -0,0 +1,189 @@ +/* Menu Cursor - Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "NintendoSwitch_MenuCursor.h" +#include "NintendoSwitch_MenuCursor_Tests.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + +namespace{ + + +std::string describe(const std::vector& moves){ + std::string out; + for (DpadPosition move : moves){ + out += (out.empty() ? "" : " ") + dpad_to_string(move); + } + return out.empty() ? "(none)" : out; +} + +struct MovesCase{ + const char* what; + MenuGrid grid; + size_t from; + size_t to; + std::vector expected; +}; + +UnitTestResult test_cursor_moves(){ + const MenuGrid list{1, 9}; + const MenuGrid looks{4, 8}; + // Seven items in rows of three: the last row holds one. + const MenuGrid ragged{3, 7}; + const MovesCase cases[] = { + {"list: stay", list, 1, 1, {}}, + {"list: down", list, 1, 2, {DPAD_DOWN}}, + {"list: up to the top", list, 1, 0, {DPAD_UP}}, + {"list: top to bottom, no wrap", list, 0, 8, std::vector(8, DPAD_DOWN)}, + {"grid: right on the top row", looks, 0, 2, {DPAD_RIGHT, DPAD_RIGHT}}, + {"grid: down a row", looks, 0, 4, {DPAD_DOWN}}, + {"grid: sideways, then down", looks, 1, 7, {DPAD_RIGHT, DPAD_RIGHT, DPAD_DOWN}}, + {"grid: up, then sideways", looks, 7, 0, {DPAD_UP, DPAD_LEFT, DPAD_LEFT, DPAD_LEFT}}, + // The short row has only column 0: sideways happens on the full row above. + {"ragged: into the short row", ragged, 2, 6, {DPAD_LEFT, DPAD_LEFT, DPAD_DOWN, DPAD_DOWN}}, + {"ragged: out of the short row", ragged, 6, 5, {DPAD_UP, DPAD_RIGHT, DPAD_RIGHT}}, + }; + for (const MovesCase& c : cases){ + const std::optional> moves = menu_cursor_moves(c.grid, c.from, c.to); + if (!moves){ + return std::string(c.what) + ": no moves."; + } + if (*moves != c.expected){ + return std::string(c.what) + ": " + describe(*moves) + ", expected " + describe(c.expected) + "."; + } + } + if (menu_cursor_moves(looks, 0, 8) || menu_cursor_moves(looks, 8, 0) || menu_cursor_moves(MenuGrid{0, 4}, 0, 1)){ + return "An item outside the grid, or a grid with no columns, must give no moves."; + } + return true; +} + +// Walks the moves from `from` to `to` on `grid`; what went wrong, or "". +std::string check_walk(const MenuGrid& grid, size_t from, size_t to){ + const std::string pair = std::to_string(grid.columns) + "x" + std::to_string(grid.count) + + " from " + std::to_string(from) + " to " + std::to_string(to); + const std::optional> moves = menu_cursor_moves(grid, from, to); + if (!moves){ + return pair + ": no moves."; + } + long row = (long)(from / grid.columns); + long column = (long)(from % grid.columns); + for (DpadPosition move : *moves){ + row += move == DPAD_DOWN ? 1 : move == DPAD_UP ? -1 : 0; + column += move == DPAD_RIGHT ? 1 : move == DPAD_LEFT ? -1 : 0; + const long index = row * (long)grid.columns + column; + if (row < 0 || column < 0 || column >= (long)grid.columns || index >= (long)grid.count){ + return pair + ": " + describe(*moves) + " leaves the grid."; + } + } + if (row * (long)grid.columns + column != (long)to){ + return pair + ": " + describe(*moves) + " ends elsewhere."; + } + return ""; +} + +// Every pair of items in a few shapes: the moves land on the target and never +// leave the grid or step on a missing item. +UnitTestResult test_cursor_moves_stay_inside(){ + for (const MenuGrid& grid : {MenuGrid{1, 9}, MenuGrid{4, 8}, MenuGrid{3, 7}, MenuGrid{5, 1}}){ + for (size_t from = 0; from < grid.count; from++){ + for (size_t to = 0; to < grid.count; to++){ + const std::string error = check_walk(grid, from, to); + if (!error.empty()){ + return error; + } + } + } + } + return true; +} + +UnitTestResult test_cursor_probe(){ + struct Case{ + const char* what; + CursorProbe probe; + bool moved; + }; + const Case cases[] = { + {"a still list: the highlight moves and comes back", {0.5, 20, 0.5}, true}, + {"a small Yes/No box", {0.3, 8, 0.4}, true}, + {"an animated preview, swapped by the press", {8, 40, 10}, true}, + {"dialogue with a blinking arrow", {2.5, 3, 2.5}, false}, + {"an animation that changes as much on its own", {10, 20, 10}, false}, + {"a press that moved something it did not put back", {0.5, 20, 15}, false}, + {"a fade", {20, 25, 45}, false}, + {"no frames", {0, 0, 0}, false}, + }; + for (const Case& c : cases){ + if (cursor_probe_moved(c.probe) != c.moved){ + return std::string(c.what) + (c.moved ? ": the cursor move was missed." : ": no cursor moved."); + } + } + return true; +} + + +struct MenuCursorInput{ + std::string name; + UnitTestResult (*body)(); +}; + +struct MenuCursorExpectation{ +}; + +struct MenuCursorCase{ + MenuCursorInput input; + MenuCursorExpectation expectation; +}; + +class Test_MenuCursor : public UnitTest{ +public: + explicit Test_MenuCursor(MenuCursorCase test_case) + : UnitTest("NintendoSwitch::MenuCursor - " + test_case.input.name) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return m_case.input.body(); + } + +private: + MenuCursorCase m_case; +}; + +const MenuCursorCase MENU_CURSOR_CASES[] = { + { + .input = {"presses from item to item", test_cursor_moves}, + .expectation = {}, + }, + { + .input = {"presses stay inside the grid", test_cursor_moves_stay_inside}, + .expectation = {}, + }, + { + .input = {"probe judgement", test_cursor_probe}, + .expectation = {}, + }, +}; + + +} // namespace + + +void add_tests_MenuCursor(UnitTestDatabase& database){ + for (const MenuCursorCase& item : MENU_CURSOR_CASES){ + database.add(item); + } +} + + + +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h new file mode 100644 index 0000000000..ce92acd5dc --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h @@ -0,0 +1,23 @@ +/* Menu Cursor - Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_NintendoSwitch_MenuCursor_Tests_H +#define PokemonAutomation_NintendoSwitch_MenuCursor_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + + +// The cursor presses for list and grid items, and the probe's judgement. No console. +void add_tests_MenuCursor(UnitTestDatabase& database); + + + +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp index 4b81977e67..6d4490bdac 100644 --- a/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp @@ -5,6 +5,7 @@ */ #include +#include #include #include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" @@ -42,9 +43,14 @@ const PlayerModelIcons& PLAYER_MODEL_ICONS(){ PlayerModelOption::PlayerModelOption() - : StringSelectOption( + : PlayerModelOption( "Character model:
The appearance chosen for the player character. " - "Each one is watched through a different eye template.", + "Each one is watched through a different eye template." + ) +{} +PlayerModelOption::PlayerModelOption(std::string label) + : StringSelectOption( + std::move(label), PLAYER_MODEL_ICONS().database, LockMode::LOCK_WHILE_RUNNING, "model-1" diff --git a/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h index e54cd04850..9f3f358206 100644 --- a/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h +++ b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h @@ -8,6 +8,7 @@ #define PokemonAutomation_PokemonBDSP_PlayerModelOption_H #include +#include #include "CommonTools/Options/StringSelectOption.h" namespace PokemonAutomation{ @@ -22,6 +23,8 @@ const uint8_t BDSP_PLAYER_MODEL_COUNT = 8; class PlayerModelOption : public StringSelectOption{ public: PlayerModelOption(); + // The same choices, under another label. + explicit PlayerModelOption(std::string label); uint8_t model_number() const{ return (uint8_t)(index() + 1); } }; diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index 49a0dd2313..5cd0661f9b 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -20,6 +20,7 @@ #include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "CommonFramework/Logging/Logger.h" +#include "NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h" #include "PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_StoryMaps.h" #include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" #include "PokemonBDSP_AutoStory_Center.h" @@ -250,6 +251,96 @@ UnitTestResult test_intro_gate(Logger&){ } +// The presses for the language select (a list starting on the Switch's +// language) and the appearance grid (two rows of four, from the first look). +UnitTestResult test_intro_pick_moves(Logger&){ + using Moves = std::optional>; + struct Case{ + const char* what; + Moves actual; + Moves expected; + }; + const Case cases[] = { + {"French on an English Switch (lab run 3)", + language_pick_moves({Language::English, Language::French, 0}), Moves(std::vector{DPAD_DOWN})}, + {"the Switch's own language", + language_pick_moves({Language::English, Language::English, 0}), Moves(std::vector{})}, + {"English on a French Switch", + language_pick_moves({Language::French, Language::English, 0}), Moves(std::vector{DPAD_UP})}, + {"top to bottom", + language_pick_moves({Language::Japanese, Language::ChineseTraditional, 0}), Moves{std::vector(8, DPAD_DOWN)}}, + {"a language the menu lacks", + language_pick_moves({Language::English, Language::None, 0}), std::nullopt}, + {"Model 1, where the cursor starts", + look_pick_moves({Language::English, Language::English, 0}), Moves(std::vector{})}, + {"Model 3, Dawn's first look (lab run 3)", + look_pick_moves({Language::English, Language::English, 2}), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT})}, + {"Model 8, the last", + look_pick_moves({Language::English, Language::English, 7}), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT, DPAD_RIGHT, DPAD_DOWN})}, + {"no Model 9", + look_pick_moves({Language::English, Language::English, 8}), std::nullopt}, + }; + for (const Case& c : cases){ + if (c.actual != c.expected){ + return std::string("Wrong presses for ") + c.what + "."; + } + } + if (bdsp_language_menu().size() != 9){ + return "BDSP's language select lists nine languages."; + } + return true; +} + +// Until its pick is made, every screen that would take A gets a probe +// first: the language's before the look's. The player's keyboard with a +// pick still to make fails. Picks the cursor starts on need no probe. +UnitTestResult test_intro_pick_probes(Logger&){ + struct Case{ + const char* what; + const char* screen; + bool language_pending; + bool look_pending; + IntroStep expected; + }; + const Case cases[] = { + {"a logo, language to pick", "other", true, true, IntroStep::PROBE_FOR_LANGUAGE}, + {"a dialogue, language to pick", "dialog", true, false, IntroStep::PROBE_FOR_LANGUAGE}, + {"Rowan, look to pick", "dialog", false, true, IntroStep::PROBE_FOR_LOOK}, + {"the preview, look to pick", "other", false, true, IntroStep::PROBE_FOR_LOOK}, + {"both picked", "dialog", false, false, IntroStep::ADVANCE}, + {"black, language to pick", "black", true, true, IntroStep::WAIT}, + {"the user select", "user-select", true, true, IntroStep::SELECT_USER}, + {"the keyboard, language missed", "keyboard", true, false, IntroStep::FAIL_PICK_MISSED}, + {"the keyboard, look missed", "keyboard", false, true, IntroStep::FAIL_PICK_MISSED}, + {"the keyboard, both picked", "keyboard", false, false, IntroStep::ENTER_PLAYER_NAME}, + }; + for (const Case& c : cases){ + IntroProgress progress = intro_progress(0); + progress.language_pending = c.language_pending; + progress.look_pending = c.look_pending; + if (decide_intro_step(c.screen, progress) != c.expected){ + return std::string("Wrong step for ") + c.what + "."; + } + } + // The wrong-game guard still stops a game that never shows BDSP's screens. + IntroProgress other_game = intro_start({Language::English, Language::French, 2}); + note_intro_screen(other_game, "other", std::chrono::milliseconds(0)); + note_intro_screen(other_game, "other", INTRO_TIME_BEFORE_BDSP); + if (decide_intro_step("other", other_game) != IntroStep::FAIL_NOT_BDSP){ + return "Past INTRO_TIME_BEFORE_BDSP, a pending pick must not keep another game going."; + } + const IntroProgress defaults = intro_start({Language::English, Language::English, 0}); + if (defaults.language_pending || defaults.look_pending){ + return "Picks the cursor starts on need no menu."; + } + const IntroProgress run3 = intro_start({Language::English, Language::French, 2}); + if (!run3.language_pending || !run3.look_pending){ + return "French and Model 3 on an English Switch need both menus."; + } + return true; +} + + // The Mart's doorway on the failure screenshots of r121a and r122a, kept in // the unit test resources (the CommandLineTests checkout): where it is // relative to the player, who is in the middle of the screen. @@ -486,6 +577,14 @@ const AutoStoryActionsCase AUTO_STORY_ACTIONS_CASES[] = { .input = {"new-game intro: BDSP or stop", test_intro_gate}, .expectation = {}, }, + { + .input = {"new-game intro: language and look presses", test_intro_pick_moves}, + .expectation = {}, + }, + { + .input = {"new-game intro: probes until the picks are made", test_intro_pick_probes}, + .expectation = {}, + }, { .input = {"lit doorway screenshots", test_lit_doorway_screenshots}, .expectation = {}, @@ -512,6 +611,8 @@ void add_tests_AutoStory_Actions(UnitTestDatabase& database){ for (const auto& item : AUTO_STORY_ACTIONS_CASES){ database.add(item); } + // The menu cursor's own tests, with those of the intro, its first user. + add_tests_MenuCursor(database); } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp index 5bd6a47b09..162a9fbad3 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp @@ -4,6 +4,7 @@ * */ +#include #include "Common/Cpp/Exceptions.h" #include "CommonFramework/ImageTools/ImageBoxes.h" #include "CommonFramework/ImageTools/ImageDiff.h" @@ -30,6 +31,54 @@ namespace PokemonBDSP{ using namespace std::chrono_literals; +const std::vector& bdsp_language_menu(){ + static const std::vector menu{ + Language::Japanese, + Language::English, + Language::French, + Language::Italian, + Language::German, + Language::Spanish, + Language::Korean, + Language::ChineseSimplified, + Language::ChineseTraditional, + }; + return menu; +} + +namespace{ + +std::optional language_item(Language language){ + const std::vector& menu = bdsp_language_menu(); + const auto it = std::find(menu.begin(), menu.end(), language); + if (it == menu.end()){ + return std::nullopt; + } + return (size_t)(it - menu.begin()); +} + +} // namespace + +std::optional> language_pick_moves(const IntroPicks& picks){ + const std::optional from = language_item(picks.switch_language); + const std::optional to = language_item(picks.game_language); + if (!from || !to){ + return std::nullopt; + } + return menu_cursor_moves(MenuGrid{1, bdsp_language_menu().size()}, *from, *to); +} +std::optional> look_pick_moves(const IntroPicks& picks){ + return menu_cursor_moves(BDSP_LOOK_GRID, 0, picks.look); +} + +IntroProgress intro_start(const IntroPicks& picks){ + IntroProgress progress; + progress.language_pending = picks.switch_language != picks.game_language; + progress.look_pending = picks.look != 0; + return progress; +} + + std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& screen){ // The Switch's own screens come first: BDSP's dialogue detector also fires on them. if (console != nullptr && StartGameUserSelectDetector(*console).detect(screen)){ @@ -64,6 +113,25 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& return "other"; } +namespace{ + +// A on a screen that may take A, after a D-pad probe while a pick waits for +// its menu. The picks come before the player's name. +IntroStep advance_or_probe(const IntroProgress& progress){ + if (progress.player_named){ + return IntroStep::ADVANCE; + } + if (progress.language_pending){ + return IntroStep::PROBE_FOR_LANGUAGE; + } + if (progress.look_pending){ + return IntroStep::PROBE_FOR_LOOK; + } + return IntroStep::ADVANCE; +} + +} // namespace + IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress){ if (screen == "user-select"){ return IntroStep::SELECT_USER; @@ -72,6 +140,9 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog return IntroStep::FAIL_NOT_IN_GAME; } if (screen == "keyboard"){ + if (!progress.player_named && (progress.language_pending || progress.look_pending)){ + return IntroStep::FAIL_PICK_MISSED; + } return progress.player_named ? IntroStep::CANCEL_RIVAL_KEYBOARD : IntroStep::ENTER_PLAYER_NAME; } if (screen == "rival-list-tell"){ @@ -87,7 +158,7 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog return IntroStep::WAIT; } if (screen == "dialog"){ - return progress.rival_named ? IntroStep::ADVANCE_WITH_B : IntroStep::ADVANCE; + return progress.rival_named ? IntroStep::ADVANCE_WITH_B : advance_or_probe(progress); } // The bedroom comes only after the rival is named. Before that, every unknown // screen (logos, the title, language, the appearance preview, cutscenes) @@ -99,7 +170,7 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog if (!progress.game_confirmed && progress.unconfirmed_for >= INTRO_TIME_BEFORE_BDSP){ return IntroStep::FAIL_NOT_BDSP; } - return IntroStep::ADVANCE; + return advance_or_probe(progress); } bool intro_screen_confirms_bdsp(const std::string& screen){ return screen == "dialog" || screen == "rival-list-tell" || screen == "rival-list-preset" || screen == "menu"; @@ -199,6 +270,7 @@ struct IntroRun{ SingleSwitchProgramEnvironment& env; ProControllerContext& context; StepTrace& trace; + const IntroPicks& picks; const std::string& player_name; WallClock start; IntroProgress progress; @@ -277,12 +349,91 @@ void probe_for_menu(IntroRun& run){ } } -// Carries out what decide_intro_step() chose for `screen`. -void act(IntroRun& run, const char* screen){ +void advance(IntroRun& run){ + pbf_press_button(run.context, BUTTON_A, 60ms, 400ms); +} + +// Probes the screen with `there` and `back`; whether a cursor moved. +bool probe_moves(IntroRun& run, DpadPosition there, DpadPosition back){ + const CursorProbe probe = probe_cursor(run.env.console, run.context, there, back); + const bool moved = cursor_probe_moved(probe); + run.trace.action("Cursor probe " + dpad_to_string(there) + ": still " + std::to_string(probe.still_change) + + ", press " + std::to_string(probe.press_change) + ", back " + std::to_string(probe.back_change) + + (moved ? ": a cursor moved." : ": no cursor.")); + return moved; +} + +// Moves the cursor with `moves`, saves a screenshot of it on the pick, and confirms with A. +void pick_on_menu(IntroRun& run, const std::vector& moves, const std::string& what){ + run.trace.action("Picking " + what + ": " + std::to_string(moves.size()) + " D-pad presses, then A."); + press_cursor_moves(run.context, moves); + run.context.wait_for_all_requests(); + run.trace.screen("pick-" + what); + press(run, BUTTON_A); + pbf_wait(run.context, 500ms); +} + +// The language select is a list: Down moves its cursor, unless it starts on +// the last item of a list that does not wrap, so Up is tried too. Sideways +// moves only on the appearance grid, which comes later. +void probe_for_language(IntroRun& run){ + if (!probe_moves(run, DPAD_DOWN, DPAD_UP) && !probe_moves(run, DPAD_UP, DPAD_DOWN)){ + advance(run); + return; + } + if (probe_moves(run, DPAD_RIGHT, DPAD_LEFT)){ + run.trace.fail("The appearance grid showed before the language select was found."); + } + pick_on_menu(run, *language_pick_moves(run.picks), "language"); + run.progress.language_pending = false; +} + +// The appearance grid's cursor starts on the first look, so Right moves it. +// The Yes/No boxes around it do not move sideways. +void probe_for_look(IntroRun& run){ + if (!probe_moves(run, DPAD_RIGHT, DPAD_LEFT)){ + advance(run); + return; + } + pick_on_menu(run, *look_pick_moves(run.picks), "look"); + run.progress.look_pending = false; +} + +// PROBE_FOR_LANGUAGE or PROBE_FOR_LOOK. +void probe_for_pick(IntroRun& run, IntroStep step){ + if (step == IntroStep::PROBE_FOR_LANGUAGE){ + probe_for_language(run); + }else{ + probe_for_look(run); + } +} + +// Notes `screen` in the run's progress, decides the step for it and counts it. +IntroStep next_intro_step(IntroRun& run, const char* screen){ note_intro_screen(run.progress, screen, std::chrono::duration_cast(current_time() - run.start)); const IntroStep step = decide_intro_step(screen, run.progress); record_intro_step(run.progress, step); + return step; +} + +// Stops the intro for one of the FAIL_ steps. +[[noreturn]] void fail_intro(const IntroRun& run, IntroStep step){ + if (step == IntroStep::FAIL_NOT_IN_GAME){ + run.trace.fail("The Switch HOME menu is showing: the game is not running in front."); + } + if (step == IntroStep::FAIL_NOT_BDSP){ + run.trace.fail("The game in front is not BDSP: no BDSP screen in " + + std::to_string(run.progress.unconfirmed_for.count() / 1000) + " s of unknown screens."); + } + run.trace.fail(std::string("The player's name keyboard showed before the ") + + (run.progress.language_pending ? "language select" : "appearance grid") + + " was found: no cursor probe above saw its cursor move."); +} + +// Carries out what decide_intro_step() chose for `screen`. +void act(IntroRun& run, const char* screen){ + const IntroStep step = next_intro_step(run, screen); switch (step){ case IntroStep::SELECT_USER: // The cursor starts on the user that launched the game. @@ -290,10 +441,13 @@ void act(IntroRun& run, const char* screen){ pbf_wait(run.context, 1000ms); return; case IntroStep::FAIL_NOT_IN_GAME: - run.trace.fail("The Switch HOME menu is showing: the game is not running in front."); case IntroStep::FAIL_NOT_BDSP: - run.trace.fail("The game in front is not BDSP: no BDSP screen in " + - std::to_string(run.progress.unconfirmed_for.count() / 1000) + " s of unknown screens."); + case IntroStep::FAIL_PICK_MISSED: + fail_intro(run, step); + case IntroStep::PROBE_FOR_LANGUAGE: + case IntroStep::PROBE_FOR_LOOK: + probe_for_pick(run, step); + return; case IntroStep::ENTER_PLAYER_NAME: enter_player_name(run.env.console, run.context, run.trace, run.player_name); return; @@ -313,7 +467,7 @@ void act(IntroRun& run, const char* screen){ close_menu_and_finish(run); return; case IntroStep::ADVANCE: - pbf_press_button(run.context, BUTTON_A, 60ms, 400ms); + advance(run); return; case IntroStep::ADVANCE_WITH_B: case IntroStep::CLEAR_WITH_B: @@ -396,11 +550,20 @@ std::vector intro_states(IntroRun& run){ void play_new_game_intro( SingleSwitchProgramEnvironment& env, ProControllerContext& context, + const IntroPicks& picks, const std::string& player_name ){ VideoStream& stream = env.console; StepTrace trace(stream, "Intro"); + if (!language_pick_moves(picks)){ + throw UserSetupError(env.console, + "BDSP's language select lacks the Switch's language or the game's language: pick ones it lists."); + } + if (!look_pick_moves(picks)){ + throw UserSetupError(env.console, "The player's look must be one of BDSP's eight."); + } + // The restart below reopens the game HOME's cursor is on, which is the // game running in front only while it runs: with none running (r1, // 2026-11-01), closing it fails, HOME is pressed again, the cursor goes @@ -423,7 +586,9 @@ void play_new_game_intro( from_home_close_and_reopen_game(env.console, context, true); wait_out_boot(stream, context, trace); - IntroRun run{env, context, trace, player_name, current_time(), {}, 0}; + IntroRun run{env, context, trace, picks, player_name, current_time(), intro_start(picks), 0}; + trace.action(std::string("Picks: language ") + language_data(picks.game_language).name + " from " + + language_data(picks.switch_language).name + ", look " + std::to_string(picks.look + 1) + "."); const std::vector states = intro_states(run); run_screen_machine(stream, context, trace, states, std::chrono::minutes(15)); trace.action("Intro done."); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h index 35fb8d8ca9..b9dae40d15 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h @@ -4,6 +4,8 @@ * * Plays a new game's intro (language, Professor Rowan, appearance, names) up to * the bedroom with the player in control, which is where the Tutorial starts. + * The language and the player's look are picked from their menus by index + * (NintendoSwitch_MenuCursor.h); the menus' shapes are the data below. * */ @@ -14,14 +16,41 @@ #include #include #include +#include #include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Language.h" #include "NintendoSwitch/NintendoSwitch_SingleSwitchProgram.h" +#include "NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h" namespace PokemonAutomation{ namespace NintendoSwitch{ namespace PokemonBDSP{ +// The language select's items, top to bottom. Its cursor starts on the +// Switch's system language. +const std::vector& bdsp_language_menu(); +// The appearance grid: two rows of four looks, the cursor on the first. +// PlayerModelOption's "Model N" is item N - 1: Lucas's two first looks, +// then Dawn's, on each row. +inline constexpr MenuGrid BDSP_LOOK_GRID{4, 8}; + +// What the intro picks on its menus. +struct IntroPicks{ + // Where the language select's cursor starts. + Language switch_language = Language::English; + Language game_language = Language::English; + // An item of BDSP_LOOK_GRID. + size_t look = 0; +}; +// The presses from the Switch's language to the game's on the language +// select; nullopt when the menu lacks either. +std::optional> language_pick_moves(const IntroPicks& picks); +// The presses from the first look to the chosen one; nullopt when it is +// outside the grid. +std::optional> look_pick_moves(const IntroPicks& picks); + + // Which intro screen `screen` shows, as the intro's steps tell them apart: // "user-select", "home", "keyboard", "rival-list-tell" (the rival's name list on // "I'll tell you..."), "rival-list-preset" (on a preset name), "menu", "dialog" or @@ -35,6 +64,9 @@ enum class IntroStep{ SELECT_USER, // A on the Switch's user select FAIL_NOT_IN_GAME, // HOME is showing FAIL_NOT_BDSP, // no BDSP screen within INTRO_TIME_BEFORE_BDSP + FAIL_PICK_MISSED, // the player's keyboard, with a pick still to make + PROBE_FOR_LANGUAGE, // D-pad probe: a list takes the language pick, else A + PROBE_FOR_LOOK, // D-pad probe: a grid takes the look pick, else A ENTER_PLAYER_NAME, // type the name on the first keyboard CANCEL_RIVAL_KEYBOARD, // X: back to the rival's name list MOVE_TO_PRESET, // Down: off "I'll tell you..." @@ -48,6 +80,10 @@ enum class IntroStep{ }; // Progress tracking state for the new-game introductory sequence. struct IntroProgress{ + // Picks still to make on the language select and the appearance grid. A + // pick the cursor already starts on needs no menu, and A takes it. + bool language_pending = false; + bool look_pending = false; bool player_named = false; // A preset name was picked from the rival's list, and how many screens // have shown since without the list coming back. "Barry, is it?" and @@ -74,7 +110,9 @@ struct IntroProgress{ // and the language select. Another game launched by mistake (r1, // 2026-11-01: FireRed) fails here instead of being played. Black screens // take no presses: BDSP boots on black for about 25 s (r3, 2026-11-01). -inline constexpr std::chrono::milliseconds INTRO_TIME_BEFORE_BDSP = std::chrono::seconds(60); +// While the language pick waits for its menu, each of those screens also +// takes two cursor probes of about 1.4 s each, hence two minutes. +inline constexpr std::chrono::milliseconds INTRO_TIME_BEFORE_BDSP = std::chrono::seconds(120); // How long the boot may stay black after the restart. inline constexpr std::chrono::milliseconds INTRO_BOOT_LIMIT = std::chrono::seconds(60); // Whether `screen` (classify_intro_screen()) is one only BDSP's intro shows: @@ -126,18 +164,25 @@ bool intro_in_control_without_menu(const XProbe& probe); // Counts `step` in `progress`. void record_intro_step(IntroProgress& progress, IntroStep step); +// The progress a new intro starts from: which of `picks` need a menu. +IntroProgress intro_start(const IntroPicks& picks); + // From the title screen or any intro screen of a game with no save, with BDSP // running in front: from HOME it throws UserSetupError, since the restart would -// launch whatever tile HOME's cursor is on. Restarts the game first, so it +// launch whatever tile HOME's cursor is on, as it does for picks no menu has. Restarts the game first, so it // always starts from the title, waits out the black boot (up to // INTRO_BOOT_LIMIT) without pressing anything, and fails if no BDSP screen shows within -// INTRO_TIME_BEFORE_BDSP of screens. Names the player `player_name` -// and picks the rival's default name (Barry) from the list. Each step reads the +// INTRO_TIME_BEFORE_BDSP of screens. Picks the language and the look in +// `picks`, names the player `player_name` and picks the rival's default name +// (Barry) from the list. Until a pick is made, each unknown screen gets a +// D-pad probe before its A; the player's keyboard showing with a pick still +// to make fails. Each step reads the // screen and waits for the expected next one; an unexpected screen fails with a // screenshot and the recent screens. void play_new_game_intro( SingleSwitchProgramEnvironment& env, ProControllerContext& context, + const IntroPicks& picks, const std::string& player_name ); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp index a3c7c676f9..16f5309bbe 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp @@ -16,6 +16,7 @@ #include "Pokemon/AutoStory/Pokemon_AutoStoryStats.h" #include "Pokemon/AutoStory/Pokemon_StorySegments.h" #include "PokemonBDSP/Programs/PokemonBDSP_GameEntry.h" +#include "PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h" #include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" #include "PokemonBDSP_AutoStory_StoryTable.h" #include "PokemonBDSP_AutoStory.h" @@ -165,6 +166,23 @@ void verify_story_checkpoint( } // namespace +namespace{ + +// The languages BDSP's language select lists, in its order. +const EnumDropdownDatabase& switch_language_database(){ + static const EnumDropdownDatabase database = []{ + EnumDropdownDatabase ret; + for (Language language : bdsp_language_menu()){ + ret.add(language, language_data(language).code, language_data(language).name, true); + } + return ret; + }(); + return database; +} + +} // namespace + + BDSPAutoStory::BDSPAutoStory() : RUN(checkpoint_database()) , NEW_GAME_INTRO( @@ -180,6 +198,14 @@ BDSPAutoStory::BDSPAutoStory() LockMode::LOCK_WHILE_RUNNING, false ) , PLAYER_NAME(false, "Player name:
For a new game. Letters and digits.", LockMode::LOCK_WHILE_RUNNING, "Kosmos", "Kosmos") + , SWITCH_LANGUAGE( + "Switch language:
For a new game: the Switch's system language, where BDSP's language select starts its cursor.", + switch_language_database(), LockMode::LOCK_WHILE_RUNNING, Language::English + ) + , PLAYER_LOOK( + "Player's look:
For a new game: the look picked on the appearance grid. " + "Models 1, 2, 5 and 6 are Lucas; 3, 4, 7 and 8 are Dawn." + ) , STARTER( "Starter Pokemon:", { @@ -200,6 +226,8 @@ BDSPAutoStory::BDSPAutoStory() PA_ADD_OPTION(NEW_GAME_INTRO); PA_ADD_OPTION(JUST_LOADED); PA_ADD_OPTION(PLAYER_NAME); + PA_ADD_OPTION(SWITCH_LANGUAGE); + PA_ADD_OPTION(PLAYER_LOOK); PA_ADD_OPTION(STARTER); PA_ADD_OPTION(NOTIFICATIONS); } @@ -228,7 +256,7 @@ void BDSPAutoStory::program(SingleSwitchProgramEnvironment& env, ProControllerCo // The reads that use text (Location.h) take the game's language from here. set_story_language(RUN.language()); - StoryEnv story{env, context, STARTER, PLAYER_NAME}; + StoryEnv story{env, context, STARTER, PLAYER_NAME, IntroPicks{SWITCH_LANGUAGE, RUN.language(), PLAYER_LOOK.index()}}; StoryExecutor executor(story); executor.set_checks(RUN.checks()); executor.set_language(RUN.language()); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h index 9fe9b929a5..753bdde767 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h @@ -19,6 +19,7 @@ #include "CommonFramework/Notifications/EventNotificationsTable.h" #include "NintendoSwitch/NintendoSwitch_SingleSwitchProgram.h" #include "Pokemon/Options/Pokemon_StoryRunOptions.h" +#include "PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h" #include "PokemonBDSP_AutoStory_StoryExecutor.h" namespace PokemonAutomation{ @@ -49,6 +50,8 @@ class BDSPAutoStory : public SingleSwitchProgramInstance{ BooleanCheckBoxOption NEW_GAME_INTRO; BooleanCheckBoxOption JUST_LOADED; StringOption PLAYER_NAME; + EnumDropdownOption SWITCH_LANGUAGE; + PlayerModelOption PLAYER_LOOK; EnumDropdownOption STARTER; EventNotificationsOption NOTIFICATIONS; diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_StoryExecutor.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_StoryExecutor.cpp index 756a377fbc..8e0ec5d42b 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_StoryExecutor.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_StoryExecutor.cpp @@ -189,7 +189,7 @@ void StoryExecutor::begin_attempt(const Pokemon::StoryCheckpoint& data){ } Outcome StoryExecutor::new_game(RunState&, const StoryStep&){ - play_new_game_intro(m_env.env, context(), m_env.player_name); + play_new_game_intro(m_env.env, context(), m_env.intro_picks, m_env.player_name); return Pokemon::outcome_ok(); } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_StoryExecutor.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_StoryExecutor.h index 178719d044..f1882551ee 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_StoryExecutor.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_StoryExecutor.h @@ -33,6 +33,7 @@ #include #include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" #include "NintendoSwitch/NintendoSwitch_SingleSwitchProgram.h" +#include "PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h" #include "PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_StoryWorld.h" #include "Pokemon/AutoStory/Pokemon_StoryActions.h" #include "Pokemon/AutoStory/Pokemon_StorySegments.h" @@ -81,6 +82,8 @@ struct StoryEnv{ ProControllerContext& context; StarterChoice starter = StarterChoice::CHIMCHAR; std::string player_name; + // The new game's language and look. + IntroPicks intro_picks = {}; }; diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index aebd01e09d..727a6d1b23 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1425,6 +1425,10 @@ file(GLOB LIBRARY_SOURCES Source/NintendoSwitch/Programs/NintendoSwitch_TurboButton.h Source/NintendoSwitch/Programs/NintendoSwitch_TurboMacro.cpp Source/NintendoSwitch/Programs/NintendoSwitch_TurboMacro.h + Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp + Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h + Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp + Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h Source/NintendoSwitch/Programs/Typewriter/NintendoSwitch_Typewriter.cpp Source/NintendoSwitch/Programs/Typewriter/NintendoSwitch_Typewriter.h Source/NintendoSwitch/Programs/Typewriter/NintendoSwitch_Typewriter_Tests.cpp From 3f009a4bc7558e179dfc2e585370d064d9192b0f Mon Sep 17 00:00:00 2001 From: dayfine Date: Mon, 9 Nov 2026 11:14:14 -0800 Subject: [PATCH 2/6] BDSP AutoStory: read the language select with a detector The language select is a 2 x 5 grid of buttons, not a 9-item list (lab-B frame, 2026-11-09): English, Espanol, Francais, Deutsch, Italiano on the left; Japanese kana and kanji, Korean, Simplified and Traditional Chinese on the right; the cursor starts on English. Right moves on it, so the probe rule took it for the appearance grid. A detector reads it from its ten buttons (grey, the cursor's blue) and which one the cursor is on. The intro classifies it as its own screen, moves from the read cursor to the game's language, reads it again (up to three tries), then A. The look grid keeps the D-pad probe, only after the language is picked. The Switch language option is gone: the cursor is read, not assumed. The wrong-game guard is back to 60 s. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 112 ++++++++++++------ .../Actions/PokemonBDSP_AutoStory_NewGame.cpp | 111 ++++++++++------- .../Actions/PokemonBDSP_AutoStory_NewGame.h | 40 ++++--- ...nBDSP_AutoStory_LanguageSelectDetector.cpp | 79 ++++++++++++ ...monBDSP_AutoStory_LanguageSelectDetector.h | 36 ++++++ .../AutoStory/PokemonBDSP_AutoStory.cpp | 24 +--- .../AutoStory/PokemonBDSP_AutoStory.h | 1 - SerialPrograms/cmake/SourceFiles.cmake | 2 + 8 files changed, 282 insertions(+), 123 deletions(-) create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index 5cd0661f9b..5283dcb2c4 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -23,6 +23,7 @@ #include "NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h" #include "PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_StoryMaps.h" #include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h" #include "PokemonBDSP_AutoStory_Center.h" #include "PokemonBDSP_AutoStory_NewGame.h" #include "PokemonBDSP_AutoStory_Actions_Tests.h" @@ -203,7 +204,7 @@ std::string check_boot(IntroProgress& progress){ } UnitTestResult check_confirming_screens(){ - for (const char* screen : {"dialog", "rival-list-tell", "rival-list-preset", "menu"}){ + for (const char* screen : {"dialog", "language-select", "rival-list-tell", "rival-list-preset", "menu"}){ if (!intro_screen_confirms_bdsp(screen)){ return std::string("Expected \"") + screen + "\" to confirm BDSP."; } @@ -251,50 +252,59 @@ UnitTestResult test_intro_gate(Logger&){ } -// The presses for the language select (a list starting on the Switch's -// language) and the appearance grid (two rows of four, from the first look). +// The presses for the language select (two columns of five, read row by +// row, from wherever its cursor is) and the appearance grid (two rows of +// four, from the first look). UnitTestResult test_intro_pick_moves(Logger&){ using Moves = std::optional>; + const IntroPicks french{Language::French, 2}; struct Case{ const char* what; Moves actual; Moves expected; }; const Case cases[] = { - {"French on an English Switch (lab run 3)", - language_pick_moves({Language::English, Language::French, 0}), Moves(std::vector{DPAD_DOWN})}, - {"the Switch's own language", - language_pick_moves({Language::English, Language::English, 0}), Moves(std::vector{})}, - {"English on a French Switch", - language_pick_moves({Language::French, Language::English, 0}), Moves(std::vector{DPAD_UP})}, - {"top to bottom", - language_pick_moves({Language::Japanese, Language::ChineseTraditional, 0}), Moves{std::vector(8, DPAD_DOWN)}}, + {"French from English, where the cursor starts (lab run 3)", + language_pick_moves(0, french), Moves(std::vector{DPAD_DOWN, DPAD_DOWN})}, + {"English from English", + language_pick_moves(0, IntroPicks{Language::English, 0}), Moves(std::vector{})}, + {"French from Traditional Chinese, bottom right", + language_pick_moves(9, french), Moves(std::vector{DPAD_UP, DPAD_UP, DPAD_LEFT})}, + {"Japanese: its kana button, right of English", + language_pick_moves(0, IntroPicks{Language::Japanese, 0}), Moves(std::vector{DPAD_RIGHT})}, + {"Traditional Chinese from English", + language_pick_moves(0, IntroPicks{Language::ChineseTraditional, 0}), + Moves(std::vector{DPAD_RIGHT, DPAD_DOWN, DPAD_DOWN, DPAD_DOWN, DPAD_DOWN})}, + {"Italian, bottom left", + language_pick_moves(0, IntroPicks{Language::Italian, 0}), Moves(std::vector(4, DPAD_DOWN))}, {"a language the menu lacks", - language_pick_moves({Language::English, Language::None, 0}), std::nullopt}, + language_pick_moves(0, IntroPicks{Language::None, 0}), std::nullopt}, + {"a cursor outside the menu", + language_pick_moves(LANGUAGE_SELECT_BUTTONS, french), std::nullopt}, {"Model 1, where the cursor starts", - look_pick_moves({Language::English, Language::English, 0}), Moves(std::vector{})}, + look_pick_moves(IntroPicks{Language::English, 0}), Moves(std::vector{})}, {"Model 3, Dawn's first look (lab run 3)", - look_pick_moves({Language::English, Language::English, 2}), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT})}, + look_pick_moves(french), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT})}, {"Model 8, the last", - look_pick_moves({Language::English, Language::English, 7}), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT, DPAD_RIGHT, DPAD_DOWN})}, + look_pick_moves(IntroPicks{Language::English, 7}), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT, DPAD_RIGHT, DPAD_DOWN})}, {"no Model 9", - look_pick_moves({Language::English, Language::English, 8}), std::nullopt}, + look_pick_moves(IntroPicks{Language::English, 8}), std::nullopt}, }; for (const Case& c : cases){ if (c.actual != c.expected){ return std::string("Wrong presses for ") + c.what + "."; } } - if (bdsp_language_menu().size() != 9){ - return "BDSP's language select lists nine languages."; + if (bdsp_language_menu().size() != BDSP_LANGUAGE_GRID.count || BDSP_LANGUAGE_GRID.count != LANGUAGE_SELECT_BUTTONS){ + return "The language select's data, grid and detector must agree on ten buttons."; } return true; } -// Until its pick is made, every screen that would take A gets a probe -// first: the language's before the look's. The player's keyboard with a -// pick still to make fails. Picks the cursor starts on need no probe. -UnitTestResult test_intro_pick_probes(Logger&){ +// The language select takes the language pick, then A if it still shows. +// Until the look is picked, every screen after the language that would take +// A gets a probe first. The player's keyboard with a pick still to make fails. +UnitTestResult test_intro_pick_steps(Logger&){ struct Case{ const char* what; const char* screen; @@ -303,12 +313,14 @@ UnitTestResult test_intro_pick_probes(Logger&){ IntroStep expected; }; const Case cases[] = { - {"a logo, language to pick", "other", true, true, IntroStep::PROBE_FOR_LANGUAGE}, - {"a dialogue, language to pick", "dialog", true, false, IntroStep::PROBE_FOR_LANGUAGE}, + {"the language select", "language-select", true, true, IntroStep::PICK_LANGUAGE}, + {"the language select, picked", "language-select", false, true, IntroStep::ADVANCE}, + {"a logo before the language", "other", true, true, IntroStep::ADVANCE}, + {"a dialogue before the language", "dialog", true, true, IntroStep::ADVANCE}, {"Rowan, look to pick", "dialog", false, true, IntroStep::PROBE_FOR_LOOK}, {"the preview, look to pick", "other", false, true, IntroStep::PROBE_FOR_LOOK}, {"both picked", "dialog", false, false, IntroStep::ADVANCE}, - {"black, language to pick", "black", true, true, IntroStep::WAIT}, + {"black, look to pick", "black", false, true, IntroStep::WAIT}, {"the user select", "user-select", true, true, IntroStep::SELECT_USER}, {"the keyboard, language missed", "keyboard", true, false, IntroStep::FAIL_PICK_MISSED}, {"the keyboard, look missed", "keyboard", false, true, IntroStep::FAIL_PICK_MISSED}, @@ -322,20 +334,40 @@ UnitTestResult test_intro_pick_probes(Logger&){ return std::string("Wrong step for ") + c.what + "."; } } - // The wrong-game guard still stops a game that never shows BDSP's screens. - IntroProgress other_game = intro_start({Language::English, Language::French, 2}); - note_intro_screen(other_game, "other", std::chrono::milliseconds(0)); - note_intro_screen(other_game, "other", INTRO_TIME_BEFORE_BDSP); - if (decide_intro_step("other", other_game) != IntroStep::FAIL_NOT_BDSP){ - return "Past INTRO_TIME_BEFORE_BDSP, a pending pick must not keep another game going."; - } - const IntroProgress defaults = intro_start({Language::English, Language::English, 0}); - if (defaults.language_pending || defaults.look_pending){ - return "Picks the cursor starts on need no menu."; + const IntroProgress first_look = intro_start(IntroPicks{Language::English, 0}); + if (!first_look.language_pending || first_look.look_pending){ + return "The language is always picked on its select; the first look needs no grid."; } - const IntroProgress run3 = intro_start({Language::English, Language::French, 2}); + const IntroProgress run3 = intro_start(IntroPicks{Language::French, 2}); if (!run3.language_pending || !run3.look_pending){ - return "French and Model 3 on an English Switch need both menus."; + return "French and Model 3 need both menus."; + } + return true; +} + +// The language select from lab-B 2026-11-09 (cursor on English), and screens +// that are not it. Screenshots are in the unit test resources; a missing one +// is skipped. +UnitTestResult test_language_select_screenshots(Logger& logger){ + struct Case{ + const char* file; + std::optional cursor; + }; + const Case cases[] = { + {.file = "PokemonBDSP/AutoStory/LanguageSelect-20261109.png", .cursor = 0}, + {.file = "PokemonBDSP/AutoStory/OreburghCenterCounter-20261108-094735.png", .cursor = std::nullopt}, + {.file = "PokemonBDSP/AutoStory/OreburghCenterDoorstep-20261108.png", .cursor = std::nullopt}, + }; + for (const Case& c : cases){ + const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + if (!Filesystem::is_regular_file(path)){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + } + const std::optional cursor = language_select_cursor(ImageRGB32(path)); + logger.log(std::string(c.file) + ": cursor " + (cursor ? std::to_string(*cursor) : "none")); + if (cursor != c.cursor){ + return std::string(c.file) + ": wrong language select read."; + } } return true; } @@ -582,7 +614,11 @@ const AutoStoryActionsCase AUTO_STORY_ACTIONS_CASES[] = { .expectation = {}, }, { - .input = {"new-game intro: probes until the picks are made", test_intro_pick_probes}, + .input = {"new-game intro: steps until the picks are made", test_intro_pick_steps}, + .expectation = {}, + }, + { + .input = {"language select screenshots", test_language_select_screenshots}, .expectation = {}, }, { diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp index 162a9fbad3..540ccb9153 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp @@ -22,6 +22,7 @@ #include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" #include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" #include "PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h" #include "PokemonBDSP_AutoStory_NewGame.h" namespace PokemonAutomation{ @@ -33,15 +34,11 @@ using namespace std::chrono_literals; const std::vector& bdsp_language_menu(){ static const std::vector menu{ - Language::Japanese, - Language::English, - Language::French, - Language::Italian, - Language::German, - Language::Spanish, - Language::Korean, - Language::ChineseSimplified, - Language::ChineseTraditional, + Language::English, Language::Japanese, + Language::Spanish, Language::Japanese, + Language::French, Language::Korean, + Language::German, Language::ChineseSimplified, + Language::Italian, Language::ChineseTraditional, }; return menu; } @@ -59,13 +56,12 @@ std::optional language_item(Language language){ } // namespace -std::optional> language_pick_moves(const IntroPicks& picks){ - const std::optional from = language_item(picks.switch_language); +std::optional> language_pick_moves(size_t cursor, const IntroPicks& picks){ const std::optional to = language_item(picks.game_language); - if (!from || !to){ + if (!to){ return std::nullopt; } - return menu_cursor_moves(MenuGrid{1, bdsp_language_menu().size()}, *from, *to); + return menu_cursor_moves(BDSP_LANGUAGE_GRID, cursor, *to); } std::optional> look_pick_moves(const IntroPicks& picks){ return menu_cursor_moves(BDSP_LOOK_GRID, 0, picks.look); @@ -73,7 +69,7 @@ std::optional> look_pick_moves(const IntroPicks& picks IntroProgress intro_start(const IntroPicks& picks){ IntroProgress progress; - progress.language_pending = picks.switch_language != picks.game_language; + progress.language_pending = true; progress.look_pending = picks.look != 0; return progress; } @@ -90,6 +86,10 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& if (SystemKeyboardDetector().detect(screen)){ return "keyboard"; } + // Before the dialogue box: its prompt sits in one. + if (language_select_cursor(screen)){ + return "language-select"; + } switch (NameListDetector().selected_row(screen)){ case -1: break; @@ -115,16 +115,14 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& namespace{ -// A on a screen that may take A, after a D-pad probe while a pick waits for -// its menu. The picks come before the player's name. +// A on a screen that may take A, after a D-pad probe while the look waits +// for its grid. The language select comes first, then the look, then the +// player's name. IntroStep advance_or_probe(const IntroProgress& progress){ if (progress.player_named){ return IntroStep::ADVANCE; } - if (progress.language_pending){ - return IntroStep::PROBE_FOR_LANGUAGE; - } - if (progress.look_pending){ + if (!progress.language_pending && progress.look_pending){ return IntroStep::PROBE_FOR_LOOK; } return IntroStep::ADVANCE; @@ -145,6 +143,10 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog } return progress.player_named ? IntroStep::CANCEL_RIVAL_KEYBOARD : IntroStep::ENTER_PLAYER_NAME; } + if (screen == "language-select"){ + // Once picked, A again only if the select still shows. + return progress.language_pending ? IntroStep::PICK_LANGUAGE : IntroStep::ADVANCE; + } if (screen == "rival-list-tell"){ return IntroStep::MOVE_TO_PRESET; } @@ -173,7 +175,7 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog return advance_or_probe(progress); } bool intro_screen_confirms_bdsp(const std::string& screen){ - return screen == "dialog" || screen == "rival-list-tell" || screen == "rival-list-preset" || screen == "menu"; + return screen == "dialog" || screen == "language-select" || screen == "rival-list-tell" || screen == "rival-list-preset" || screen == "menu"; } void note_intro_screen(IntroProgress& progress, const std::string& screen, std::chrono::milliseconds since_start){ @@ -373,19 +375,40 @@ void pick_on_menu(IntroRun& run, const std::vector& moves, const s pbf_wait(run.context, 500ms); } -// The language select is a list: Down moves its cursor, unless it starts on -// the last item of a list that does not wrap, so Up is tried too. Sideways -// moves only on the appearance grid, which comes later. -void probe_for_language(IntroRun& run){ - if (!probe_moves(run, DPAD_DOWN, DPAD_UP) && !probe_moves(run, DPAD_UP, DPAD_DOWN)){ - advance(run); - return; +// Reads of the language select before the cursor must be on the game's language. +const size_t LANGUAGE_PICK_TRIES = 3; + +// The language select's cursor as read now; nullopt when it is not showing. +std::optional read_language_cursor(IntroRun& run){ + run.context.wait_for_all_requests(); + const VideoSnapshot screen = run.env.console.video().snapshot(); + if (!screen){ + return std::nullopt; } - if (probe_moves(run, DPAD_RIGHT, DPAD_LEFT)){ - run.trace.fail("The appearance grid showed before the language select was found."); + return language_select_cursor(*screen.frame); +} + +// Reads the cursor, moves it to the game's language and reads it there, +// up to LANGUAGE_PICK_TRIES times (a press dropped as the screen fades in), +// then A. +void pick_language(IntroRun& run){ + const std::optional target = language_item(run.picks.game_language); + for (size_t attempt = 0; attempt < LANGUAGE_PICK_TRIES; attempt++){ + const std::optional cursor = read_language_cursor(run); + if (!cursor){ + return; // Gone or fading: the next read decides. + } + run.trace.action("Language select: cursor on item " + std::to_string(*cursor) + + ", the game's language is item " + std::to_string(*target) + "."); + if (*cursor == *target){ + pick_on_menu(run, {}, "language"); + run.progress.language_pending = false; + return; + } + press_cursor_moves(run.context, *language_pick_moves(*cursor, run.picks)); } - pick_on_menu(run, *language_pick_moves(run.picks), "language"); - run.progress.language_pending = false; + run.trace.fail("The language select's cursor did not reach the game's language in " + + std::to_string(LANGUAGE_PICK_TRIES) + " tries."); } // The appearance grid's cursor starts on the first look, so Right moves it. @@ -399,10 +422,10 @@ void probe_for_look(IntroRun& run){ run.progress.look_pending = false; } -// PROBE_FOR_LANGUAGE or PROBE_FOR_LOOK. -void probe_for_pick(IntroRun& run, IntroStep step){ - if (step == IntroStep::PROBE_FOR_LANGUAGE){ - probe_for_language(run); +// PICK_LANGUAGE or PROBE_FOR_LOOK. +void pick_on_intro_menu(IntroRun& run, IntroStep step){ + if (step == IntroStep::PICK_LANGUAGE){ + pick_language(run); }else{ probe_for_look(run); } @@ -444,9 +467,9 @@ void act(IntroRun& run, const char* screen){ case IntroStep::FAIL_NOT_BDSP: case IntroStep::FAIL_PICK_MISSED: fail_intro(run, step); - case IntroStep::PROBE_FOR_LANGUAGE: + case IntroStep::PICK_LANGUAGE: case IntroStep::PROBE_FOR_LOOK: - probe_for_pick(run, step); + pick_on_intro_menu(run, step); return; case IntroStep::ENTER_PLAYER_NAME: enter_player_name(run.env.console, run.context, run.trace, run.player_name); @@ -521,6 +544,9 @@ std::vector intro_states(IntroRun& run){ {"rival-keyboard", [=, &run](const ImageViewRGB32& s){ return run.progress.player_named && screen_is(s, "keyboard"); }, [&run]{ act(run, "keyboard"); }, {"rival-list-tell", "dialog", "black", "other"}, 3}, + {"language-select", + [=](const ImageViewRGB32& s){ return screen_is(s, "language-select"); }, + [&run]{ act(run, "language-select"); }, {}, 10}, {"rival-list-tell", [=](const ImageViewRGB32& s){ return screen_is(s, "rival-list-tell"); }, [&run]{ act(run, "rival-list-tell"); }, {"rival-list-preset", "black", "other"}, 6}, @@ -556,9 +582,8 @@ void play_new_game_intro( VideoStream& stream = env.console; StepTrace trace(stream, "Intro"); - if (!language_pick_moves(picks)){ - throw UserSetupError(env.console, - "BDSP's language select lacks the Switch's language or the game's language: pick ones it lists."); + if (!language_pick_moves(0, picks)){ + throw UserSetupError(env.console, "BDSP's language select lacks the game's language: pick one it lists."); } if (!look_pick_moves(picks)){ throw UserSetupError(env.console, "The player's look must be one of BDSP's eight."); @@ -587,8 +612,8 @@ void play_new_game_intro( wait_out_boot(stream, context, trace); IntroRun run{env, context, trace, picks, player_name, current_time(), intro_start(picks), 0}; - trace.action(std::string("Picks: language ") + language_data(picks.game_language).name + " from " + - language_data(picks.switch_language).name + ", look " + std::to_string(picks.look + 1) + "."); + trace.action(std::string("Picks: language ") + language_data(picks.game_language).name + + ", look " + std::to_string(picks.look + 1) + "."); const std::vector states = intro_states(run); run_screen_machine(stream, context, trace, states, std::chrono::minutes(15)); trace.action("Intro done."); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h index b9dae40d15..d3e8516d24 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h @@ -6,6 +6,8 @@ * the bedroom with the player in control, which is where the Tutorial starts. * The language and the player's look are picked from their menus by index * (NintendoSwitch_MenuCursor.h); the menus' shapes are the data below. + * The language select is read by its detector; the appearance grid, which + * has none yet, is found with a D-pad probe. * */ @@ -27,8 +29,13 @@ namespace NintendoSwitch{ namespace PokemonBDSP{ -// The language select's items, top to bottom. Its cursor starts on the -// Switch's system language. +// The language select: two columns of five buttons, English, Espanol, +// Francais, Deutsch, Italiano on the left; Japanese in kana, Japanese in +// kanji, Korean, Simplified and Traditional Chinese on the right (lab-B +// 2026-11-09). Items are numbered row by row, as MenuGrid reads them. +inline constexpr MenuGrid BDSP_LANGUAGE_GRID{2, 10}; +// The language of each item of BDSP_LANGUAGE_GRID. Japanese is there twice; +// its first item, the kana, is the one picked. const std::vector& bdsp_language_menu(); // The appearance grid: two rows of four looks, the cursor on the first. // PlayerModelOption's "Model N" is item N - 1: Lucas's two first looks, @@ -37,22 +44,20 @@ inline constexpr MenuGrid BDSP_LOOK_GRID{4, 8}; // What the intro picks on its menus. struct IntroPicks{ - // Where the language select's cursor starts. - Language switch_language = Language::English; Language game_language = Language::English; // An item of BDSP_LOOK_GRID. size_t look = 0; }; -// The presses from the Switch's language to the game's on the language -// select; nullopt when the menu lacks either. -std::optional> language_pick_moves(const IntroPicks& picks); +// The presses from the language select's item `cursor` to the game's +// language; nullopt when the menu lacks it or `cursor` is outside it. +std::optional> language_pick_moves(size_t cursor, const IntroPicks& picks); // The presses from the first look to the chosen one; nullopt when it is // outside the grid. std::optional> look_pick_moves(const IntroPicks& picks); // Which intro screen `screen` shows, as the intro's steps tell them apart: -// "user-select", "home", "keyboard", "rival-list-tell" (the rival's name list on +// "user-select", "home", "keyboard", "language-select", "rival-list-tell" (the rival's name list on // "I'll tell you..."), "rival-list-preset" (on a preset name), "menu", "dialog" or // "black" (the boot, fades) or "other". The Switch's own screens need // `console`; without it they read as whatever else they look like. @@ -65,7 +70,7 @@ enum class IntroStep{ FAIL_NOT_IN_GAME, // HOME is showing FAIL_NOT_BDSP, // no BDSP screen within INTRO_TIME_BEFORE_BDSP FAIL_PICK_MISSED, // the player's keyboard, with a pick still to make - PROBE_FOR_LANGUAGE, // D-pad probe: a list takes the language pick, else A + PICK_LANGUAGE, // the language select: move to the game's language, A PROBE_FOR_LOOK, // D-pad probe: a grid takes the look pick, else A ENTER_PLAYER_NAME, // type the name on the first keyboard CANCEL_RIVAL_KEYBOARD, // X: back to the rival's name list @@ -80,8 +85,8 @@ enum class IntroStep{ }; // Progress tracking state for the new-game introductory sequence. struct IntroProgress{ - // Picks still to make on the language select and the appearance grid. A - // pick the cursor already starts on needs no menu, and A takes it. + // Picks still to make on the language select and the appearance grid. + // The first look needs no menu: A takes it. bool language_pending = false; bool look_pending = false; bool player_named = false; @@ -110,13 +115,11 @@ struct IntroProgress{ // and the language select. Another game launched by mistake (r1, // 2026-11-01: FireRed) fails here instead of being played. Black screens // take no presses: BDSP boots on black for about 25 s (r3, 2026-11-01). -// While the language pick waits for its menu, each of those screens also -// takes two cursor probes of about 1.4 s each, hence two minutes. -inline constexpr std::chrono::milliseconds INTRO_TIME_BEFORE_BDSP = std::chrono::seconds(120); +inline constexpr std::chrono::milliseconds INTRO_TIME_BEFORE_BDSP = std::chrono::seconds(60); // How long the boot may stay black after the restart. inline constexpr std::chrono::milliseconds INTRO_BOOT_LIMIT = std::chrono::seconds(60); // Whether `screen` (classify_intro_screen()) is one only BDSP's intro shows: -// its dialogue box, the rival's name list, its main menu. The keyboard and +// its dialogue box, its language select, the rival's name list, its main menu. The keyboard and // the user select are the Switch's. bool intro_screen_confirms_bdsp(const std::string& screen); @@ -174,9 +177,10 @@ IntroProgress intro_start(const IntroPicks& picks); // INTRO_BOOT_LIMIT) without pressing anything, and fails if no BDSP screen shows within // INTRO_TIME_BEFORE_BDSP of screens. Picks the language and the look in // `picks`, names the player `player_name` and picks the rival's default name -// (Barry) from the list. Until a pick is made, each unknown screen gets a -// D-pad probe before its A; the player's keyboard showing with a pick still -// to make fails. Each step reads the +// (Barry) from the list. The language select is read, moved on and read +// again before its A. Until the look is picked, each unknown screen after it +// gets a D-pad probe before its A. The player's keyboard showing with a pick +// still to make fails. Each step reads the // screen and waits for the expected next one; an unexpected screen fails with a // screenshot and the recent screens. void play_new_game_intro( diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp new file mode 100644 index 0000000000..66384f774c --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp @@ -0,0 +1,79 @@ +/* BDSP AutoStory - Language Select Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "PokemonBDSP_AutoStory_LanguageSelectDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ + +// A strip inside each button, left of its text (1080p: x 330-460 on the left +// column, 1040-1170 on the right; 60 px tall around each row's centre). +const double COLUMN_X[2] = {0.172, 0.542}; +const double ROW_CENTER_Y[5] = {0.154, 0.280, 0.404, 0.528, 0.652}; +const double STRIP_WIDTH = 0.068; +const double STRIP_HEIGHT = 0.056; + +// Buttons are flat: the frame's measured stddev sum is 2 to 6. +const double MAX_STDDEV_SUM = 15; + +enum class ButtonLook{ GREY, BLUE, OTHER }; + +ImageFloatBox button_strip(size_t item){ + return ImageFloatBox(COLUMN_X[item % 2], ROW_CENTER_Y[item / 2] - STRIP_HEIGHT / 2, STRIP_WIDTH, STRIP_HEIGHT); +} + +// Grey (225, 233, 238) and blue (167, 206, 255) on the frame. The background +// between the buttons is a bluer (203, 232, 253), not grey. +ButtonLook button_look(const ImageViewRGB32& screen, size_t item){ + const ImageStats stats = image_stats(extract_box_reference(screen, button_strip(item))); + if (stats.stddev.sum() > MAX_STDDEV_SUM){ + return ButtonLook::OTHER; + } + const FloatPixel& c = stats.average; + const double low = std::min({c.r, c.g, c.b}); + const double high = std::max({c.r, c.g, c.b}); + if (low >= 200 && high - low <= 25){ + return ButtonLook::GREY; + } + if (c.b >= 230 && c.r >= 120 && c.b - c.r >= 50){ + return ButtonLook::BLUE; + } + return ButtonLook::OTHER; +} + +} // namespace + + +std::optional language_select_cursor(const ImageViewRGB32& screen){ + std::optional cursor; + for (size_t item = 0; item < LANGUAGE_SELECT_BUTTONS; item++){ + switch (button_look(screen, item)){ + case ButtonLook::GREY: + break; + case ButtonLook::BLUE: + if (cursor){ + return std::nullopt; + } + cursor = item; + break; + case ButtonLook::OTHER: + return std::nullopt; + } + } + return cursor; +} + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h new file mode 100644 index 0000000000..29524a35c4 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h @@ -0,0 +1,36 @@ +/* BDSP AutoStory - Language Select Detector + * + * From: https://github.com/PokemonAutomation/ + * + * BDSP's language select, shown on a new game's first boot: ten buttons in + * two columns of five over a crystal background, "Select the language you + * would like to play in." below them. The cursor's button is blue with a red + * outline, the others light grey (lab-B 2026-11-09, 1080p). + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_LanguageSelectDetector_H +#define PokemonAutomation_PokemonBDSP_AutoStory_LanguageSelectDetector_H + +#include +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// The buttons, numbered row by row: English 0, its right neighbour 1, the +// next row 2 and 3, and so on (BDSP_LANGUAGE_GRID). +inline constexpr size_t LANGUAGE_SELECT_BUTTONS = 10; + +// The button the cursor is on, or nullopt when `screen` is not the language +// select: every button must read as plain grey but exactly one, blue. +std::optional language_select_cursor(const ImageViewRGB32& screen); + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp index 16f5309bbe..3684ac7733 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp @@ -166,23 +166,6 @@ void verify_story_checkpoint( } // namespace -namespace{ - -// The languages BDSP's language select lists, in its order. -const EnumDropdownDatabase& switch_language_database(){ - static const EnumDropdownDatabase database = []{ - EnumDropdownDatabase ret; - for (Language language : bdsp_language_menu()){ - ret.add(language, language_data(language).code, language_data(language).name, true); - } - return ret; - }(); - return database; -} - -} // namespace - - BDSPAutoStory::BDSPAutoStory() : RUN(checkpoint_database()) , NEW_GAME_INTRO( @@ -198,10 +181,6 @@ BDSPAutoStory::BDSPAutoStory() LockMode::LOCK_WHILE_RUNNING, false ) , PLAYER_NAME(false, "Player name:
For a new game. Letters and digits.", LockMode::LOCK_WHILE_RUNNING, "Kosmos", "Kosmos") - , SWITCH_LANGUAGE( - "Switch language:
For a new game: the Switch's system language, where BDSP's language select starts its cursor.", - switch_language_database(), LockMode::LOCK_WHILE_RUNNING, Language::English - ) , PLAYER_LOOK( "Player's look:
For a new game: the look picked on the appearance grid. " "Models 1, 2, 5 and 6 are Lucas; 3, 4, 7 and 8 are Dawn." @@ -226,7 +205,6 @@ BDSPAutoStory::BDSPAutoStory() PA_ADD_OPTION(NEW_GAME_INTRO); PA_ADD_OPTION(JUST_LOADED); PA_ADD_OPTION(PLAYER_NAME); - PA_ADD_OPTION(SWITCH_LANGUAGE); PA_ADD_OPTION(PLAYER_LOOK); PA_ADD_OPTION(STARTER); PA_ADD_OPTION(NOTIFICATIONS); @@ -256,7 +234,7 @@ void BDSPAutoStory::program(SingleSwitchProgramEnvironment& env, ProControllerCo // The reads that use text (Location.h) take the game's language from here. set_story_language(RUN.language()); - StoryEnv story{env, context, STARTER, PLAYER_NAME, IntroPicks{SWITCH_LANGUAGE, RUN.language(), PLAYER_LOOK.index()}}; + StoryEnv story{env, context, STARTER, PLAYER_NAME, IntroPicks{RUN.language(), PLAYER_LOOK.index()}}; StoryExecutor executor(story); executor.set_checks(RUN.checks()); executor.set_language(RUN.language()); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h index 753bdde767..71a1a45f7d 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h @@ -50,7 +50,6 @@ class BDSPAutoStory : public SingleSwitchProgramInstance{ BooleanCheckBoxOption NEW_GAME_INTRO; BooleanCheckBoxOption JUST_LOADED; StringOption PLAYER_NAME; - EnumDropdownOption SWITCH_LANGUAGE; PlayerModelOption PLAYER_LOOK; EnumDropdownOption STARTER; diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 727a6d1b23..27c7a3f56d 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1708,6 +1708,8 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.cpp From f84574a543022ead2e578b6cfd3f14677559566e Mon Sep 17 00:00:00 2001 From: dayfine Date: Mon, 9 Nov 2026 11:15:32 -0800 Subject: [PATCH 3/6] BDSP AutoStory: language select test reads each screenshot it finds A missing negative screenshot no longer skips the whole test, and a black screen must not read as the language select. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index 5283dcb2c4..4800c7fac1 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -347,8 +347,13 @@ UnitTestResult test_intro_pick_steps(Logger&){ // The language select from lab-B 2026-11-09 (cursor on English), and screens // that are not it. Screenshots are in the unit test resources; a missing one -// is skipped. +// is passed over, and the test is skipped when all are. UnitTestResult test_language_select_screenshots(Logger& logger){ + ImageRGB32 black(1920, 1080); + black.fill(0xff000000); + if (language_select_cursor(black)){ + return "A black screen read as the language select."; + } struct Case{ const char* file; std::optional cursor; @@ -358,17 +363,23 @@ UnitTestResult test_language_select_screenshots(Logger& logger){ {.file = "PokemonBDSP/AutoStory/OreburghCenterCounter-20261108-094735.png", .cursor = std::nullopt}, {.file = "PokemonBDSP/AutoStory/OreburghCenterDoorstep-20261108.png", .cursor = std::nullopt}, }; + size_t read = 0; for (const Case& c : cases){ const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; if (!Filesystem::is_regular_file(path)){ - return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + logger.log("Screenshot not found: " + path); + continue; } + read++; const std::optional cursor = language_select_cursor(ImageRGB32(path)); logger.log(std::string(c.file) + ": cursor " + (cursor ? std::to_string(*cursor) : "none")); if (cursor != c.cursor){ return std::string(c.file) + ": wrong language select read."; } } + if (read == 0){ + return UnitTestResult(UnitTestResult::SKIPPED, "No language select screenshots found."); + } return true; } From 69a7c37794cbb642aeda7aa7cbd285f3184a966e Mon Sep 17 00:00:00 2001 From: dayfine Date: Mon, 9 Nov 2026 11:27:05 -0800 Subject: [PATCH 4/6] BDSP AutoStory: read the look select with a detector The Right/Left probe never saw the look select's cursor move (console B, 2026-11-09: 20 probes, then the name keyboard). The look select is a 2 x 4 grid of portraits, the cursor's background blue and the others grey, like the language select's buttons (lab-B frame). The language detector becomes IntroMenuDetector, one card-grid reader with a layout per menu: language buttons, and two background strips beside each portrait's face. The intro classifies the look select as its own screen and picks on it as on the language select: read the cursor, move to the chosen look, read again (3 tries), then A. Model N is portrait N - 1, row by row, matching PlayerModelOption's icons: 1 Lucas light, 2 Lucas dark, 3 Dawn dark-haired, 4 Dawn dark; 5 Lucas blond, 6 Lucas brown, 7 Dawn blond, 8 Dawn brown-haired. The option text says so. The D-pad probe is gone from MenuCursor: nothing uses it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../MenuCursor/NintendoSwitch_MenuCursor.cpp | 34 ----- .../MenuCursor/NintendoSwitch_MenuCursor.h | 31 +--- .../NintendoSwitch_MenuCursor_Tests.cpp | 29 ---- .../NintendoSwitch_MenuCursor_Tests.h | 2 +- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 82 +++++++---- .../Actions/PokemonBDSP_AutoStory_NewGame.cpp | 137 +++++++----------- .../Actions/PokemonBDSP_AutoStory_NewGame.h | 28 ++-- ...okemonBDSP_AutoStory_IntroMenuDetector.cpp | 118 +++++++++++++++ .../PokemonBDSP_AutoStory_IntroMenuDetector.h | 48 ++++++ ...nBDSP_AutoStory_LanguageSelectDetector.cpp | 79 ---------- ...monBDSP_AutoStory_LanguageSelectDetector.h | 36 ----- .../AutoStory/PokemonBDSP_AutoStory.cpp | 3 +- SerialPrograms/cmake/SourceFiles.cmake | 4 +- 13 files changed, 293 insertions(+), 338 deletions(-) create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h delete mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp delete mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp index 17dc468105..d90e5fc5d1 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp @@ -4,8 +4,6 @@ * */ -#include "CommonFramework/ImageTools/ImageDiff.h" -#include "CommonFramework/VideoPipeline/VideoFeed.h" #include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" #include "NintendoSwitch_MenuCursor.h" @@ -28,14 +26,6 @@ void add_sideways(std::vector& moves, size_t from_column, size_t t } } -// 0 when either frame is missing: a probe without frames sees nothing move. -double screen_change(const VideoSnapshot& before, const VideoSnapshot& after){ - if (!before || !after){ - return 0; - } - return ImageMatch::pixel_RMSD(*before.frame, *after.frame); -} - } // namespace @@ -61,30 +51,6 @@ std::optional> menu_cursor_moves(const MenuGrid& grid, } -bool cursor_probe_moved(const CursorProbe& probe){ - return probe.press_change >= CURSOR_PRESS_CHANGE && - probe.press_change >= CURSOR_NOISE_RATIO * probe.still_change && - 2 * probe.back_change <= probe.press_change; -} - -CursorProbe probe_cursor(VideoStream& stream, ProControllerContext& context, DpadPosition there, DpadPosition back){ - context.wait_for_all_requests(); - const VideoSnapshot first = stream.video().snapshot(); - context.wait_for(CURSOR_PROBE_WAIT); - const VideoSnapshot before = stream.video().snapshot(); - pbf_press_dpad(context, there, 80ms, CURSOR_PROBE_WAIT); - context.wait_for_all_requests(); - const VideoSnapshot pressed = stream.video().snapshot(); - pbf_press_dpad(context, back, 80ms, CURSOR_PROBE_WAIT); - context.wait_for_all_requests(); - const VideoSnapshot returned = stream.video().snapshot(); - return CursorProbe{ - screen_change(first, before), - screen_change(before, pressed), - screen_change(before, returned), - }; -} - void press_cursor_moves(ProControllerContext& context, const std::vector& moves){ for (DpadPosition move : moves){ pbf_press_dpad(context, move, 80ms, 320ms); diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h index fd3ad96de6..077dfd0175 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h @@ -4,8 +4,7 @@ * * Moves the cursor of a fixed list or grid (a game's language select, the * player's look) to an item by its index. The menu's shape is data; the - * presses follow from it. A D-pad probe tells whether a screen has a - * cursor at all, for menus no detector knows. + * presses follow from it; a detector for the menu reads where the cursor is. * */ @@ -15,7 +14,6 @@ #include #include #include -#include "CommonFramework/Tools/VideoStream.h" #include "NintendoSwitch/Controllers/NintendoSwitch_ControllerButtons.h" #include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" @@ -36,33 +34,6 @@ struct MenuGrid{ // the grid. std::optional> menu_cursor_moves(const MenuGrid& grid, size_t from, size_t to); - -// How much the screen changed (pixel RMSD) around a D-pad probe, each against -// the frame just before the press: over a wait with no input, after the press, -// and after the opposite press. -struct CursorProbe{ - double still_change; - double press_change; - double back_change; -}; -// The least change a press that moved a cursor makes. -inline constexpr double CURSOR_PRESS_CHANGE = 6; -// How many times what the screen changes on its own (an animated preview, a -// blinking arrow) a press that moved a cursor changes it. -inline constexpr double CURSOR_NOISE_RATIO = 3; -// Whether `probe` saw a cursor move and come back: the press changed the -// screen by CURSOR_PRESS_CHANGE and CURSOR_NOISE_RATIO times the still -// change, and the opposite press put back at least half of it. -bool cursor_probe_moved(const CursorProbe& probe); - -// The wait before a probe's first press, and after each press. -inline constexpr Milliseconds CURSOR_PROBE_WAIT = Milliseconds(400); -// Waits with no input, presses `there`, waits, presses `back`, waits, and -// measures the screen at each step. On a screen with no cursor `there` and -// `back` do nothing; on one with a cursor they leave it where it was, unless -// `there` meets an edge that does not wrap. -CursorProbe probe_cursor(VideoStream& stream, ProControllerContext& context, DpadPosition there, DpadPosition back); - // Presses `moves` in order, one D-pad tap each. void press_cursor_moves(ProControllerContext& context, const std::vector& moves); diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp index 77bc1152e2..71ada25348 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp @@ -106,31 +106,6 @@ UnitTestResult test_cursor_moves_stay_inside(){ return true; } -UnitTestResult test_cursor_probe(){ - struct Case{ - const char* what; - CursorProbe probe; - bool moved; - }; - const Case cases[] = { - {"a still list: the highlight moves and comes back", {0.5, 20, 0.5}, true}, - {"a small Yes/No box", {0.3, 8, 0.4}, true}, - {"an animated preview, swapped by the press", {8, 40, 10}, true}, - {"dialogue with a blinking arrow", {2.5, 3, 2.5}, false}, - {"an animation that changes as much on its own", {10, 20, 10}, false}, - {"a press that moved something it did not put back", {0.5, 20, 15}, false}, - {"a fade", {20, 25, 45}, false}, - {"no frames", {0, 0, 0}, false}, - }; - for (const Case& c : cases){ - if (cursor_probe_moved(c.probe) != c.moved){ - return std::string(c.what) + (c.moved ? ": the cursor move was missed." : ": no cursor moved."); - } - } - return true; -} - - struct MenuCursorInput{ std::string name; UnitTestResult (*body)(); @@ -167,10 +142,6 @@ const MenuCursorCase MENU_CURSOR_CASES[] = { .input = {"presses stay inside the grid", test_cursor_moves_stay_inside}, .expectation = {}, }, - { - .input = {"probe judgement", test_cursor_probe}, - .expectation = {}, - }, }; diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h index ce92acd5dc..10b6131cb3 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h @@ -13,7 +13,7 @@ namespace PokemonAutomation{ namespace NintendoSwitch{ -// The cursor presses for list and grid items, and the probe's judgement. No console. +// The cursor presses for list and grid items. No console. void add_tests_MenuCursor(UnitTestDatabase& database); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index 4800c7fac1..6ee60be3a5 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -23,7 +23,8 @@ #include "NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h" #include "PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_StoryMaps.h" #include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" -#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h" +#include "PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h" #include "PokemonBDSP_AutoStory_Center.h" #include "PokemonBDSP_AutoStory_NewGame.h" #include "PokemonBDSP_AutoStory_Actions_Tests.h" @@ -252,6 +253,17 @@ UnitTestResult test_intro_gate(Logger&){ } +// The menus' data, grids, detectors and the look option agree on their sizes. +UnitTestResult check_intro_menu_sizes(){ + if (bdsp_language_menu().size() != BDSP_LANGUAGE_GRID.count || BDSP_LANGUAGE_GRID.count != LANGUAGE_SELECT_BUTTONS){ + return "The language select's data, grid and detector must agree on ten buttons."; + } + if (BDSP_LOOK_GRID.count != LOOK_SELECT_PORTRAITS || BDSP_LOOK_GRID.count != BDSP_PLAYER_MODEL_COUNT){ + return "The look grid, its detector and PlayerModelOption must agree on eight looks."; + } + return true; +} + // The presses for the language select (two columns of five, read row by // row, from wherever its cursor is) and the appearance grid (two rows of // four, from the first look). @@ -282,28 +294,28 @@ UnitTestResult test_intro_pick_moves(Logger&){ {"a cursor outside the menu", language_pick_moves(LANGUAGE_SELECT_BUTTONS, french), std::nullopt}, {"Model 1, where the cursor starts", - look_pick_moves(IntroPicks{Language::English, 0}), Moves(std::vector{})}, + look_pick_moves(0, IntroPicks{Language::English, 0}), Moves(std::vector{})}, {"Model 3, Dawn's first look (lab run 3)", - look_pick_moves(french), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT})}, + look_pick_moves(0, french), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT})}, + {"Model 3 from Model 8", + look_pick_moves(7, french), Moves(std::vector{DPAD_UP, DPAD_LEFT})}, {"Model 8, the last", - look_pick_moves(IntroPicks{Language::English, 7}), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT, DPAD_RIGHT, DPAD_DOWN})}, + look_pick_moves(0, IntroPicks{Language::English, 7}), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT, DPAD_RIGHT, DPAD_DOWN})}, {"no Model 9", - look_pick_moves(IntroPicks{Language::English, 8}), std::nullopt}, + look_pick_moves(0, IntroPicks{Language::English, 8}), std::nullopt}, + {"a cursor outside the look grid", + look_pick_moves(LOOK_SELECT_PORTRAITS, french), std::nullopt}, }; for (const Case& c : cases){ if (c.actual != c.expected){ return std::string("Wrong presses for ") + c.what + "."; } } - if (bdsp_language_menu().size() != BDSP_LANGUAGE_GRID.count || BDSP_LANGUAGE_GRID.count != LANGUAGE_SELECT_BUTTONS){ - return "The language select's data, grid and detector must agree on ten buttons."; - } - return true; + return check_intro_menu_sizes(); } -// The language select takes the language pick, then A if it still shows. -// Until the look is picked, every screen after the language that would take -// A gets a probe first. The player's keyboard with a pick still to make fails. +// Each select takes its pick, then A if it still shows. Other screens take +// A. The player's keyboard with a pick still to make fails. UnitTestResult test_intro_pick_steps(Logger&){ struct Case{ const char* what; @@ -317,8 +329,10 @@ UnitTestResult test_intro_pick_steps(Logger&){ {"the language select, picked", "language-select", false, true, IntroStep::ADVANCE}, {"a logo before the language", "other", true, true, IntroStep::ADVANCE}, {"a dialogue before the language", "dialog", true, true, IntroStep::ADVANCE}, - {"Rowan, look to pick", "dialog", false, true, IntroStep::PROBE_FOR_LOOK}, - {"the preview, look to pick", "other", false, true, IntroStep::PROBE_FOR_LOOK}, + {"the look select", "look-select", false, true, IntroStep::PICK_LOOK}, + {"the look select, picked", "look-select", false, false, IntroStep::ADVANCE}, + {"Rowan, look to pick", "dialog", false, true, IntroStep::ADVANCE}, + {"a cutscene, look to pick", "other", false, true, IntroStep::ADVANCE}, {"both picked", "dialog", false, false, IntroStep::ADVANCE}, {"black, look to pick", "black", false, true, IntroStep::WAIT}, {"the user select", "user-select", true, true, IntroStep::SELECT_USER}, @@ -345,23 +359,28 @@ UnitTestResult test_intro_pick_steps(Logger&){ return true; } -// The language select from lab-B 2026-11-09 (cursor on English), and screens -// that are not it. Screenshots are in the unit test resources; a missing one -// is passed over, and the test is skipped when all are. -UnitTestResult test_language_select_screenshots(Logger& logger){ +// The intro's two selects from lab-B 2026-11-09 (each cursor on its first +// item), the screens one A before and after the look select, and screens +// that are neither. Screenshots are in the unit test resources; a missing +// one is passed over, and the test is skipped when all are. +UnitTestResult test_intro_menu_screenshots(Logger& logger){ ImageRGB32 black(1920, 1080); black.fill(0xff000000); - if (language_select_cursor(black)){ - return "A black screen read as the language select."; + if (language_select_cursor(black) || look_select_cursor(black)){ + return "A black screen read as an intro menu."; } struct Case{ const char* file; - std::optional cursor; + std::optional language; + std::optional look; }; const Case cases[] = { - {.file = "PokemonBDSP/AutoStory/LanguageSelect-20261109.png", .cursor = 0}, - {.file = "PokemonBDSP/AutoStory/OreburghCenterCounter-20261108-094735.png", .cursor = std::nullopt}, - {.file = "PokemonBDSP/AutoStory/OreburghCenterDoorstep-20261108.png", .cursor = std::nullopt}, + {.file = "PokemonBDSP/AutoStory/LanguageSelect-20261109.png", .language = 0, .look = std::nullopt}, + {.file = "PokemonBDSP/AutoStory/LookSelect-20261109.png", .language = std::nullopt, .look = 0}, + {.file = "PokemonBDSP/AutoStory/LookSelectBefore-20261109.png", .language = std::nullopt, .look = std::nullopt}, + {.file = "PokemonBDSP/AutoStory/LookSelectAfterA-20261109.png", .language = std::nullopt, .look = std::nullopt}, + {.file = "PokemonBDSP/AutoStory/OreburghCenterCounter-20261108-094735.png", .language = std::nullopt, .look = std::nullopt}, + {.file = "PokemonBDSP/AutoStory/OreburghCenterDoorstep-20261108.png", .language = std::nullopt, .look = std::nullopt}, }; size_t read = 0; for (const Case& c : cases){ @@ -371,14 +390,17 @@ UnitTestResult test_language_select_screenshots(Logger& logger){ continue; } read++; - const std::optional cursor = language_select_cursor(ImageRGB32(path)); - logger.log(std::string(c.file) + ": cursor " + (cursor ? std::to_string(*cursor) : "none")); - if (cursor != c.cursor){ - return std::string(c.file) + ": wrong language select read."; + const ImageRGB32 screen(path); + const std::optional language = language_select_cursor(screen); + const std::optional look = look_select_cursor(screen); + logger.log(std::string(c.file) + ": language " + (language ? std::to_string(*language) : "none") + + ", look " + (look ? std::to_string(*look) : "none")); + if (language != c.language || look != c.look){ + return std::string(c.file) + ": wrong intro menu read."; } } if (read == 0){ - return UnitTestResult(UnitTestResult::SKIPPED, "No language select screenshots found."); + return UnitTestResult(UnitTestResult::SKIPPED, "No intro menu screenshots found."); } return true; } @@ -629,7 +651,7 @@ const AutoStoryActionsCase AUTO_STORY_ACTIONS_CASES[] = { .expectation = {}, }, { - .input = {"language select screenshots", test_language_select_screenshots}, + .input = {"intro menu screenshots", test_intro_menu_screenshots}, .expectation = {}, }, { diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp index 540ccb9153..43aac240c0 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp @@ -22,7 +22,7 @@ #include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" #include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" #include "PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h" -#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h" #include "PokemonBDSP_AutoStory_NewGame.h" namespace PokemonAutomation{ @@ -63,8 +63,8 @@ std::optional> language_pick_moves(size_t cursor, cons } return menu_cursor_moves(BDSP_LANGUAGE_GRID, cursor, *to); } -std::optional> look_pick_moves(const IntroPicks& picks){ - return menu_cursor_moves(BDSP_LOOK_GRID, 0, picks.look); +std::optional> look_pick_moves(size_t cursor, const IntroPicks& picks){ + return menu_cursor_moves(BDSP_LOOK_GRID, cursor, picks.look); } IntroProgress intro_start(const IntroPicks& picks){ @@ -90,6 +90,9 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& if (language_select_cursor(screen)){ return "language-select"; } + if (look_select_cursor(screen)){ + return "look-select"; + } switch (NameListDetector().selected_row(screen)){ case -1: break; @@ -113,23 +116,6 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& return "other"; } -namespace{ - -// A on a screen that may take A, after a D-pad probe while the look waits -// for its grid. The language select comes first, then the look, then the -// player's name. -IntroStep advance_or_probe(const IntroProgress& progress){ - if (progress.player_named){ - return IntroStep::ADVANCE; - } - if (!progress.language_pending && progress.look_pending){ - return IntroStep::PROBE_FOR_LOOK; - } - return IntroStep::ADVANCE; -} - -} // namespace - IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress){ if (screen == "user-select"){ return IntroStep::SELECT_USER; @@ -147,6 +133,9 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog // Once picked, A again only if the select still shows. return progress.language_pending ? IntroStep::PICK_LANGUAGE : IntroStep::ADVANCE; } + if (screen == "look-select"){ + return progress.look_pending ? IntroStep::PICK_LOOK : IntroStep::ADVANCE; + } if (screen == "rival-list-tell"){ return IntroStep::MOVE_TO_PRESET; } @@ -160,7 +149,7 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog return IntroStep::WAIT; } if (screen == "dialog"){ - return progress.rival_named ? IntroStep::ADVANCE_WITH_B : advance_or_probe(progress); + return progress.rival_named ? IntroStep::ADVANCE_WITH_B : IntroStep::ADVANCE; } // The bedroom comes only after the rival is named. Before that, every unknown // screen (logos, the title, language, the appearance preview, cutscenes) @@ -172,10 +161,10 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog if (!progress.game_confirmed && progress.unconfirmed_for >= INTRO_TIME_BEFORE_BDSP){ return IntroStep::FAIL_NOT_BDSP; } - return advance_or_probe(progress); + return IntroStep::ADVANCE; } bool intro_screen_confirms_bdsp(const std::string& screen){ - return screen == "dialog" || screen == "language-select" || screen == "rival-list-tell" || screen == "rival-list-preset" || screen == "menu"; + return screen == "dialog" || screen == "language-select" || screen == "look-select" || screen == "rival-list-tell" || screen == "rival-list-preset" || screen == "menu"; } void note_intro_screen(IntroProgress& progress, const std::string& screen, std::chrono::milliseconds since_start){ @@ -355,79 +344,60 @@ void advance(IntroRun& run){ pbf_press_button(run.context, BUTTON_A, 60ms, 400ms); } -// Probes the screen with `there` and `back`; whether a cursor moved. -bool probe_moves(IntroRun& run, DpadPosition there, DpadPosition back){ - const CursorProbe probe = probe_cursor(run.env.console, run.context, there, back); - const bool moved = cursor_probe_moved(probe); - run.trace.action("Cursor probe " + dpad_to_string(there) + ": still " + std::to_string(probe.still_change) + - ", press " + std::to_string(probe.press_change) + ", back " + std::to_string(probe.back_change) + - (moved ? ": a cursor moved." : ": no cursor.")); - return moved; -} - -// Moves the cursor with `moves`, saves a screenshot of it on the pick, and confirms with A. -void pick_on_menu(IntroRun& run, const std::vector& moves, const std::string& what){ - run.trace.action("Picking " + what + ": " + std::to_string(moves.size()) + " D-pad presses, then A."); - press_cursor_moves(run.context, moves); - run.context.wait_for_all_requests(); - run.trace.screen("pick-" + what); - press(run, BUTTON_A); - pbf_wait(run.context, 500ms); -} +// One of the intro's two menus: its shape and its cursor reader. +struct IntroMenu{ + const char* what; + MenuGrid grid; + std::optional (*read)(const ImageViewRGB32& screen); +}; +const IntroMenu LANGUAGE_MENU{"language", BDSP_LANGUAGE_GRID, language_select_cursor}; +const IntroMenu LOOK_MENU{"look", BDSP_LOOK_GRID, look_select_cursor}; -// Reads of the language select before the cursor must be on the game's language. -const size_t LANGUAGE_PICK_TRIES = 3; +// Reads of a menu before its cursor must be on the pick. +const size_t INTRO_PICK_TRIES = 3; -// The language select's cursor as read now; nullopt when it is not showing. -std::optional read_language_cursor(IntroRun& run){ +// The menu's cursor as read now; nullopt when the menu is not showing. +std::optional read_cursor(IntroRun& run, const IntroMenu& menu){ run.context.wait_for_all_requests(); const VideoSnapshot screen = run.env.console.video().snapshot(); if (!screen){ return std::nullopt; } - return language_select_cursor(*screen.frame); + return menu.read(*screen.frame); } -// Reads the cursor, moves it to the game's language and reads it there, -// up to LANGUAGE_PICK_TRIES times (a press dropped as the screen fades in), -// then A. -void pick_language(IntroRun& run){ - const std::optional target = language_item(run.picks.game_language); - for (size_t attempt = 0; attempt < LANGUAGE_PICK_TRIES; attempt++){ - const std::optional cursor = read_language_cursor(run); +// Reads the cursor, moves it to `target` and reads it there, up to +// INTRO_PICK_TRIES times (a press dropped as the screen fades in), then +// saves a screenshot and presses A. Whether it picked: false when the menu +// went away, and the next read decides. +bool pick_on_grid(IntroRun& run, const IntroMenu& menu, size_t target){ + for (size_t attempt = 0; attempt < INTRO_PICK_TRIES; attempt++){ + const std::optional cursor = read_cursor(run, menu); if (!cursor){ - return; // Gone or fading: the next read decides. + return false; } - run.trace.action("Language select: cursor on item " + std::to_string(*cursor) + - ", the game's language is item " + std::to_string(*target) + "."); - if (*cursor == *target){ - pick_on_menu(run, {}, "language"); - run.progress.language_pending = false; - return; + run.trace.action(std::string("Picking the ") + menu.what + ": cursor on item " + std::to_string(*cursor) + + ", the pick is item " + std::to_string(target) + "."); + if (*cursor == target){ + run.trace.screen(std::string("pick-") + menu.what); + press(run, BUTTON_A); + pbf_wait(run.context, 500ms); + return true; } - press_cursor_moves(run.context, *language_pick_moves(*cursor, run.picks)); + press_cursor_moves(run.context, *menu_cursor_moves(menu.grid, *cursor, target)); } - run.trace.fail("The language select's cursor did not reach the game's language in " + - std::to_string(LANGUAGE_PICK_TRIES) + " tries."); + run.trace.fail(std::string("The ") + menu.what + " menu's cursor did not reach its pick in " + + std::to_string(INTRO_PICK_TRIES) + " tries."); } -// The appearance grid's cursor starts on the first look, so Right moves it. -// The Yes/No boxes around it do not move sideways. -void probe_for_look(IntroRun& run){ - if (!probe_moves(run, DPAD_RIGHT, DPAD_LEFT)){ - advance(run); - return; - } - pick_on_menu(run, *look_pick_moves(run.picks), "look"); - run.progress.look_pending = false; -} - -// PICK_LANGUAGE or PROBE_FOR_LOOK. +// PICK_LANGUAGE or PICK_LOOK. void pick_on_intro_menu(IntroRun& run, IntroStep step){ if (step == IntroStep::PICK_LANGUAGE){ - pick_language(run); - }else{ - probe_for_look(run); + if (pick_on_grid(run, LANGUAGE_MENU, *language_item(run.picks.game_language))){ + run.progress.language_pending = false; + } + }else if (pick_on_grid(run, LOOK_MENU, run.picks.look)){ + run.progress.look_pending = false; } } @@ -451,7 +421,7 @@ IntroStep next_intro_step(IntroRun& run, const char* screen){ } run.trace.fail(std::string("The player's name keyboard showed before the ") + (run.progress.language_pending ? "language select" : "appearance grid") + - " was found: no cursor probe above saw its cursor move."); + " was read."); } // Carries out what decide_intro_step() chose for `screen`. @@ -468,7 +438,7 @@ void act(IntroRun& run, const char* screen){ case IntroStep::FAIL_PICK_MISSED: fail_intro(run, step); case IntroStep::PICK_LANGUAGE: - case IntroStep::PROBE_FOR_LOOK: + case IntroStep::PICK_LOOK: pick_on_intro_menu(run, step); return; case IntroStep::ENTER_PLAYER_NAME: @@ -547,6 +517,9 @@ std::vector intro_states(IntroRun& run){ {"language-select", [=](const ImageViewRGB32& s){ return screen_is(s, "language-select"); }, [&run]{ act(run, "language-select"); }, {}, 10}, + {"look-select", + [=](const ImageViewRGB32& s){ return screen_is(s, "look-select"); }, + [&run]{ act(run, "look-select"); }, {}, 10}, {"rival-list-tell", [=](const ImageViewRGB32& s){ return screen_is(s, "rival-list-tell"); }, [&run]{ act(run, "rival-list-tell"); }, {"rival-list-preset", "black", "other"}, 6}, @@ -585,7 +558,7 @@ void play_new_game_intro( if (!language_pick_moves(0, picks)){ throw UserSetupError(env.console, "BDSP's language select lacks the game's language: pick one it lists."); } - if (!look_pick_moves(picks)){ + if (!look_pick_moves(0, picks)){ throw UserSetupError(env.console, "The player's look must be one of BDSP's eight."); } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h index d3e8516d24..d4a7b57e2b 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h @@ -6,8 +6,7 @@ * the bedroom with the player in control, which is where the Tutorial starts. * The language and the player's look are picked from their menus by index * (NintendoSwitch_MenuCursor.h); the menus' shapes are the data below. - * The language select is read by its detector; the appearance grid, which - * has none yet, is found with a D-pad probe. + * Both menus are read by their detector (PokemonBDSP_AutoStory_IntroMenuDetector.h). * */ @@ -37,9 +36,11 @@ inline constexpr MenuGrid BDSP_LANGUAGE_GRID{2, 10}; // The language of each item of BDSP_LANGUAGE_GRID. Japanese is there twice; // its first item, the kana, is the one picked. const std::vector& bdsp_language_menu(); -// The appearance grid: two rows of four looks, the cursor on the first. -// PlayerModelOption's "Model N" is item N - 1: Lucas's two first looks, -// then Dawn's, on each row. +// The look select ("Choose your photo."): two rows of four portraits, the +// cursor on the first (lab-B 2026-11-09). PlayerModelOption's "Model N" is +// item N - 1, row by row: Lucas light, Lucas dark, Dawn dark-haired, Dawn +// dark; Lucas blond, Lucas brown, Dawn blond, Dawn brown-haired. A on it +// goes straight to the name keyboard. inline constexpr MenuGrid BDSP_LOOK_GRID{4, 8}; // What the intro picks on its menus. @@ -51,13 +52,13 @@ struct IntroPicks{ // The presses from the language select's item `cursor` to the game's // language; nullopt when the menu lacks it or `cursor` is outside it. std::optional> language_pick_moves(size_t cursor, const IntroPicks& picks); -// The presses from the first look to the chosen one; nullopt when it is -// outside the grid. -std::optional> look_pick_moves(const IntroPicks& picks); +// The presses from the look select's item `cursor` to the chosen look; +// nullopt when either is outside the grid. +std::optional> look_pick_moves(size_t cursor, const IntroPicks& picks); // Which intro screen `screen` shows, as the intro's steps tell them apart: -// "user-select", "home", "keyboard", "language-select", "rival-list-tell" (the rival's name list on +// "user-select", "home", "keyboard", "language-select", "look-select", "rival-list-tell" (the rival's name list on // "I'll tell you..."), "rival-list-preset" (on a preset name), "menu", "dialog" or // "black" (the boot, fades) or "other". The Switch's own screens need // `console`; without it they read as whatever else they look like. @@ -71,7 +72,7 @@ enum class IntroStep{ FAIL_NOT_BDSP, // no BDSP screen within INTRO_TIME_BEFORE_BDSP FAIL_PICK_MISSED, // the player's keyboard, with a pick still to make PICK_LANGUAGE, // the language select: move to the game's language, A - PROBE_FOR_LOOK, // D-pad probe: a grid takes the look pick, else A + PICK_LOOK, // the look select: move to the chosen look, A ENTER_PLAYER_NAME, // type the name on the first keyboard CANCEL_RIVAL_KEYBOARD, // X: back to the rival's name list MOVE_TO_PRESET, // Down: off "I'll tell you..." @@ -119,7 +120,7 @@ inline constexpr std::chrono::milliseconds INTRO_TIME_BEFORE_BDSP = std::chrono: // How long the boot may stay black after the restart. inline constexpr std::chrono::milliseconds INTRO_BOOT_LIMIT = std::chrono::seconds(60); // Whether `screen` (classify_intro_screen()) is one only BDSP's intro shows: -// its dialogue box, its language select, the rival's name list, its main menu. The keyboard and +// its dialogue box, its language and look selects, the rival's name list, its main menu. The keyboard and // the user select are the Switch's. bool intro_screen_confirms_bdsp(const std::string& screen); @@ -177,9 +178,8 @@ IntroProgress intro_start(const IntroPicks& picks); // INTRO_BOOT_LIMIT) without pressing anything, and fails if no BDSP screen shows within // INTRO_TIME_BEFORE_BDSP of screens. Picks the language and the look in // `picks`, names the player `player_name` and picks the rival's default name -// (Barry) from the list. The language select is read, moved on and read -// again before its A. Until the look is picked, each unknown screen after it -// gets a D-pad probe before its A. The player's keyboard showing with a pick +// (Barry) from the list. The language and look selects are read, moved on +// and read again before their A. The player's keyboard showing with a pick // still to make fails. Each step reads the // screen and waits for the expected next one; an unexpected screen fails with a // screenshot and the recent screens. diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.cpp new file mode 100644 index 0000000000..14be3abf8a --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.cpp @@ -0,0 +1,118 @@ +/* BDSP AutoStory - Intro Menu Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "PokemonBDSP_AutoStory_IntroMenuDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ + +// Cards are flat: the frames' strips have stddev sums of 2 to 18. +const double MAX_STDDEV_SUM = 25; + +enum class CardLook{ GREY, BLUE, OTHER }; + +// Grey on the frames: (225, 233, 238) language buttons, (210, 218, 223) +// portraits. Blue: (167, 206, 255) and (152, 195, 251). The background +// between the cards is a bluer (203, 232, 253), neither. +CardLook strip_look(const ImageViewRGB32& screen, const ImageFloatBox& strip){ + const ImageStats stats = image_stats(extract_box_reference(screen, strip)); + if (stats.stddev.sum() > MAX_STDDEV_SUM){ + return CardLook::OTHER; + } + const FloatPixel& c = stats.average; + const double low = std::min({c.r, c.g, c.b}); + const double high = std::max({c.r, c.g, c.b}); + if (low >= 195 && high - low <= 25){ + return CardLook::GREY; + } + if (c.b >= 230 && c.r >= 120 && c.b - c.r >= 50){ + return CardLook::BLUE; + } + return CardLook::OTHER; +} + +// A card's look: all its strips must agree. +CardLook card_look(const ImageViewRGB32& screen, const std::vector& strips){ + const CardLook first = strip_look(screen, strips[0]); + for (size_t c = 1; c < strips.size(); c++){ + if (strip_look(screen, strips[c]) != first){ + return CardLook::OTHER; + } + } + return first; +} + +// The one blue card among grey ones; nullopt otherwise. +std::optional grid_cursor(const ImageViewRGB32& screen, const std::vector>& cards){ + std::optional cursor; + for (size_t card = 0; card < cards.size(); card++){ + const CardLook look = card_look(screen, cards[card]); + if (look == CardLook::OTHER || (look == CardLook::BLUE && cursor)){ + return std::nullopt; + } + if (look == CardLook::BLUE){ + cursor = card; + } + } + return cursor; +} + +// A strip inside each button, left of its text (1080p: x 330-460 on the left +// column, 1040-1170 on the right; 60 px tall around each row's centre). +const std::vector>& language_buttons(){ + static const std::vector> cards = []{ + const double column_x[2] = {0.172, 0.542}; + const double row_center_y[5] = {0.154, 0.280, 0.404, 0.528, 0.652}; + std::vector> ret; + for (size_t item = 0; item < LANGUAGE_SELECT_BUTTONS; item++){ + ret.push_back({ImageFloatBox(column_x[item % 2], row_center_y[item / 2] - 0.028, 0.068, 0.056)}); + } + return ret; + }(); + return cards; +} + +// Two strips of background beside each face (1080p: portraits from x 168, +// 576, 984, 1392 and y 124, 456; the strips 12-44 and 320-356 px in, 80-180 +// px down). +const std::vector>& look_portraits(){ + static const std::vector> cards = []{ + const double column_x[4] = {0.0875, 0.3000, 0.5125, 0.7250}; + const double row_y[2] = {0.1148, 0.4222}; + std::vector> ret; + for (size_t item = 0; item < LOOK_SELECT_PORTRAITS; item++){ + const double x = column_x[item % 4]; + const double y = row_y[item / 4] + 0.0741; + ret.push_back({ImageFloatBox(x + 0.0063, y, 0.0167, 0.0926), ImageFloatBox(x + 0.1667, y, 0.0188, 0.0926)}); + } + return ret; + }(); + return cards; +} + +} // namespace + + +std::optional language_select_cursor(const ImageViewRGB32& screen){ + return grid_cursor(screen, language_buttons()); +} + +std::optional look_select_cursor(const ImageViewRGB32& screen){ + return grid_cursor(screen, look_portraits()); +} + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h new file mode 100644 index 0000000000..f413659405 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h @@ -0,0 +1,48 @@ +/* BDSP AutoStory - Intro Menu Detector + * + * From: https://github.com/PokemonAutomation/ + * + * The two menus of a new game's intro, each a grid of flat cards over a + * crystal background, the cursor's card blue and the others light grey + * (lab-B 2026-11-09, 1080p): + * - the language select: ten buttons in two columns of five, "Select the + * language you would like to play in." below; + * - the look select: two rows of four portraits, "Choose your photo." + * below. Top row: Lucas light, Lucas dark, Dawn dark-haired, Dawn dark; + * bottom row: Lucas blond, Lucas brown, Dawn blond, Dawn brown-haired. + * Cards are numbered row by row, as MenuGrid reads them. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_IntroMenuDetector_H +#define PokemonAutomation_PokemonBDSP_AutoStory_IntroMenuDetector_H + +#include +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// The language select's buttons: English 0, its right neighbour 1, the next +// row 2 and 3, and so on. +inline constexpr size_t LANGUAGE_SELECT_BUTTONS = 10; +// The look select's portraits: the top row 0 to 3, the bottom row 4 to 7. +inline constexpr size_t LOOK_SELECT_PORTRAITS = 8; + +// The button the cursor is on, or nullopt when `screen` is not the language +// select: every button must read as plain grey but exactly one, blue. +std::optional language_select_cursor(const ImageViewRGB32& screen); + +// The portrait the cursor is on, or nullopt when `screen` is not the look +// select: the background of every portrait must read as plain grey but +// exactly one, blue. +std::optional look_select_cursor(const ImageViewRGB32& screen); + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp deleted file mode 100644 index 66384f774c..0000000000 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp +++ /dev/null @@ -1,79 +0,0 @@ -/* BDSP AutoStory - Language Select Detector - * - * From: https://github.com/PokemonAutomation/ - * - */ - -#include -#include "CommonFramework/ImageTools/ImageBoxes.h" -#include "CommonFramework/ImageTools/ImageStats.h" -#include "PokemonBDSP_AutoStory_LanguageSelectDetector.h" - -namespace PokemonAutomation{ -namespace NintendoSwitch{ -namespace PokemonBDSP{ - - -namespace{ - -// A strip inside each button, left of its text (1080p: x 330-460 on the left -// column, 1040-1170 on the right; 60 px tall around each row's centre). -const double COLUMN_X[2] = {0.172, 0.542}; -const double ROW_CENTER_Y[5] = {0.154, 0.280, 0.404, 0.528, 0.652}; -const double STRIP_WIDTH = 0.068; -const double STRIP_HEIGHT = 0.056; - -// Buttons are flat: the frame's measured stddev sum is 2 to 6. -const double MAX_STDDEV_SUM = 15; - -enum class ButtonLook{ GREY, BLUE, OTHER }; - -ImageFloatBox button_strip(size_t item){ - return ImageFloatBox(COLUMN_X[item % 2], ROW_CENTER_Y[item / 2] - STRIP_HEIGHT / 2, STRIP_WIDTH, STRIP_HEIGHT); -} - -// Grey (225, 233, 238) and blue (167, 206, 255) on the frame. The background -// between the buttons is a bluer (203, 232, 253), not grey. -ButtonLook button_look(const ImageViewRGB32& screen, size_t item){ - const ImageStats stats = image_stats(extract_box_reference(screen, button_strip(item))); - if (stats.stddev.sum() > MAX_STDDEV_SUM){ - return ButtonLook::OTHER; - } - const FloatPixel& c = stats.average; - const double low = std::min({c.r, c.g, c.b}); - const double high = std::max({c.r, c.g, c.b}); - if (low >= 200 && high - low <= 25){ - return ButtonLook::GREY; - } - if (c.b >= 230 && c.r >= 120 && c.b - c.r >= 50){ - return ButtonLook::BLUE; - } - return ButtonLook::OTHER; -} - -} // namespace - - -std::optional language_select_cursor(const ImageViewRGB32& screen){ - std::optional cursor; - for (size_t item = 0; item < LANGUAGE_SELECT_BUTTONS; item++){ - switch (button_look(screen, item)){ - case ButtonLook::GREY: - break; - case ButtonLook::BLUE: - if (cursor){ - return std::nullopt; - } - cursor = item; - break; - case ButtonLook::OTHER: - return std::nullopt; - } - } - return cursor; -} - - -} // namespace PokemonBDSP -} // namespace NintendoSwitch -} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h deleted file mode 100644 index 29524a35c4..0000000000 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h +++ /dev/null @@ -1,36 +0,0 @@ -/* BDSP AutoStory - Language Select Detector - * - * From: https://github.com/PokemonAutomation/ - * - * BDSP's language select, shown on a new game's first boot: ten buttons in - * two columns of five over a crystal background, "Select the language you - * would like to play in." below them. The cursor's button is blue with a red - * outline, the others light grey (lab-B 2026-11-09, 1080p). - * - */ - -#ifndef PokemonAutomation_PokemonBDSP_AutoStory_LanguageSelectDetector_H -#define PokemonAutomation_PokemonBDSP_AutoStory_LanguageSelectDetector_H - -#include -#include -#include "CommonFramework/ImageTypes/ImageViewRGB32.h" - -namespace PokemonAutomation{ -namespace NintendoSwitch{ -namespace PokemonBDSP{ - - -// The buttons, numbered row by row: English 0, its right neighbour 1, the -// next row 2 and 3, and so on (BDSP_LANGUAGE_GRID). -inline constexpr size_t LANGUAGE_SELECT_BUTTONS = 10; - -// The button the cursor is on, or nullopt when `screen` is not the language -// select: every button must read as plain grey but exactly one, blue. -std::optional language_select_cursor(const ImageViewRGB32& screen); - - -} // namespace PokemonBDSP -} // namespace NintendoSwitch -} // namespace PokemonAutomation -#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp index 3684ac7733..a141956dcf 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp @@ -183,7 +183,8 @@ BDSPAutoStory::BDSPAutoStory() , PLAYER_NAME(false, "Player name:
For a new game. Letters and digits.", LockMode::LOCK_WHILE_RUNNING, "Kosmos", "Kosmos") , PLAYER_LOOK( "Player's look:
For a new game: the look picked on the appearance grid. " - "Models 1, 2, 5 and 6 are Lucas; 3, 4, 7 and 8 are Dawn." + "The portraits row by row: 1 Lucas light, 2 Lucas dark, 3 Dawn dark-haired, 4 Dawn dark; " + "5 Lucas blond, 6 Lucas brown, 7 Dawn blond, 8 Dawn brown-haired." ) , STARTER( "Starter Pokemon:", diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 27c7a3f56d..493a31e851 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1708,8 +1708,8 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h - Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp - Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.cpp + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.cpp From 742345be9a73f95c9098de48a2bc0863050f039f Mon Sep 17 00:00:00 2001 From: dayfine Date: Tue, 10 Nov 2026 12:40:24 -0800 Subject: [PATCH 5/6] BDSP AutoStory intro picks: code-health fixes and std::string_view migration --- .../NintendoSwitch_MenuCursor_Tests.cpp | 2 +- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 27 +++++++++--------- .../Actions/PokemonBDSP_AutoStory_NewGame.cpp | 23 +++++++++------ .../Actions/PokemonBDSP_AutoStory_NewGame.h | 28 +++++++++---------- 4 files changed, 43 insertions(+), 37 deletions(-) diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp index 71ada25348..3f41a2cee2 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp @@ -26,7 +26,7 @@ std::string describe(const std::vector& moves){ } struct MovesCase{ - const char* what; + std::string_view what; MenuGrid grid; size_t from; size_t to; diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index bc67863078..27f6502f7c 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -35,6 +35,7 @@ namespace PokemonAutomation{ namespace NintendoSwitch{ namespace PokemonBDSP{ +using namespace detail; namespace{ @@ -207,7 +208,7 @@ std::string check_boot(IntroProgress& progress){ } UnitTestResult check_confirming_screens(){ - for (const char* screen : {"dialog", "language-select", "rival-list-tell", "rival-list-preset", "menu"}){ + for (std::string screen : {"dialog", "language-select", "rival-list-tell", "rival-list-preset", "menu"}){ if (!intro_screen_confirms_bdsp(screen)){ return std::string("Expected \"") + screen + "\" to confirm BDSP."; } @@ -273,7 +274,7 @@ UnitTestResult test_intro_pick_moves(Logger&){ using Moves = std::optional>; const IntroPicks french{Language::French, 2}; struct Case{ - const char* what; + std::string_view what; Moves actual; Moves expected; }; @@ -310,7 +311,7 @@ UnitTestResult test_intro_pick_moves(Logger&){ }; for (const Case& c : cases){ if (c.actual != c.expected){ - return std::string("Wrong presses for ") + c.what + "."; + return std::string("Wrong presses for ") + std::string(c.what) + "."; } } return check_intro_menu_sizes(); @@ -320,8 +321,8 @@ UnitTestResult test_intro_pick_moves(Logger&){ // A. The player's keyboard with a pick still to make fails. UnitTestResult test_intro_pick_steps(Logger&){ struct Case{ - const char* what; - const char* screen; + std::string_view what; + std::string_view screen; bool language_pending; bool look_pending; IntroStep expected; @@ -347,7 +348,7 @@ UnitTestResult test_intro_pick_steps(Logger&){ progress.language_pending = c.language_pending; progress.look_pending = c.look_pending; if (decide_intro_step(c.screen, progress) != c.expected){ - return std::string("Wrong step for ") + c.what + "."; + return std::string("Wrong step for ") + std::string(c.what) + "."; } } const IntroProgress first_look = intro_start(IntroPicks{Language::English, 0}); @@ -372,7 +373,7 @@ UnitTestResult test_intro_menu_screenshots(Logger& logger){ return "A black screen read as an intro menu."; } struct Case{ - const char* file; + std::string_view file; std::optional language; std::optional look; }; @@ -386,7 +387,7 @@ UnitTestResult test_intro_menu_screenshots(Logger& logger){ }; size_t read = 0; for (const Case& c : cases){ - const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + const std::string path = UNIT_TEST_RESOURCE_PATH() + std::string(c.file); if (!Filesystem::is_regular_file(path)){ logger.log("Screenshot not found: " + path); continue; @@ -426,7 +427,7 @@ UnitTestResult test_lit_doorway_screenshots(Logger& logger){ {"PokemonBDSP/AutoStory/SandgemMartDoorFarLeft-20261030-024648.png", -1.3, -0.5, 2.5, 4.5}, }; for (const Case& c : cases){ - const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + const std::string path = UNIT_TEST_RESOURCE_PATH() + std::string(c.file); if (!Filesystem::is_regular_file(path)){ return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); } @@ -497,7 +498,7 @@ UnitTestResult test_pokemon_center_inside(Logger& logger){ {.file = "PokemonBDSP/AutoStory/JubilifeCenterDoorstep-20261030-c5.png", .inside = false, .doorway = true}, }; for (const Case& c : cases){ - const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + const std::string path = UNIT_TEST_RESOURCE_PATH() + std::string(c.file); if (!Filesystem::is_regular_file(path)){ return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); } @@ -535,7 +536,7 @@ UnitTestResult test_pokemon_center_counter(Logger& logger){ {.file = "PokemonBDSP/AutoStory/OreburghCenterDoorstep-20261108.png", .inside = false}, }; for (const Case& c : cases){ - const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + const std::string path = UNIT_TEST_RESOURCE_PATH() + std::string(c.file); if (!Filesystem::is_regular_file(path)){ return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); } @@ -569,7 +570,7 @@ UnitTestResult test_school_lamp(Logger& logger){ {"PokemonBDSP/AutoStory/JubilifeSchoolOneTilePast-20261030.png", 0.5, 1.5}, }; for (const Case& c : cases){ - const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + const std::string path = UNIT_TEST_RESOURCE_PATH() + std::string(c.file); if (!Filesystem::is_regular_file(path)){ return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); } @@ -634,7 +635,7 @@ UnitTestResult test_move_menu_screenshots(Logger& logger){ }; const Language story = story_language(); for (const Case& c : cases){ - const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + const std::string path = UNIT_TEST_RESOURCE_PATH() + std::string(c.file); if (!Filesystem::is_regular_file(path)){ return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp index 43aac240c0..f388a8832e 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp @@ -30,6 +30,7 @@ namespace NintendoSwitch{ namespace PokemonBDSP{ using namespace std::chrono_literals; +using namespace detail; const std::vector& bdsp_language_menu(){ @@ -67,12 +68,14 @@ std::optional> look_pick_moves(size_t cursor, const In return menu_cursor_moves(BDSP_LOOK_GRID, cursor, picks.look); } +namespace detail{ IntroProgress intro_start(const IntroPicks& picks){ IntroProgress progress; progress.language_pending = true; progress.look_pending = picks.look != 0; return progress; } +} // namespace detail std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& screen){ @@ -116,7 +119,7 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& return "other"; } -IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress){ +IntroStep decide_intro_step(std::string_view screen, const IntroProgress& progress){ if (screen == "user-select"){ return IntroStep::SELECT_USER; } @@ -163,11 +166,12 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog } return IntroStep::ADVANCE; } -bool intro_screen_confirms_bdsp(const std::string& screen){ +bool intro_screen_confirms_bdsp(std::string_view screen){ return screen == "dialog" || screen == "language-select" || screen == "look-select" || screen == "rival-list-tell" || screen == "rival-list-preset" || screen == "menu"; } -void note_intro_screen(IntroProgress& progress, const std::string& screen, std::chrono::milliseconds since_start){ +namespace detail{ +void note_intro_screen(IntroProgress& progress, std::string_view screen, std::chrono::milliseconds since_start){ if (intro_screen_confirms_bdsp(screen)){ progress.game_confirmed = true; } @@ -215,6 +219,7 @@ void record_intro_step(IntroProgress& progress, IntroStep step){ return; } } +} // namespace detail namespace{ @@ -346,7 +351,7 @@ void advance(IntroRun& run){ // One of the intro's two menus: its shape and its cursor reader. struct IntroMenu{ - const char* what; + std::string_view what; MenuGrid grid; std::optional (*read)(const ImageViewRGB32& screen); }; @@ -376,17 +381,17 @@ bool pick_on_grid(IntroRun& run, const IntroMenu& menu, size_t target){ if (!cursor){ return false; } - run.trace.action(std::string("Picking the ") + menu.what + ": cursor on item " + std::to_string(*cursor) + + run.trace.action(std::string("Picking the ") + std::string(menu.what) + ": cursor on item " + std::to_string(*cursor) + ", the pick is item " + std::to_string(target) + "."); if (*cursor == target){ - run.trace.screen(std::string("pick-") + menu.what); + run.trace.screen(std::string("pick-") + std::string(menu.what)); press(run, BUTTON_A); pbf_wait(run.context, 500ms); return true; } press_cursor_moves(run.context, *menu_cursor_moves(menu.grid, *cursor, target)); } - run.trace.fail(std::string("The ") + menu.what + " menu's cursor did not reach its pick in " + + run.trace.fail(std::string("The ") + std::string(menu.what) + " menu's cursor did not reach its pick in " + std::to_string(INTRO_PICK_TRIES) + " tries."); } @@ -402,7 +407,7 @@ void pick_on_intro_menu(IntroRun& run, IntroStep step){ } // Notes `screen` in the run's progress, decides the step for it and counts it. -IntroStep next_intro_step(IntroRun& run, const char* screen){ +IntroStep next_intro_step(IntroRun& run, std::string_view screen){ note_intro_screen(run.progress, screen, std::chrono::duration_cast(current_time() - run.start)); const IntroStep step = decide_intro_step(screen, run.progress); @@ -425,7 +430,7 @@ IntroStep next_intro_step(IntroRun& run, const char* screen){ } // Carries out what decide_intro_step() chose for `screen`. -void act(IntroRun& run, const char* screen){ +void act(IntroRun& run, std::string_view screen){ const IntroStep step = next_intro_step(run, screen); switch (step){ case IntroStep::SELECT_USER: diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h index d4a7b57e2b..364e80d5c0 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h @@ -8,6 +8,7 @@ * (NintendoSwitch_MenuCursor.h); the menus' shapes are the data below. * Both menus are read by their detector (PokemonBDSP_AutoStory_IntroMenuDetector.h). * +// code-health: allow header_symbols -- new game intro flow, menu cursors, detectors and picks */ #ifndef PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H @@ -111,6 +112,15 @@ struct IntroProgress{ // B presses on unknown screens since the last X. size_t b_presses = 0; }; +// Whether `screen` (classify_intro_screen()) is one only BDSP's intro shows: +// its dialogue box, its language and look selects, the rival's name list, its main menu. The keyboard and +// the user select are the Switch's. +bool intro_screen_confirms_bdsp(std::string_view screen); + +// Decides the next controller action based on the current screen and intro progress. +IntroStep decide_intro_step(std::string_view screen, const IntroProgress& progress); + +namespace detail{ // How long the intro presses A on unknown screens, from the first screen // that is not black, before a BDSP screen must show: the logos, the title // and the language select. Another game launched by mistake (r1, @@ -119,16 +129,10 @@ struct IntroProgress{ inline constexpr std::chrono::milliseconds INTRO_TIME_BEFORE_BDSP = std::chrono::seconds(60); // How long the boot may stay black after the restart. inline constexpr std::chrono::milliseconds INTRO_BOOT_LIMIT = std::chrono::seconds(60); -// Whether `screen` (classify_intro_screen()) is one only BDSP's intro shows: -// its dialogue box, its language and look selects, the rival's name list, its main menu. The keyboard and -// the user select are the Switch's. -bool intro_screen_confirms_bdsp(const std::string& screen); - // Screens after the rival's name is picked, each taking A, before the name // counts as confirmed and B takes over: "Barry, is it?", the box closing // (Barry alone), the Yes/No, and a few of Rowan's lines after it. inline constexpr size_t INTRO_SCREENS_AFTER_RIVAL_PICK = 8; - // Once the rival is named, the intro only has dialogue and cutscenes left // before the bedroom, and screens the classifier misses (r2: a dialogue box // without its arrow). Each unknown screen takes B, which advances dialogue, @@ -138,13 +142,6 @@ inline constexpr size_t INTRO_B_BEFORE_X = 5; // X presses that opened no menu before the intro gives up: with // INTRO_B_BEFORE_X B presses between them, about two minutes. inline constexpr size_t INTRO_X_WITHOUT_MENU = 20; - -// Notes `screen`, read `since_start` into the intro, in `progress`: the -// game is confirmed on a BDSP screen; black screens leave the clock alone. -void note_intro_screen(IntroProgress& progress, const std::string& screen, std::chrono::milliseconds since_start); -// Decides the next controller action based on the current screen and intro progress. -IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress); - // When X opens nothing the menu detector knows, the player is still in // control if the screen held still for INTRO_STILL_TIME before X (no // dialogue, no fade, no intro sparkles), X changed the menu's area by more @@ -167,9 +164,12 @@ struct XProbe{ bool intro_in_control_without_menu(const XProbe& probe); // Counts `step` in `progress`. void record_intro_step(IntroProgress& progress, IntroStep step); - // The progress a new intro starts from: which of `picks` need a menu. IntroProgress intro_start(const IntroPicks& picks); +// Notes `screen`, read `since_start` into the intro, in `progress`: the +// game is confirmed on a BDSP screen; black screens leave the clock alone. +void note_intro_screen(IntroProgress& progress, std::string_view screen, std::chrono::milliseconds since_start); +} // namespace detail // From the title screen or any intro screen of a game with no save, with BDSP // running in front: from HOME it throws UserSetupError, since the restart would From 449381f5d5afe4b764a150fbce935a8a1eaa35ce Mon Sep 17 00:00:00 2001 From: dayfine Date: Thu, 12 Nov 2026 23:48:54 -0800 Subject: [PATCH 6/6] BDSP AutoStory: an empty or off-frame move name box reads as unread, not a crash gray_box() sorted and indexed an empty vector when the box was outside the frame or the image empty (SIGSEGV at 0x0, 2026-11-12). read_move_name now returns "" for it and the menu falls back to the PP guess. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 25 +++++++++++++++++++ .../PokemonBDSP_AutoStory_BattleMoves.cpp | 7 ++++++ 2 files changed, 32 insertions(+) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index b98e2440f4..2ba033d092 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -598,6 +598,27 @@ UnitTestResult test_school_lamp(Logger& logger){ // no name read, Piplup's Pound at 35 PP and Growl at 22 PP, and the old // "newest move that is not a status move" fallback used Growl until the // training gave up after 25 turns. +// A menu read on an empty image, or with buttons outside the frame, reads +// nothing instead of crashing (SIGSEGV in gray_box, 2026-11-12). +UnitTestResult test_move_menu_unreadable(Logger&){ + BattleMoveMenuState moves; + moves.move_count = 2; + moves.buttons[0] = ImageFloatBox{0.5, 0.5, 0.4, 0.1}; + moves.buttons[1] = ImageFloatBox{1.5, 1.5, 0.4, 0.1}; + const ImageRGB32 frame(64, 36); + const ImageViewRGB32 inputs[] = {ImageViewRGB32(), frame}; + for (const ImageViewRGB32& screen : inputs){ + const std::vector slots = read_move_menu(screen, moves); + if (slots.size() != 2){ + return "read_move_menu returned " + std::to_string(slots.size()) + " slots, expected 2."; + } + if (!slots[1].slug.empty() || slots[1].pp != -1){ + return "a button outside the frame read a move."; + } + } + return true; +} + UnitTestResult test_choose_move(Logger&){ struct Case{ std::string_view name; std::vector slots; size_t best; }; const Case cases[] = { @@ -759,6 +780,10 @@ const AutoStoryActionsCase AUTO_STORY_ACTIONS_CASES[] = { .input = {"battle moves: the pick from names and PP", test_choose_move}, .expectation = {}, }, + { + .input = {"battle moves: unreadable menu reads nothing", test_move_menu_unreadable}, + .expectation = {}, + }, { .input = {"battle moves: move menu screenshots in each language", test_move_menu_screenshots}, .expectation = {}, diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.cpp index cae25a7d4d..6557c928a8 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.cpp @@ -221,6 +221,10 @@ GrayBox gray_box(const ImageViewRGB32& box){ ret.gray[y * box.width() + x] = (int)(((pixel >> 16) & 0xff) + ((pixel >> 8) & 0xff) + (pixel & 0xff)) / 3; } } + if (ret.gray.empty()){ + // Nothing to read: a box outside the frame, or an empty image. + return ret; + } std::vector sorted = ret.gray; std::nth_element(sorted.begin(), sorted.begin() + sorted.size() / 2, sorted.end()); const int background = sorted[sorted.size() / 2]; @@ -259,6 +263,9 @@ std::string read_move_name(const ImageViewRGB32& screen, const ImageFloatBox& bu const ImageFloatBox name_box{button.x + button.width * 0.17, button.y + button.height * 0.08, button.width * 0.53, button.height * 0.80}; const GrayBox box = gray_box(extract_box_reference(screen, name_box)); + if (box.gray.empty()){ + return ""; + } const auto [band_top, band_bottom] = tallest_text_band(box); // Black text on white, with a margin, for the OCR. const size_t pad = 6;