From 09acde1e384d0cf350731e7d7eda9584abbae4ec Mon Sep 17 00:00:00 2001 From: dayfine Date: Mon, 9 Nov 2026 10:34:17 -0800 Subject: [PATCH 01/19] BDSP AutoStory: pick the language and the player's look in the new-game intro The intro pressed A on every screen, so it took the language the cursor starts on and the first look (Lucas); run 3 (French, Dawn) needed both picked by hand. NintendoSwitch/Programs/MenuCursor: the D-pad presses from one item of a list or grid to another, none across an edge, so they hold whether or not the menu wraps; and a D-pad probe (press, opposite press, measure) that tells a screen with a cursor from one without, for menus no detector knows. Nothing in it is BDSP's, for PLZA's new game too. BDSP: the language select's order and the appearance grid's shape are data. Until a pick is made, screens that would take A get a probe first: a list that moves up/down but not sideways takes the language pick, a grid that moves sideways takes the look pick, then A. The player's keyboard with a pick still to make fails. New options: Switch language (where the list's cursor starts) and Player's look (models 1-8, PlayerModelOption's icons). Defaults keep the old behavior. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../MenuCursor/NintendoSwitch_MenuCursor.cpp | 97 +++++++++ .../MenuCursor/NintendoSwitch_MenuCursor.h | 73 +++++++ .../NintendoSwitch_MenuCursor_Tests.cpp | 189 ++++++++++++++++++ .../NintendoSwitch_MenuCursor_Tests.h | 23 +++ .../Options/PokemonBDSP_PlayerModelOption.cpp | 10 +- .../Options/PokemonBDSP_PlayerModelOption.h | 3 + .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 101 ++++++++++ .../Actions/PokemonBDSP_AutoStory_NewGame.cpp | 183 ++++++++++++++++- .../Actions/PokemonBDSP_AutoStory_NewGame.h | 53 ++++- .../AutoStory/PokemonBDSP_AutoStory.cpp | 30 ++- .../AutoStory/PokemonBDSP_AutoStory.h | 3 + .../PokemonBDSP_AutoStory_StoryExecutor.cpp | 2 +- .../PokemonBDSP_AutoStory_StoryExecutor.h | 3 + SerialPrograms/cmake/SourceFiles.cmake | 4 + 14 files changed, 757 insertions(+), 17 deletions(-) create mode 100644 SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp create mode 100644 SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h create mode 100644 SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp create mode 100644 SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp new file mode 100644 index 0000000000..17dc468105 --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp @@ -0,0 +1,97 @@ +/* Menu Cursor + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTools/ImageDiff.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "NintendoSwitch_MenuCursor.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + +using namespace std::chrono_literals; + + +namespace{ + +void add_presses(std::vector& moves, DpadPosition direction, size_t count){ + moves.insert(moves.end(), count, direction); +} +void add_sideways(std::vector& moves, size_t from_column, size_t to_column){ + if (to_column > from_column){ + add_presses(moves, DPAD_RIGHT, to_column - from_column); + }else{ + add_presses(moves, DPAD_LEFT, from_column - to_column); + } +} + +// 0 when either frame is missing: a probe without frames sees nothing move. +double screen_change(const VideoSnapshot& before, const VideoSnapshot& after){ + if (!before || !after){ + return 0; + } + return ImageMatch::pixel_RMSD(*before.frame, *after.frame); +} + +} // namespace + + +std::optional> menu_cursor_moves(const MenuGrid& grid, size_t from, size_t to){ + if (grid.columns == 0 || from >= grid.count || to >= grid.count){ + return std::nullopt; + } + const size_t from_row = from / grid.columns; + const size_t from_column = from % grid.columns; + const size_t to_row = to / grid.columns; + const size_t to_column = to % grid.columns; + std::vector moves; + if (to_row > from_row){ + // The start's row is above the target's, so it is full: sideways first. + add_sideways(moves, from_column, to_column); + add_presses(moves, DPAD_DOWN, to_row - from_row); + }else{ + // Up to the target's row first, which is full unless it is the start's. + add_presses(moves, DPAD_UP, from_row - to_row); + add_sideways(moves, from_column, to_column); + } + return moves; +} + + +bool cursor_probe_moved(const CursorProbe& probe){ + return probe.press_change >= CURSOR_PRESS_CHANGE && + probe.press_change >= CURSOR_NOISE_RATIO * probe.still_change && + 2 * probe.back_change <= probe.press_change; +} + +CursorProbe probe_cursor(VideoStream& stream, ProControllerContext& context, DpadPosition there, DpadPosition back){ + context.wait_for_all_requests(); + const VideoSnapshot first = stream.video().snapshot(); + context.wait_for(CURSOR_PROBE_WAIT); + const VideoSnapshot before = stream.video().snapshot(); + pbf_press_dpad(context, there, 80ms, CURSOR_PROBE_WAIT); + context.wait_for_all_requests(); + const VideoSnapshot pressed = stream.video().snapshot(); + pbf_press_dpad(context, back, 80ms, CURSOR_PROBE_WAIT); + context.wait_for_all_requests(); + const VideoSnapshot returned = stream.video().snapshot(); + return CursorProbe{ + screen_change(first, before), + screen_change(before, pressed), + screen_change(before, returned), + }; +} + +void press_cursor_moves(ProControllerContext& context, const std::vector& moves){ + for (DpadPosition move : moves){ + pbf_press_dpad(context, move, 80ms, 320ms); + } +} + + + +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h new file mode 100644 index 0000000000..fd3ad96de6 --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h @@ -0,0 +1,73 @@ +/* Menu Cursor + * + * From: https://github.com/PokemonAutomation/ + * + * Moves the cursor of a fixed list or grid (a game's language select, the + * player's look) to an item by its index. The menu's shape is data; the + * presses follow from it. A D-pad probe tells whether a screen has a + * cursor at all, for menus no detector knows. + * + */ + +#ifndef PokemonAutomation_NintendoSwitch_MenuCursor_H +#define PokemonAutomation_NintendoSwitch_MenuCursor_H + +#include +#include +#include +#include "CommonFramework/Tools/VideoStream.h" +#include "NintendoSwitch/Controllers/NintendoSwitch_ControllerButtons.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + + +// A menu of `count` items in rows of `columns`, numbered left to right, then +// top to bottom; a list has one column. Only the last row may be short. +struct MenuGrid{ + size_t columns = 1; + size_t count = 0; +}; + +// The D-pad presses that take the cursor from item `from` to item `to`. +// None crosses an edge, so they hold whether or not the menu wraps, and +// sideways presses happen on a full row. nullopt when either item is outside +// the grid. +std::optional> menu_cursor_moves(const MenuGrid& grid, size_t from, size_t to); + + +// How much the screen changed (pixel RMSD) around a D-pad probe, each against +// the frame just before the press: over a wait with no input, after the press, +// and after the opposite press. +struct CursorProbe{ + double still_change; + double press_change; + double back_change; +}; +// The least change a press that moved a cursor makes. +inline constexpr double CURSOR_PRESS_CHANGE = 6; +// How many times what the screen changes on its own (an animated preview, a +// blinking arrow) a press that moved a cursor changes it. +inline constexpr double CURSOR_NOISE_RATIO = 3; +// Whether `probe` saw a cursor move and come back: the press changed the +// screen by CURSOR_PRESS_CHANGE and CURSOR_NOISE_RATIO times the still +// change, and the opposite press put back at least half of it. +bool cursor_probe_moved(const CursorProbe& probe); + +// The wait before a probe's first press, and after each press. +inline constexpr Milliseconds CURSOR_PROBE_WAIT = Milliseconds(400); +// Waits with no input, presses `there`, waits, presses `back`, waits, and +// measures the screen at each step. On a screen with no cursor `there` and +// `back` do nothing; on one with a cursor they leave it where it was, unless +// `there` meets an edge that does not wrap. +CursorProbe probe_cursor(VideoStream& stream, ProControllerContext& context, DpadPosition there, DpadPosition back); + +// Presses `moves` in order, one D-pad tap each. +void press_cursor_moves(ProControllerContext& context, const std::vector& moves); + + + +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp new file mode 100644 index 0000000000..77bc1152e2 --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp @@ -0,0 +1,189 @@ +/* Menu Cursor - Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "NintendoSwitch_MenuCursor.h" +#include "NintendoSwitch_MenuCursor_Tests.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + +namespace{ + + +std::string describe(const std::vector& moves){ + std::string out; + for (DpadPosition move : moves){ + out += (out.empty() ? "" : " ") + dpad_to_string(move); + } + return out.empty() ? "(none)" : out; +} + +struct MovesCase{ + const char* what; + MenuGrid grid; + size_t from; + size_t to; + std::vector expected; +}; + +UnitTestResult test_cursor_moves(){ + const MenuGrid list{1, 9}; + const MenuGrid looks{4, 8}; + // Seven items in rows of three: the last row holds one. + const MenuGrid ragged{3, 7}; + const MovesCase cases[] = { + {"list: stay", list, 1, 1, {}}, + {"list: down", list, 1, 2, {DPAD_DOWN}}, + {"list: up to the top", list, 1, 0, {DPAD_UP}}, + {"list: top to bottom, no wrap", list, 0, 8, std::vector(8, DPAD_DOWN)}, + {"grid: right on the top row", looks, 0, 2, {DPAD_RIGHT, DPAD_RIGHT}}, + {"grid: down a row", looks, 0, 4, {DPAD_DOWN}}, + {"grid: sideways, then down", looks, 1, 7, {DPAD_RIGHT, DPAD_RIGHT, DPAD_DOWN}}, + {"grid: up, then sideways", looks, 7, 0, {DPAD_UP, DPAD_LEFT, DPAD_LEFT, DPAD_LEFT}}, + // The short row has only column 0: sideways happens on the full row above. + {"ragged: into the short row", ragged, 2, 6, {DPAD_LEFT, DPAD_LEFT, DPAD_DOWN, DPAD_DOWN}}, + {"ragged: out of the short row", ragged, 6, 5, {DPAD_UP, DPAD_RIGHT, DPAD_RIGHT}}, + }; + for (const MovesCase& c : cases){ + const std::optional> moves = menu_cursor_moves(c.grid, c.from, c.to); + if (!moves){ + return std::string(c.what) + ": no moves."; + } + if (*moves != c.expected){ + return std::string(c.what) + ": " + describe(*moves) + ", expected " + describe(c.expected) + "."; + } + } + if (menu_cursor_moves(looks, 0, 8) || menu_cursor_moves(looks, 8, 0) || menu_cursor_moves(MenuGrid{0, 4}, 0, 1)){ + return "An item outside the grid, or a grid with no columns, must give no moves."; + } + return true; +} + +// Walks the moves from `from` to `to` on `grid`; what went wrong, or "". +std::string check_walk(const MenuGrid& grid, size_t from, size_t to){ + const std::string pair = std::to_string(grid.columns) + "x" + std::to_string(grid.count) + + " from " + std::to_string(from) + " to " + std::to_string(to); + const std::optional> moves = menu_cursor_moves(grid, from, to); + if (!moves){ + return pair + ": no moves."; + } + long row = (long)(from / grid.columns); + long column = (long)(from % grid.columns); + for (DpadPosition move : *moves){ + row += move == DPAD_DOWN ? 1 : move == DPAD_UP ? -1 : 0; + column += move == DPAD_RIGHT ? 1 : move == DPAD_LEFT ? -1 : 0; + const long index = row * (long)grid.columns + column; + if (row < 0 || column < 0 || column >= (long)grid.columns || index >= (long)grid.count){ + return pair + ": " + describe(*moves) + " leaves the grid."; + } + } + if (row * (long)grid.columns + column != (long)to){ + return pair + ": " + describe(*moves) + " ends elsewhere."; + } + return ""; +} + +// Every pair of items in a few shapes: the moves land on the target and never +// leave the grid or step on a missing item. +UnitTestResult test_cursor_moves_stay_inside(){ + for (const MenuGrid& grid : {MenuGrid{1, 9}, MenuGrid{4, 8}, MenuGrid{3, 7}, MenuGrid{5, 1}}){ + for (size_t from = 0; from < grid.count; from++){ + for (size_t to = 0; to < grid.count; to++){ + const std::string error = check_walk(grid, from, to); + if (!error.empty()){ + return error; + } + } + } + } + return true; +} + +UnitTestResult test_cursor_probe(){ + struct Case{ + const char* what; + CursorProbe probe; + bool moved; + }; + const Case cases[] = { + {"a still list: the highlight moves and comes back", {0.5, 20, 0.5}, true}, + {"a small Yes/No box", {0.3, 8, 0.4}, true}, + {"an animated preview, swapped by the press", {8, 40, 10}, true}, + {"dialogue with a blinking arrow", {2.5, 3, 2.5}, false}, + {"an animation that changes as much on its own", {10, 20, 10}, false}, + {"a press that moved something it did not put back", {0.5, 20, 15}, false}, + {"a fade", {20, 25, 45}, false}, + {"no frames", {0, 0, 0}, false}, + }; + for (const Case& c : cases){ + if (cursor_probe_moved(c.probe) != c.moved){ + return std::string(c.what) + (c.moved ? ": the cursor move was missed." : ": no cursor moved."); + } + } + return true; +} + + +struct MenuCursorInput{ + std::string name; + UnitTestResult (*body)(); +}; + +struct MenuCursorExpectation{ +}; + +struct MenuCursorCase{ + MenuCursorInput input; + MenuCursorExpectation expectation; +}; + +class Test_MenuCursor : public UnitTest{ +public: + explicit Test_MenuCursor(MenuCursorCase test_case) + : UnitTest("NintendoSwitch::MenuCursor - " + test_case.input.name) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return m_case.input.body(); + } + +private: + MenuCursorCase m_case; +}; + +const MenuCursorCase MENU_CURSOR_CASES[] = { + { + .input = {"presses from item to item", test_cursor_moves}, + .expectation = {}, + }, + { + .input = {"presses stay inside the grid", test_cursor_moves_stay_inside}, + .expectation = {}, + }, + { + .input = {"probe judgement", test_cursor_probe}, + .expectation = {}, + }, +}; + + +} // namespace + + +void add_tests_MenuCursor(UnitTestDatabase& database){ + for (const MenuCursorCase& item : MENU_CURSOR_CASES){ + database.add(item); + } +} + + + +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h new file mode 100644 index 0000000000..ce92acd5dc --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h @@ -0,0 +1,23 @@ +/* Menu Cursor - Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_NintendoSwitch_MenuCursor_Tests_H +#define PokemonAutomation_NintendoSwitch_MenuCursor_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + + +// The cursor presses for list and grid items, and the probe's judgement. No console. +void add_tests_MenuCursor(UnitTestDatabase& database); + + + +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp index 4b81977e67..6d4490bdac 100644 --- a/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp @@ -5,6 +5,7 @@ */ #include +#include #include #include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" @@ -42,9 +43,14 @@ const PlayerModelIcons& PLAYER_MODEL_ICONS(){ PlayerModelOption::PlayerModelOption() - : StringSelectOption( + : PlayerModelOption( "Character model:
The appearance chosen for the player character. " - "Each one is watched through a different eye template.", + "Each one is watched through a different eye template." + ) +{} +PlayerModelOption::PlayerModelOption(std::string label) + : StringSelectOption( + std::move(label), PLAYER_MODEL_ICONS().database, LockMode::LOCK_WHILE_RUNNING, "model-1" diff --git a/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h index e54cd04850..9f3f358206 100644 --- a/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h +++ b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h @@ -8,6 +8,7 @@ #define PokemonAutomation_PokemonBDSP_PlayerModelOption_H #include +#include #include "CommonTools/Options/StringSelectOption.h" namespace PokemonAutomation{ @@ -22,6 +23,8 @@ const uint8_t BDSP_PLAYER_MODEL_COUNT = 8; class PlayerModelOption : public StringSelectOption{ public: PlayerModelOption(); + // The same choices, under another label. + explicit PlayerModelOption(std::string label); uint8_t model_number() const{ return (uint8_t)(index() + 1); } }; diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index 49a0dd2313..5cd0661f9b 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -20,6 +20,7 @@ #include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "CommonFramework/Logging/Logger.h" +#include "NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h" #include "PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_StoryMaps.h" #include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" #include "PokemonBDSP_AutoStory_Center.h" @@ -250,6 +251,96 @@ UnitTestResult test_intro_gate(Logger&){ } +// The presses for the language select (a list starting on the Switch's +// language) and the appearance grid (two rows of four, from the first look). +UnitTestResult test_intro_pick_moves(Logger&){ + using Moves = std::optional>; + struct Case{ + const char* what; + Moves actual; + Moves expected; + }; + const Case cases[] = { + {"French on an English Switch (lab run 3)", + language_pick_moves({Language::English, Language::French, 0}), Moves(std::vector{DPAD_DOWN})}, + {"the Switch's own language", + language_pick_moves({Language::English, Language::English, 0}), Moves(std::vector{})}, + {"English on a French Switch", + language_pick_moves({Language::French, Language::English, 0}), Moves(std::vector{DPAD_UP})}, + {"top to bottom", + language_pick_moves({Language::Japanese, Language::ChineseTraditional, 0}), Moves{std::vector(8, DPAD_DOWN)}}, + {"a language the menu lacks", + language_pick_moves({Language::English, Language::None, 0}), std::nullopt}, + {"Model 1, where the cursor starts", + look_pick_moves({Language::English, Language::English, 0}), Moves(std::vector{})}, + {"Model 3, Dawn's first look (lab run 3)", + look_pick_moves({Language::English, Language::English, 2}), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT})}, + {"Model 8, the last", + look_pick_moves({Language::English, Language::English, 7}), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT, DPAD_RIGHT, DPAD_DOWN})}, + {"no Model 9", + look_pick_moves({Language::English, Language::English, 8}), std::nullopt}, + }; + for (const Case& c : cases){ + if (c.actual != c.expected){ + return std::string("Wrong presses for ") + c.what + "."; + } + } + if (bdsp_language_menu().size() != 9){ + return "BDSP's language select lists nine languages."; + } + return true; +} + +// Until its pick is made, every screen that would take A gets a probe +// first: the language's before the look's. The player's keyboard with a +// pick still to make fails. Picks the cursor starts on need no probe. +UnitTestResult test_intro_pick_probes(Logger&){ + struct Case{ + const char* what; + const char* screen; + bool language_pending; + bool look_pending; + IntroStep expected; + }; + const Case cases[] = { + {"a logo, language to pick", "other", true, true, IntroStep::PROBE_FOR_LANGUAGE}, + {"a dialogue, language to pick", "dialog", true, false, IntroStep::PROBE_FOR_LANGUAGE}, + {"Rowan, look to pick", "dialog", false, true, IntroStep::PROBE_FOR_LOOK}, + {"the preview, look to pick", "other", false, true, IntroStep::PROBE_FOR_LOOK}, + {"both picked", "dialog", false, false, IntroStep::ADVANCE}, + {"black, language to pick", "black", true, true, IntroStep::WAIT}, + {"the user select", "user-select", true, true, IntroStep::SELECT_USER}, + {"the keyboard, language missed", "keyboard", true, false, IntroStep::FAIL_PICK_MISSED}, + {"the keyboard, look missed", "keyboard", false, true, IntroStep::FAIL_PICK_MISSED}, + {"the keyboard, both picked", "keyboard", false, false, IntroStep::ENTER_PLAYER_NAME}, + }; + for (const Case& c : cases){ + IntroProgress progress = intro_progress(0); + progress.language_pending = c.language_pending; + progress.look_pending = c.look_pending; + if (decide_intro_step(c.screen, progress) != c.expected){ + return std::string("Wrong step for ") + c.what + "."; + } + } + // The wrong-game guard still stops a game that never shows BDSP's screens. + IntroProgress other_game = intro_start({Language::English, Language::French, 2}); + note_intro_screen(other_game, "other", std::chrono::milliseconds(0)); + note_intro_screen(other_game, "other", INTRO_TIME_BEFORE_BDSP); + if (decide_intro_step("other", other_game) != IntroStep::FAIL_NOT_BDSP){ + return "Past INTRO_TIME_BEFORE_BDSP, a pending pick must not keep another game going."; + } + const IntroProgress defaults = intro_start({Language::English, Language::English, 0}); + if (defaults.language_pending || defaults.look_pending){ + return "Picks the cursor starts on need no menu."; + } + const IntroProgress run3 = intro_start({Language::English, Language::French, 2}); + if (!run3.language_pending || !run3.look_pending){ + return "French and Model 3 on an English Switch need both menus."; + } + return true; +} + + // The Mart's doorway on the failure screenshots of r121a and r122a, kept in // the unit test resources (the CommandLineTests checkout): where it is // relative to the player, who is in the middle of the screen. @@ -486,6 +577,14 @@ const AutoStoryActionsCase AUTO_STORY_ACTIONS_CASES[] = { .input = {"new-game intro: BDSP or stop", test_intro_gate}, .expectation = {}, }, + { + .input = {"new-game intro: language and look presses", test_intro_pick_moves}, + .expectation = {}, + }, + { + .input = {"new-game intro: probes until the picks are made", test_intro_pick_probes}, + .expectation = {}, + }, { .input = {"lit doorway screenshots", test_lit_doorway_screenshots}, .expectation = {}, @@ -512,6 +611,8 @@ void add_tests_AutoStory_Actions(UnitTestDatabase& database){ for (const auto& item : AUTO_STORY_ACTIONS_CASES){ database.add(item); } + // The menu cursor's own tests, with those of the intro, its first user. + add_tests_MenuCursor(database); } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp index 5bd6a47b09..162a9fbad3 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp @@ -4,6 +4,7 @@ * */ +#include #include "Common/Cpp/Exceptions.h" #include "CommonFramework/ImageTools/ImageBoxes.h" #include "CommonFramework/ImageTools/ImageDiff.h" @@ -30,6 +31,54 @@ namespace PokemonBDSP{ using namespace std::chrono_literals; +const std::vector& bdsp_language_menu(){ + static const std::vector menu{ + Language::Japanese, + Language::English, + Language::French, + Language::Italian, + Language::German, + Language::Spanish, + Language::Korean, + Language::ChineseSimplified, + Language::ChineseTraditional, + }; + return menu; +} + +namespace{ + +std::optional language_item(Language language){ + const std::vector& menu = bdsp_language_menu(); + const auto it = std::find(menu.begin(), menu.end(), language); + if (it == menu.end()){ + return std::nullopt; + } + return (size_t)(it - menu.begin()); +} + +} // namespace + +std::optional> language_pick_moves(const IntroPicks& picks){ + const std::optional from = language_item(picks.switch_language); + const std::optional to = language_item(picks.game_language); + if (!from || !to){ + return std::nullopt; + } + return menu_cursor_moves(MenuGrid{1, bdsp_language_menu().size()}, *from, *to); +} +std::optional> look_pick_moves(const IntroPicks& picks){ + return menu_cursor_moves(BDSP_LOOK_GRID, 0, picks.look); +} + +IntroProgress intro_start(const IntroPicks& picks){ + IntroProgress progress; + progress.language_pending = picks.switch_language != picks.game_language; + progress.look_pending = picks.look != 0; + return progress; +} + + std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& screen){ // The Switch's own screens come first: BDSP's dialogue detector also fires on them. if (console != nullptr && StartGameUserSelectDetector(*console).detect(screen)){ @@ -64,6 +113,25 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& return "other"; } +namespace{ + +// A on a screen that may take A, after a D-pad probe while a pick waits for +// its menu. The picks come before the player's name. +IntroStep advance_or_probe(const IntroProgress& progress){ + if (progress.player_named){ + return IntroStep::ADVANCE; + } + if (progress.language_pending){ + return IntroStep::PROBE_FOR_LANGUAGE; + } + if (progress.look_pending){ + return IntroStep::PROBE_FOR_LOOK; + } + return IntroStep::ADVANCE; +} + +} // namespace + IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress){ if (screen == "user-select"){ return IntroStep::SELECT_USER; @@ -72,6 +140,9 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog return IntroStep::FAIL_NOT_IN_GAME; } if (screen == "keyboard"){ + if (!progress.player_named && (progress.language_pending || progress.look_pending)){ + return IntroStep::FAIL_PICK_MISSED; + } return progress.player_named ? IntroStep::CANCEL_RIVAL_KEYBOARD : IntroStep::ENTER_PLAYER_NAME; } if (screen == "rival-list-tell"){ @@ -87,7 +158,7 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog return IntroStep::WAIT; } if (screen == "dialog"){ - return progress.rival_named ? IntroStep::ADVANCE_WITH_B : IntroStep::ADVANCE; + return progress.rival_named ? IntroStep::ADVANCE_WITH_B : advance_or_probe(progress); } // The bedroom comes only after the rival is named. Before that, every unknown // screen (logos, the title, language, the appearance preview, cutscenes) @@ -99,7 +170,7 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog if (!progress.game_confirmed && progress.unconfirmed_for >= INTRO_TIME_BEFORE_BDSP){ return IntroStep::FAIL_NOT_BDSP; } - return IntroStep::ADVANCE; + return advance_or_probe(progress); } bool intro_screen_confirms_bdsp(const std::string& screen){ return screen == "dialog" || screen == "rival-list-tell" || screen == "rival-list-preset" || screen == "menu"; @@ -199,6 +270,7 @@ struct IntroRun{ SingleSwitchProgramEnvironment& env; ProControllerContext& context; StepTrace& trace; + const IntroPicks& picks; const std::string& player_name; WallClock start; IntroProgress progress; @@ -277,12 +349,91 @@ void probe_for_menu(IntroRun& run){ } } -// Carries out what decide_intro_step() chose for `screen`. -void act(IntroRun& run, const char* screen){ +void advance(IntroRun& run){ + pbf_press_button(run.context, BUTTON_A, 60ms, 400ms); +} + +// Probes the screen with `there` and `back`; whether a cursor moved. +bool probe_moves(IntroRun& run, DpadPosition there, DpadPosition back){ + const CursorProbe probe = probe_cursor(run.env.console, run.context, there, back); + const bool moved = cursor_probe_moved(probe); + run.trace.action("Cursor probe " + dpad_to_string(there) + ": still " + std::to_string(probe.still_change) + + ", press " + std::to_string(probe.press_change) + ", back " + std::to_string(probe.back_change) + + (moved ? ": a cursor moved." : ": no cursor.")); + return moved; +} + +// Moves the cursor with `moves`, saves a screenshot of it on the pick, and confirms with A. +void pick_on_menu(IntroRun& run, const std::vector& moves, const std::string& what){ + run.trace.action("Picking " + what + ": " + std::to_string(moves.size()) + " D-pad presses, then A."); + press_cursor_moves(run.context, moves); + run.context.wait_for_all_requests(); + run.trace.screen("pick-" + what); + press(run, BUTTON_A); + pbf_wait(run.context, 500ms); +} + +// The language select is a list: Down moves its cursor, unless it starts on +// the last item of a list that does not wrap, so Up is tried too. Sideways +// moves only on the appearance grid, which comes later. +void probe_for_language(IntroRun& run){ + if (!probe_moves(run, DPAD_DOWN, DPAD_UP) && !probe_moves(run, DPAD_UP, DPAD_DOWN)){ + advance(run); + return; + } + if (probe_moves(run, DPAD_RIGHT, DPAD_LEFT)){ + run.trace.fail("The appearance grid showed before the language select was found."); + } + pick_on_menu(run, *language_pick_moves(run.picks), "language"); + run.progress.language_pending = false; +} + +// The appearance grid's cursor starts on the first look, so Right moves it. +// The Yes/No boxes around it do not move sideways. +void probe_for_look(IntroRun& run){ + if (!probe_moves(run, DPAD_RIGHT, DPAD_LEFT)){ + advance(run); + return; + } + pick_on_menu(run, *look_pick_moves(run.picks), "look"); + run.progress.look_pending = false; +} + +// PROBE_FOR_LANGUAGE or PROBE_FOR_LOOK. +void probe_for_pick(IntroRun& run, IntroStep step){ + if (step == IntroStep::PROBE_FOR_LANGUAGE){ + probe_for_language(run); + }else{ + probe_for_look(run); + } +} + +// Notes `screen` in the run's progress, decides the step for it and counts it. +IntroStep next_intro_step(IntroRun& run, const char* screen){ note_intro_screen(run.progress, screen, std::chrono::duration_cast(current_time() - run.start)); const IntroStep step = decide_intro_step(screen, run.progress); record_intro_step(run.progress, step); + return step; +} + +// Stops the intro for one of the FAIL_ steps. +[[noreturn]] void fail_intro(const IntroRun& run, IntroStep step){ + if (step == IntroStep::FAIL_NOT_IN_GAME){ + run.trace.fail("The Switch HOME menu is showing: the game is not running in front."); + } + if (step == IntroStep::FAIL_NOT_BDSP){ + run.trace.fail("The game in front is not BDSP: no BDSP screen in " + + std::to_string(run.progress.unconfirmed_for.count() / 1000) + " s of unknown screens."); + } + run.trace.fail(std::string("The player's name keyboard showed before the ") + + (run.progress.language_pending ? "language select" : "appearance grid") + + " was found: no cursor probe above saw its cursor move."); +} + +// Carries out what decide_intro_step() chose for `screen`. +void act(IntroRun& run, const char* screen){ + const IntroStep step = next_intro_step(run, screen); switch (step){ case IntroStep::SELECT_USER: // The cursor starts on the user that launched the game. @@ -290,10 +441,13 @@ void act(IntroRun& run, const char* screen){ pbf_wait(run.context, 1000ms); return; case IntroStep::FAIL_NOT_IN_GAME: - run.trace.fail("The Switch HOME menu is showing: the game is not running in front."); case IntroStep::FAIL_NOT_BDSP: - run.trace.fail("The game in front is not BDSP: no BDSP screen in " + - std::to_string(run.progress.unconfirmed_for.count() / 1000) + " s of unknown screens."); + case IntroStep::FAIL_PICK_MISSED: + fail_intro(run, step); + case IntroStep::PROBE_FOR_LANGUAGE: + case IntroStep::PROBE_FOR_LOOK: + probe_for_pick(run, step); + return; case IntroStep::ENTER_PLAYER_NAME: enter_player_name(run.env.console, run.context, run.trace, run.player_name); return; @@ -313,7 +467,7 @@ void act(IntroRun& run, const char* screen){ close_menu_and_finish(run); return; case IntroStep::ADVANCE: - pbf_press_button(run.context, BUTTON_A, 60ms, 400ms); + advance(run); return; case IntroStep::ADVANCE_WITH_B: case IntroStep::CLEAR_WITH_B: @@ -396,11 +550,20 @@ std::vector intro_states(IntroRun& run){ void play_new_game_intro( SingleSwitchProgramEnvironment& env, ProControllerContext& context, + const IntroPicks& picks, const std::string& player_name ){ VideoStream& stream = env.console; StepTrace trace(stream, "Intro"); + if (!language_pick_moves(picks)){ + throw UserSetupError(env.console, + "BDSP's language select lacks the Switch's language or the game's language: pick ones it lists."); + } + if (!look_pick_moves(picks)){ + throw UserSetupError(env.console, "The player's look must be one of BDSP's eight."); + } + // The restart below reopens the game HOME's cursor is on, which is the // game running in front only while it runs: with none running (r1, // 2026-11-01), closing it fails, HOME is pressed again, the cursor goes @@ -423,7 +586,9 @@ void play_new_game_intro( from_home_close_and_reopen_game(env.console, context, true); wait_out_boot(stream, context, trace); - IntroRun run{env, context, trace, player_name, current_time(), {}, 0}; + IntroRun run{env, context, trace, picks, player_name, current_time(), intro_start(picks), 0}; + trace.action(std::string("Picks: language ") + language_data(picks.game_language).name + " from " + + language_data(picks.switch_language).name + ", look " + std::to_string(picks.look + 1) + "."); const std::vector states = intro_states(run); run_screen_machine(stream, context, trace, states, std::chrono::minutes(15)); trace.action("Intro done."); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h index 35fb8d8ca9..b9dae40d15 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h @@ -4,6 +4,8 @@ * * Plays a new game's intro (language, Professor Rowan, appearance, names) up to * the bedroom with the player in control, which is where the Tutorial starts. + * The language and the player's look are picked from their menus by index + * (NintendoSwitch_MenuCursor.h); the menus' shapes are the data below. * */ @@ -14,14 +16,41 @@ #include #include #include +#include #include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Language.h" #include "NintendoSwitch/NintendoSwitch_SingleSwitchProgram.h" +#include "NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h" namespace PokemonAutomation{ namespace NintendoSwitch{ namespace PokemonBDSP{ +// The language select's items, top to bottom. Its cursor starts on the +// Switch's system language. +const std::vector& bdsp_language_menu(); +// The appearance grid: two rows of four looks, the cursor on the first. +// PlayerModelOption's "Model N" is item N - 1: Lucas's two first looks, +// then Dawn's, on each row. +inline constexpr MenuGrid BDSP_LOOK_GRID{4, 8}; + +// What the intro picks on its menus. +struct IntroPicks{ + // Where the language select's cursor starts. + Language switch_language = Language::English; + Language game_language = Language::English; + // An item of BDSP_LOOK_GRID. + size_t look = 0; +}; +// The presses from the Switch's language to the game's on the language +// select; nullopt when the menu lacks either. +std::optional> language_pick_moves(const IntroPicks& picks); +// The presses from the first look to the chosen one; nullopt when it is +// outside the grid. +std::optional> look_pick_moves(const IntroPicks& picks); + + // Which intro screen `screen` shows, as the intro's steps tell them apart: // "user-select", "home", "keyboard", "rival-list-tell" (the rival's name list on // "I'll tell you..."), "rival-list-preset" (on a preset name), "menu", "dialog" or @@ -35,6 +64,9 @@ enum class IntroStep{ SELECT_USER, // A on the Switch's user select FAIL_NOT_IN_GAME, // HOME is showing FAIL_NOT_BDSP, // no BDSP screen within INTRO_TIME_BEFORE_BDSP + FAIL_PICK_MISSED, // the player's keyboard, with a pick still to make + PROBE_FOR_LANGUAGE, // D-pad probe: a list takes the language pick, else A + PROBE_FOR_LOOK, // D-pad probe: a grid takes the look pick, else A ENTER_PLAYER_NAME, // type the name on the first keyboard CANCEL_RIVAL_KEYBOARD, // X: back to the rival's name list MOVE_TO_PRESET, // Down: off "I'll tell you..." @@ -48,6 +80,10 @@ enum class IntroStep{ }; // Progress tracking state for the new-game introductory sequence. struct IntroProgress{ + // Picks still to make on the language select and the appearance grid. A + // pick the cursor already starts on needs no menu, and A takes it. + bool language_pending = false; + bool look_pending = false; bool player_named = false; // A preset name was picked from the rival's list, and how many screens // have shown since without the list coming back. "Barry, is it?" and @@ -74,7 +110,9 @@ struct IntroProgress{ // and the language select. Another game launched by mistake (r1, // 2026-11-01: FireRed) fails here instead of being played. Black screens // take no presses: BDSP boots on black for about 25 s (r3, 2026-11-01). -inline constexpr std::chrono::milliseconds INTRO_TIME_BEFORE_BDSP = std::chrono::seconds(60); +// While the language pick waits for its menu, each of those screens also +// takes two cursor probes of about 1.4 s each, hence two minutes. +inline constexpr std::chrono::milliseconds INTRO_TIME_BEFORE_BDSP = std::chrono::seconds(120); // How long the boot may stay black after the restart. inline constexpr std::chrono::milliseconds INTRO_BOOT_LIMIT = std::chrono::seconds(60); // Whether `screen` (classify_intro_screen()) is one only BDSP's intro shows: @@ -126,18 +164,25 @@ bool intro_in_control_without_menu(const XProbe& probe); // Counts `step` in `progress`. void record_intro_step(IntroProgress& progress, IntroStep step); +// The progress a new intro starts from: which of `picks` need a menu. +IntroProgress intro_start(const IntroPicks& picks); + // From the title screen or any intro screen of a game with no save, with BDSP // running in front: from HOME it throws UserSetupError, since the restart would -// launch whatever tile HOME's cursor is on. Restarts the game first, so it +// launch whatever tile HOME's cursor is on, as it does for picks no menu has. Restarts the game first, so it // always starts from the title, waits out the black boot (up to // INTRO_BOOT_LIMIT) without pressing anything, and fails if no BDSP screen shows within -// INTRO_TIME_BEFORE_BDSP of screens. Names the player `player_name` -// and picks the rival's default name (Barry) from the list. Each step reads the +// INTRO_TIME_BEFORE_BDSP of screens. Picks the language and the look in +// `picks`, names the player `player_name` and picks the rival's default name +// (Barry) from the list. Until a pick is made, each unknown screen gets a +// D-pad probe before its A; the player's keyboard showing with a pick still +// to make fails. Each step reads the // screen and waits for the expected next one; an unexpected screen fails with a // screenshot and the recent screens. void play_new_game_intro( SingleSwitchProgramEnvironment& env, ProControllerContext& context, + const IntroPicks& picks, const std::string& player_name ); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp index a3c7c676f9..16f5309bbe 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp @@ -16,6 +16,7 @@ #include "Pokemon/AutoStory/Pokemon_AutoStoryStats.h" #include "Pokemon/AutoStory/Pokemon_StorySegments.h" #include "PokemonBDSP/Programs/PokemonBDSP_GameEntry.h" +#include "PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h" #include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" #include "PokemonBDSP_AutoStory_StoryTable.h" #include "PokemonBDSP_AutoStory.h" @@ -165,6 +166,23 @@ void verify_story_checkpoint( } // namespace +namespace{ + +// The languages BDSP's language select lists, in its order. +const EnumDropdownDatabase& switch_language_database(){ + static const EnumDropdownDatabase database = []{ + EnumDropdownDatabase ret; + for (Language language : bdsp_language_menu()){ + ret.add(language, language_data(language).code, language_data(language).name, true); + } + return ret; + }(); + return database; +} + +} // namespace + + BDSPAutoStory::BDSPAutoStory() : RUN(checkpoint_database()) , NEW_GAME_INTRO( @@ -180,6 +198,14 @@ BDSPAutoStory::BDSPAutoStory() LockMode::LOCK_WHILE_RUNNING, false ) , PLAYER_NAME(false, "Player name:
For a new game. Letters and digits.", LockMode::LOCK_WHILE_RUNNING, "Kosmos", "Kosmos") + , SWITCH_LANGUAGE( + "Switch language:
For a new game: the Switch's system language, where BDSP's language select starts its cursor.", + switch_language_database(), LockMode::LOCK_WHILE_RUNNING, Language::English + ) + , PLAYER_LOOK( + "Player's look:
For a new game: the look picked on the appearance grid. " + "Models 1, 2, 5 and 6 are Lucas; 3, 4, 7 and 8 are Dawn." + ) , STARTER( "Starter Pokemon:", { @@ -200,6 +226,8 @@ BDSPAutoStory::BDSPAutoStory() PA_ADD_OPTION(NEW_GAME_INTRO); PA_ADD_OPTION(JUST_LOADED); PA_ADD_OPTION(PLAYER_NAME); + PA_ADD_OPTION(SWITCH_LANGUAGE); + PA_ADD_OPTION(PLAYER_LOOK); PA_ADD_OPTION(STARTER); PA_ADD_OPTION(NOTIFICATIONS); } @@ -228,7 +256,7 @@ void BDSPAutoStory::program(SingleSwitchProgramEnvironment& env, ProControllerCo // The reads that use text (Location.h) take the game's language from here. set_story_language(RUN.language()); - StoryEnv story{env, context, STARTER, PLAYER_NAME}; + StoryEnv story{env, context, STARTER, PLAYER_NAME, IntroPicks{SWITCH_LANGUAGE, RUN.language(), PLAYER_LOOK.index()}}; StoryExecutor executor(story); executor.set_checks(RUN.checks()); executor.set_language(RUN.language()); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h index 9fe9b929a5..753bdde767 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h @@ -19,6 +19,7 @@ #include "CommonFramework/Notifications/EventNotificationsTable.h" #include "NintendoSwitch/NintendoSwitch_SingleSwitchProgram.h" #include "Pokemon/Options/Pokemon_StoryRunOptions.h" +#include "PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h" #include "PokemonBDSP_AutoStory_StoryExecutor.h" namespace PokemonAutomation{ @@ -49,6 +50,8 @@ class BDSPAutoStory : public SingleSwitchProgramInstance{ BooleanCheckBoxOption NEW_GAME_INTRO; BooleanCheckBoxOption JUST_LOADED; StringOption PLAYER_NAME; + EnumDropdownOption SWITCH_LANGUAGE; + PlayerModelOption PLAYER_LOOK; EnumDropdownOption STARTER; EventNotificationsOption NOTIFICATIONS; diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_StoryExecutor.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_StoryExecutor.cpp index 756a377fbc..8e0ec5d42b 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_StoryExecutor.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_StoryExecutor.cpp @@ -189,7 +189,7 @@ void StoryExecutor::begin_attempt(const Pokemon::StoryCheckpoint& data){ } Outcome StoryExecutor::new_game(RunState&, const StoryStep&){ - play_new_game_intro(m_env.env, context(), m_env.player_name); + play_new_game_intro(m_env.env, context(), m_env.intro_picks, m_env.player_name); return Pokemon::outcome_ok(); } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_StoryExecutor.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_StoryExecutor.h index 178719d044..f1882551ee 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_StoryExecutor.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_StoryExecutor.h @@ -33,6 +33,7 @@ #include #include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" #include "NintendoSwitch/NintendoSwitch_SingleSwitchProgram.h" +#include "PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h" #include "PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_StoryWorld.h" #include "Pokemon/AutoStory/Pokemon_StoryActions.h" #include "Pokemon/AutoStory/Pokemon_StorySegments.h" @@ -81,6 +82,8 @@ struct StoryEnv{ ProControllerContext& context; StarterChoice starter = StarterChoice::CHIMCHAR; std::string player_name; + // The new game's language and look. + IntroPicks intro_picks = {}; }; diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index aebd01e09d..727a6d1b23 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1425,6 +1425,10 @@ file(GLOB LIBRARY_SOURCES Source/NintendoSwitch/Programs/NintendoSwitch_TurboButton.h Source/NintendoSwitch/Programs/NintendoSwitch_TurboMacro.cpp Source/NintendoSwitch/Programs/NintendoSwitch_TurboMacro.h + Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp + Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h + Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp + Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h Source/NintendoSwitch/Programs/Typewriter/NintendoSwitch_Typewriter.cpp Source/NintendoSwitch/Programs/Typewriter/NintendoSwitch_Typewriter.h Source/NintendoSwitch/Programs/Typewriter/NintendoSwitch_Typewriter_Tests.cpp From 3f009a4bc7558e179dfc2e585370d064d9192b0f Mon Sep 17 00:00:00 2001 From: dayfine Date: Mon, 9 Nov 2026 11:14:14 -0800 Subject: [PATCH 02/19] BDSP AutoStory: read the language select with a detector The language select is a 2 x 5 grid of buttons, not a 9-item list (lab-B frame, 2026-11-09): English, Espanol, Francais, Deutsch, Italiano on the left; Japanese kana and kanji, Korean, Simplified and Traditional Chinese on the right; the cursor starts on English. Right moves on it, so the probe rule took it for the appearance grid. A detector reads it from its ten buttons (grey, the cursor's blue) and which one the cursor is on. The intro classifies it as its own screen, moves from the read cursor to the game's language, reads it again (up to three tries), then A. The look grid keeps the D-pad probe, only after the language is picked. The Switch language option is gone: the cursor is read, not assumed. The wrong-game guard is back to 60 s. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 112 ++++++++++++------ .../Actions/PokemonBDSP_AutoStory_NewGame.cpp | 111 ++++++++++------- .../Actions/PokemonBDSP_AutoStory_NewGame.h | 40 ++++--- ...nBDSP_AutoStory_LanguageSelectDetector.cpp | 79 ++++++++++++ ...monBDSP_AutoStory_LanguageSelectDetector.h | 36 ++++++ .../AutoStory/PokemonBDSP_AutoStory.cpp | 24 +--- .../AutoStory/PokemonBDSP_AutoStory.h | 1 - SerialPrograms/cmake/SourceFiles.cmake | 2 + 8 files changed, 282 insertions(+), 123 deletions(-) create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index 5cd0661f9b..5283dcb2c4 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -23,6 +23,7 @@ #include "NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h" #include "PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_StoryMaps.h" #include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h" #include "PokemonBDSP_AutoStory_Center.h" #include "PokemonBDSP_AutoStory_NewGame.h" #include "PokemonBDSP_AutoStory_Actions_Tests.h" @@ -203,7 +204,7 @@ std::string check_boot(IntroProgress& progress){ } UnitTestResult check_confirming_screens(){ - for (const char* screen : {"dialog", "rival-list-tell", "rival-list-preset", "menu"}){ + for (const char* screen : {"dialog", "language-select", "rival-list-tell", "rival-list-preset", "menu"}){ if (!intro_screen_confirms_bdsp(screen)){ return std::string("Expected \"") + screen + "\" to confirm BDSP."; } @@ -251,50 +252,59 @@ UnitTestResult test_intro_gate(Logger&){ } -// The presses for the language select (a list starting on the Switch's -// language) and the appearance grid (two rows of four, from the first look). +// The presses for the language select (two columns of five, read row by +// row, from wherever its cursor is) and the appearance grid (two rows of +// four, from the first look). UnitTestResult test_intro_pick_moves(Logger&){ using Moves = std::optional>; + const IntroPicks french{Language::French, 2}; struct Case{ const char* what; Moves actual; Moves expected; }; const Case cases[] = { - {"French on an English Switch (lab run 3)", - language_pick_moves({Language::English, Language::French, 0}), Moves(std::vector{DPAD_DOWN})}, - {"the Switch's own language", - language_pick_moves({Language::English, Language::English, 0}), Moves(std::vector{})}, - {"English on a French Switch", - language_pick_moves({Language::French, Language::English, 0}), Moves(std::vector{DPAD_UP})}, - {"top to bottom", - language_pick_moves({Language::Japanese, Language::ChineseTraditional, 0}), Moves{std::vector(8, DPAD_DOWN)}}, + {"French from English, where the cursor starts (lab run 3)", + language_pick_moves(0, french), Moves(std::vector{DPAD_DOWN, DPAD_DOWN})}, + {"English from English", + language_pick_moves(0, IntroPicks{Language::English, 0}), Moves(std::vector{})}, + {"French from Traditional Chinese, bottom right", + language_pick_moves(9, french), Moves(std::vector{DPAD_UP, DPAD_UP, DPAD_LEFT})}, + {"Japanese: its kana button, right of English", + language_pick_moves(0, IntroPicks{Language::Japanese, 0}), Moves(std::vector{DPAD_RIGHT})}, + {"Traditional Chinese from English", + language_pick_moves(0, IntroPicks{Language::ChineseTraditional, 0}), + Moves(std::vector{DPAD_RIGHT, DPAD_DOWN, DPAD_DOWN, DPAD_DOWN, DPAD_DOWN})}, + {"Italian, bottom left", + language_pick_moves(0, IntroPicks{Language::Italian, 0}), Moves(std::vector(4, DPAD_DOWN))}, {"a language the menu lacks", - language_pick_moves({Language::English, Language::None, 0}), std::nullopt}, + language_pick_moves(0, IntroPicks{Language::None, 0}), std::nullopt}, + {"a cursor outside the menu", + language_pick_moves(LANGUAGE_SELECT_BUTTONS, french), std::nullopt}, {"Model 1, where the cursor starts", - look_pick_moves({Language::English, Language::English, 0}), Moves(std::vector{})}, + look_pick_moves(IntroPicks{Language::English, 0}), Moves(std::vector{})}, {"Model 3, Dawn's first look (lab run 3)", - look_pick_moves({Language::English, Language::English, 2}), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT})}, + look_pick_moves(french), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT})}, {"Model 8, the last", - look_pick_moves({Language::English, Language::English, 7}), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT, DPAD_RIGHT, DPAD_DOWN})}, + look_pick_moves(IntroPicks{Language::English, 7}), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT, DPAD_RIGHT, DPAD_DOWN})}, {"no Model 9", - look_pick_moves({Language::English, Language::English, 8}), std::nullopt}, + look_pick_moves(IntroPicks{Language::English, 8}), std::nullopt}, }; for (const Case& c : cases){ if (c.actual != c.expected){ return std::string("Wrong presses for ") + c.what + "."; } } - if (bdsp_language_menu().size() != 9){ - return "BDSP's language select lists nine languages."; + if (bdsp_language_menu().size() != BDSP_LANGUAGE_GRID.count || BDSP_LANGUAGE_GRID.count != LANGUAGE_SELECT_BUTTONS){ + return "The language select's data, grid and detector must agree on ten buttons."; } return true; } -// Until its pick is made, every screen that would take A gets a probe -// first: the language's before the look's. The player's keyboard with a -// pick still to make fails. Picks the cursor starts on need no probe. -UnitTestResult test_intro_pick_probes(Logger&){ +// The language select takes the language pick, then A if it still shows. +// Until the look is picked, every screen after the language that would take +// A gets a probe first. The player's keyboard with a pick still to make fails. +UnitTestResult test_intro_pick_steps(Logger&){ struct Case{ const char* what; const char* screen; @@ -303,12 +313,14 @@ UnitTestResult test_intro_pick_probes(Logger&){ IntroStep expected; }; const Case cases[] = { - {"a logo, language to pick", "other", true, true, IntroStep::PROBE_FOR_LANGUAGE}, - {"a dialogue, language to pick", "dialog", true, false, IntroStep::PROBE_FOR_LANGUAGE}, + {"the language select", "language-select", true, true, IntroStep::PICK_LANGUAGE}, + {"the language select, picked", "language-select", false, true, IntroStep::ADVANCE}, + {"a logo before the language", "other", true, true, IntroStep::ADVANCE}, + {"a dialogue before the language", "dialog", true, true, IntroStep::ADVANCE}, {"Rowan, look to pick", "dialog", false, true, IntroStep::PROBE_FOR_LOOK}, {"the preview, look to pick", "other", false, true, IntroStep::PROBE_FOR_LOOK}, {"both picked", "dialog", false, false, IntroStep::ADVANCE}, - {"black, language to pick", "black", true, true, IntroStep::WAIT}, + {"black, look to pick", "black", false, true, IntroStep::WAIT}, {"the user select", "user-select", true, true, IntroStep::SELECT_USER}, {"the keyboard, language missed", "keyboard", true, false, IntroStep::FAIL_PICK_MISSED}, {"the keyboard, look missed", "keyboard", false, true, IntroStep::FAIL_PICK_MISSED}, @@ -322,20 +334,40 @@ UnitTestResult test_intro_pick_probes(Logger&){ return std::string("Wrong step for ") + c.what + "."; } } - // The wrong-game guard still stops a game that never shows BDSP's screens. - IntroProgress other_game = intro_start({Language::English, Language::French, 2}); - note_intro_screen(other_game, "other", std::chrono::milliseconds(0)); - note_intro_screen(other_game, "other", INTRO_TIME_BEFORE_BDSP); - if (decide_intro_step("other", other_game) != IntroStep::FAIL_NOT_BDSP){ - return "Past INTRO_TIME_BEFORE_BDSP, a pending pick must not keep another game going."; - } - const IntroProgress defaults = intro_start({Language::English, Language::English, 0}); - if (defaults.language_pending || defaults.look_pending){ - return "Picks the cursor starts on need no menu."; + const IntroProgress first_look = intro_start(IntroPicks{Language::English, 0}); + if (!first_look.language_pending || first_look.look_pending){ + return "The language is always picked on its select; the first look needs no grid."; } - const IntroProgress run3 = intro_start({Language::English, Language::French, 2}); + const IntroProgress run3 = intro_start(IntroPicks{Language::French, 2}); if (!run3.language_pending || !run3.look_pending){ - return "French and Model 3 on an English Switch need both menus."; + return "French and Model 3 need both menus."; + } + return true; +} + +// The language select from lab-B 2026-11-09 (cursor on English), and screens +// that are not it. Screenshots are in the unit test resources; a missing one +// is skipped. +UnitTestResult test_language_select_screenshots(Logger& logger){ + struct Case{ + const char* file; + std::optional cursor; + }; + const Case cases[] = { + {.file = "PokemonBDSP/AutoStory/LanguageSelect-20261109.png", .cursor = 0}, + {.file = "PokemonBDSP/AutoStory/OreburghCenterCounter-20261108-094735.png", .cursor = std::nullopt}, + {.file = "PokemonBDSP/AutoStory/OreburghCenterDoorstep-20261108.png", .cursor = std::nullopt}, + }; + for (const Case& c : cases){ + const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + if (!Filesystem::is_regular_file(path)){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + } + const std::optional cursor = language_select_cursor(ImageRGB32(path)); + logger.log(std::string(c.file) + ": cursor " + (cursor ? std::to_string(*cursor) : "none")); + if (cursor != c.cursor){ + return std::string(c.file) + ": wrong language select read."; + } } return true; } @@ -582,7 +614,11 @@ const AutoStoryActionsCase AUTO_STORY_ACTIONS_CASES[] = { .expectation = {}, }, { - .input = {"new-game intro: probes until the picks are made", test_intro_pick_probes}, + .input = {"new-game intro: steps until the picks are made", test_intro_pick_steps}, + .expectation = {}, + }, + { + .input = {"language select screenshots", test_language_select_screenshots}, .expectation = {}, }, { diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp index 162a9fbad3..540ccb9153 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp @@ -22,6 +22,7 @@ #include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" #include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" #include "PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h" #include "PokemonBDSP_AutoStory_NewGame.h" namespace PokemonAutomation{ @@ -33,15 +34,11 @@ using namespace std::chrono_literals; const std::vector& bdsp_language_menu(){ static const std::vector menu{ - Language::Japanese, - Language::English, - Language::French, - Language::Italian, - Language::German, - Language::Spanish, - Language::Korean, - Language::ChineseSimplified, - Language::ChineseTraditional, + Language::English, Language::Japanese, + Language::Spanish, Language::Japanese, + Language::French, Language::Korean, + Language::German, Language::ChineseSimplified, + Language::Italian, Language::ChineseTraditional, }; return menu; } @@ -59,13 +56,12 @@ std::optional language_item(Language language){ } // namespace -std::optional> language_pick_moves(const IntroPicks& picks){ - const std::optional from = language_item(picks.switch_language); +std::optional> language_pick_moves(size_t cursor, const IntroPicks& picks){ const std::optional to = language_item(picks.game_language); - if (!from || !to){ + if (!to){ return std::nullopt; } - return menu_cursor_moves(MenuGrid{1, bdsp_language_menu().size()}, *from, *to); + return menu_cursor_moves(BDSP_LANGUAGE_GRID, cursor, *to); } std::optional> look_pick_moves(const IntroPicks& picks){ return menu_cursor_moves(BDSP_LOOK_GRID, 0, picks.look); @@ -73,7 +69,7 @@ std::optional> look_pick_moves(const IntroPicks& picks IntroProgress intro_start(const IntroPicks& picks){ IntroProgress progress; - progress.language_pending = picks.switch_language != picks.game_language; + progress.language_pending = true; progress.look_pending = picks.look != 0; return progress; } @@ -90,6 +86,10 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& if (SystemKeyboardDetector().detect(screen)){ return "keyboard"; } + // Before the dialogue box: its prompt sits in one. + if (language_select_cursor(screen)){ + return "language-select"; + } switch (NameListDetector().selected_row(screen)){ case -1: break; @@ -115,16 +115,14 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& namespace{ -// A on a screen that may take A, after a D-pad probe while a pick waits for -// its menu. The picks come before the player's name. +// A on a screen that may take A, after a D-pad probe while the look waits +// for its grid. The language select comes first, then the look, then the +// player's name. IntroStep advance_or_probe(const IntroProgress& progress){ if (progress.player_named){ return IntroStep::ADVANCE; } - if (progress.language_pending){ - return IntroStep::PROBE_FOR_LANGUAGE; - } - if (progress.look_pending){ + if (!progress.language_pending && progress.look_pending){ return IntroStep::PROBE_FOR_LOOK; } return IntroStep::ADVANCE; @@ -145,6 +143,10 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog } return progress.player_named ? IntroStep::CANCEL_RIVAL_KEYBOARD : IntroStep::ENTER_PLAYER_NAME; } + if (screen == "language-select"){ + // Once picked, A again only if the select still shows. + return progress.language_pending ? IntroStep::PICK_LANGUAGE : IntroStep::ADVANCE; + } if (screen == "rival-list-tell"){ return IntroStep::MOVE_TO_PRESET; } @@ -173,7 +175,7 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog return advance_or_probe(progress); } bool intro_screen_confirms_bdsp(const std::string& screen){ - return screen == "dialog" || screen == "rival-list-tell" || screen == "rival-list-preset" || screen == "menu"; + return screen == "dialog" || screen == "language-select" || screen == "rival-list-tell" || screen == "rival-list-preset" || screen == "menu"; } void note_intro_screen(IntroProgress& progress, const std::string& screen, std::chrono::milliseconds since_start){ @@ -373,19 +375,40 @@ void pick_on_menu(IntroRun& run, const std::vector& moves, const s pbf_wait(run.context, 500ms); } -// The language select is a list: Down moves its cursor, unless it starts on -// the last item of a list that does not wrap, so Up is tried too. Sideways -// moves only on the appearance grid, which comes later. -void probe_for_language(IntroRun& run){ - if (!probe_moves(run, DPAD_DOWN, DPAD_UP) && !probe_moves(run, DPAD_UP, DPAD_DOWN)){ - advance(run); - return; +// Reads of the language select before the cursor must be on the game's language. +const size_t LANGUAGE_PICK_TRIES = 3; + +// The language select's cursor as read now; nullopt when it is not showing. +std::optional read_language_cursor(IntroRun& run){ + run.context.wait_for_all_requests(); + const VideoSnapshot screen = run.env.console.video().snapshot(); + if (!screen){ + return std::nullopt; } - if (probe_moves(run, DPAD_RIGHT, DPAD_LEFT)){ - run.trace.fail("The appearance grid showed before the language select was found."); + return language_select_cursor(*screen.frame); +} + +// Reads the cursor, moves it to the game's language and reads it there, +// up to LANGUAGE_PICK_TRIES times (a press dropped as the screen fades in), +// then A. +void pick_language(IntroRun& run){ + const std::optional target = language_item(run.picks.game_language); + for (size_t attempt = 0; attempt < LANGUAGE_PICK_TRIES; attempt++){ + const std::optional cursor = read_language_cursor(run); + if (!cursor){ + return; // Gone or fading: the next read decides. + } + run.trace.action("Language select: cursor on item " + std::to_string(*cursor) + + ", the game's language is item " + std::to_string(*target) + "."); + if (*cursor == *target){ + pick_on_menu(run, {}, "language"); + run.progress.language_pending = false; + return; + } + press_cursor_moves(run.context, *language_pick_moves(*cursor, run.picks)); } - pick_on_menu(run, *language_pick_moves(run.picks), "language"); - run.progress.language_pending = false; + run.trace.fail("The language select's cursor did not reach the game's language in " + + std::to_string(LANGUAGE_PICK_TRIES) + " tries."); } // The appearance grid's cursor starts on the first look, so Right moves it. @@ -399,10 +422,10 @@ void probe_for_look(IntroRun& run){ run.progress.look_pending = false; } -// PROBE_FOR_LANGUAGE or PROBE_FOR_LOOK. -void probe_for_pick(IntroRun& run, IntroStep step){ - if (step == IntroStep::PROBE_FOR_LANGUAGE){ - probe_for_language(run); +// PICK_LANGUAGE or PROBE_FOR_LOOK. +void pick_on_intro_menu(IntroRun& run, IntroStep step){ + if (step == IntroStep::PICK_LANGUAGE){ + pick_language(run); }else{ probe_for_look(run); } @@ -444,9 +467,9 @@ void act(IntroRun& run, const char* screen){ case IntroStep::FAIL_NOT_BDSP: case IntroStep::FAIL_PICK_MISSED: fail_intro(run, step); - case IntroStep::PROBE_FOR_LANGUAGE: + case IntroStep::PICK_LANGUAGE: case IntroStep::PROBE_FOR_LOOK: - probe_for_pick(run, step); + pick_on_intro_menu(run, step); return; case IntroStep::ENTER_PLAYER_NAME: enter_player_name(run.env.console, run.context, run.trace, run.player_name); @@ -521,6 +544,9 @@ std::vector intro_states(IntroRun& run){ {"rival-keyboard", [=, &run](const ImageViewRGB32& s){ return run.progress.player_named && screen_is(s, "keyboard"); }, [&run]{ act(run, "keyboard"); }, {"rival-list-tell", "dialog", "black", "other"}, 3}, + {"language-select", + [=](const ImageViewRGB32& s){ return screen_is(s, "language-select"); }, + [&run]{ act(run, "language-select"); }, {}, 10}, {"rival-list-tell", [=](const ImageViewRGB32& s){ return screen_is(s, "rival-list-tell"); }, [&run]{ act(run, "rival-list-tell"); }, {"rival-list-preset", "black", "other"}, 6}, @@ -556,9 +582,8 @@ void play_new_game_intro( VideoStream& stream = env.console; StepTrace trace(stream, "Intro"); - if (!language_pick_moves(picks)){ - throw UserSetupError(env.console, - "BDSP's language select lacks the Switch's language or the game's language: pick ones it lists."); + if (!language_pick_moves(0, picks)){ + throw UserSetupError(env.console, "BDSP's language select lacks the game's language: pick one it lists."); } if (!look_pick_moves(picks)){ throw UserSetupError(env.console, "The player's look must be one of BDSP's eight."); @@ -587,8 +612,8 @@ void play_new_game_intro( wait_out_boot(stream, context, trace); IntroRun run{env, context, trace, picks, player_name, current_time(), intro_start(picks), 0}; - trace.action(std::string("Picks: language ") + language_data(picks.game_language).name + " from " + - language_data(picks.switch_language).name + ", look " + std::to_string(picks.look + 1) + "."); + trace.action(std::string("Picks: language ") + language_data(picks.game_language).name + + ", look " + std::to_string(picks.look + 1) + "."); const std::vector states = intro_states(run); run_screen_machine(stream, context, trace, states, std::chrono::minutes(15)); trace.action("Intro done."); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h index b9dae40d15..d3e8516d24 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h @@ -6,6 +6,8 @@ * the bedroom with the player in control, which is where the Tutorial starts. * The language and the player's look are picked from their menus by index * (NintendoSwitch_MenuCursor.h); the menus' shapes are the data below. + * The language select is read by its detector; the appearance grid, which + * has none yet, is found with a D-pad probe. * */ @@ -27,8 +29,13 @@ namespace NintendoSwitch{ namespace PokemonBDSP{ -// The language select's items, top to bottom. Its cursor starts on the -// Switch's system language. +// The language select: two columns of five buttons, English, Espanol, +// Francais, Deutsch, Italiano on the left; Japanese in kana, Japanese in +// kanji, Korean, Simplified and Traditional Chinese on the right (lab-B +// 2026-11-09). Items are numbered row by row, as MenuGrid reads them. +inline constexpr MenuGrid BDSP_LANGUAGE_GRID{2, 10}; +// The language of each item of BDSP_LANGUAGE_GRID. Japanese is there twice; +// its first item, the kana, is the one picked. const std::vector& bdsp_language_menu(); // The appearance grid: two rows of four looks, the cursor on the first. // PlayerModelOption's "Model N" is item N - 1: Lucas's two first looks, @@ -37,22 +44,20 @@ inline constexpr MenuGrid BDSP_LOOK_GRID{4, 8}; // What the intro picks on its menus. struct IntroPicks{ - // Where the language select's cursor starts. - Language switch_language = Language::English; Language game_language = Language::English; // An item of BDSP_LOOK_GRID. size_t look = 0; }; -// The presses from the Switch's language to the game's on the language -// select; nullopt when the menu lacks either. -std::optional> language_pick_moves(const IntroPicks& picks); +// The presses from the language select's item `cursor` to the game's +// language; nullopt when the menu lacks it or `cursor` is outside it. +std::optional> language_pick_moves(size_t cursor, const IntroPicks& picks); // The presses from the first look to the chosen one; nullopt when it is // outside the grid. std::optional> look_pick_moves(const IntroPicks& picks); // Which intro screen `screen` shows, as the intro's steps tell them apart: -// "user-select", "home", "keyboard", "rival-list-tell" (the rival's name list on +// "user-select", "home", "keyboard", "language-select", "rival-list-tell" (the rival's name list on // "I'll tell you..."), "rival-list-preset" (on a preset name), "menu", "dialog" or // "black" (the boot, fades) or "other". The Switch's own screens need // `console`; without it they read as whatever else they look like. @@ -65,7 +70,7 @@ enum class IntroStep{ FAIL_NOT_IN_GAME, // HOME is showing FAIL_NOT_BDSP, // no BDSP screen within INTRO_TIME_BEFORE_BDSP FAIL_PICK_MISSED, // the player's keyboard, with a pick still to make - PROBE_FOR_LANGUAGE, // D-pad probe: a list takes the language pick, else A + PICK_LANGUAGE, // the language select: move to the game's language, A PROBE_FOR_LOOK, // D-pad probe: a grid takes the look pick, else A ENTER_PLAYER_NAME, // type the name on the first keyboard CANCEL_RIVAL_KEYBOARD, // X: back to the rival's name list @@ -80,8 +85,8 @@ enum class IntroStep{ }; // Progress tracking state for the new-game introductory sequence. struct IntroProgress{ - // Picks still to make on the language select and the appearance grid. A - // pick the cursor already starts on needs no menu, and A takes it. + // Picks still to make on the language select and the appearance grid. + // The first look needs no menu: A takes it. bool language_pending = false; bool look_pending = false; bool player_named = false; @@ -110,13 +115,11 @@ struct IntroProgress{ // and the language select. Another game launched by mistake (r1, // 2026-11-01: FireRed) fails here instead of being played. Black screens // take no presses: BDSP boots on black for about 25 s (r3, 2026-11-01). -// While the language pick waits for its menu, each of those screens also -// takes two cursor probes of about 1.4 s each, hence two minutes. -inline constexpr std::chrono::milliseconds INTRO_TIME_BEFORE_BDSP = std::chrono::seconds(120); +inline constexpr std::chrono::milliseconds INTRO_TIME_BEFORE_BDSP = std::chrono::seconds(60); // How long the boot may stay black after the restart. inline constexpr std::chrono::milliseconds INTRO_BOOT_LIMIT = std::chrono::seconds(60); // Whether `screen` (classify_intro_screen()) is one only BDSP's intro shows: -// its dialogue box, the rival's name list, its main menu. The keyboard and +// its dialogue box, its language select, the rival's name list, its main menu. The keyboard and // the user select are the Switch's. bool intro_screen_confirms_bdsp(const std::string& screen); @@ -174,9 +177,10 @@ IntroProgress intro_start(const IntroPicks& picks); // INTRO_BOOT_LIMIT) without pressing anything, and fails if no BDSP screen shows within // INTRO_TIME_BEFORE_BDSP of screens. Picks the language and the look in // `picks`, names the player `player_name` and picks the rival's default name -// (Barry) from the list. Until a pick is made, each unknown screen gets a -// D-pad probe before its A; the player's keyboard showing with a pick still -// to make fails. Each step reads the +// (Barry) from the list. The language select is read, moved on and read +// again before its A. Until the look is picked, each unknown screen after it +// gets a D-pad probe before its A. The player's keyboard showing with a pick +// still to make fails. Each step reads the // screen and waits for the expected next one; an unexpected screen fails with a // screenshot and the recent screens. void play_new_game_intro( diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp new file mode 100644 index 0000000000..66384f774c --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp @@ -0,0 +1,79 @@ +/* BDSP AutoStory - Language Select Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "PokemonBDSP_AutoStory_LanguageSelectDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ + +// A strip inside each button, left of its text (1080p: x 330-460 on the left +// column, 1040-1170 on the right; 60 px tall around each row's centre). +const double COLUMN_X[2] = {0.172, 0.542}; +const double ROW_CENTER_Y[5] = {0.154, 0.280, 0.404, 0.528, 0.652}; +const double STRIP_WIDTH = 0.068; +const double STRIP_HEIGHT = 0.056; + +// Buttons are flat: the frame's measured stddev sum is 2 to 6. +const double MAX_STDDEV_SUM = 15; + +enum class ButtonLook{ GREY, BLUE, OTHER }; + +ImageFloatBox button_strip(size_t item){ + return ImageFloatBox(COLUMN_X[item % 2], ROW_CENTER_Y[item / 2] - STRIP_HEIGHT / 2, STRIP_WIDTH, STRIP_HEIGHT); +} + +// Grey (225, 233, 238) and blue (167, 206, 255) on the frame. The background +// between the buttons is a bluer (203, 232, 253), not grey. +ButtonLook button_look(const ImageViewRGB32& screen, size_t item){ + const ImageStats stats = image_stats(extract_box_reference(screen, button_strip(item))); + if (stats.stddev.sum() > MAX_STDDEV_SUM){ + return ButtonLook::OTHER; + } + const FloatPixel& c = stats.average; + const double low = std::min({c.r, c.g, c.b}); + const double high = std::max({c.r, c.g, c.b}); + if (low >= 200 && high - low <= 25){ + return ButtonLook::GREY; + } + if (c.b >= 230 && c.r >= 120 && c.b - c.r >= 50){ + return ButtonLook::BLUE; + } + return ButtonLook::OTHER; +} + +} // namespace + + +std::optional language_select_cursor(const ImageViewRGB32& screen){ + std::optional cursor; + for (size_t item = 0; item < LANGUAGE_SELECT_BUTTONS; item++){ + switch (button_look(screen, item)){ + case ButtonLook::GREY: + break; + case ButtonLook::BLUE: + if (cursor){ + return std::nullopt; + } + cursor = item; + break; + case ButtonLook::OTHER: + return std::nullopt; + } + } + return cursor; +} + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h new file mode 100644 index 0000000000..29524a35c4 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h @@ -0,0 +1,36 @@ +/* BDSP AutoStory - Language Select Detector + * + * From: https://github.com/PokemonAutomation/ + * + * BDSP's language select, shown on a new game's first boot: ten buttons in + * two columns of five over a crystal background, "Select the language you + * would like to play in." below them. The cursor's button is blue with a red + * outline, the others light grey (lab-B 2026-11-09, 1080p). + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_LanguageSelectDetector_H +#define PokemonAutomation_PokemonBDSP_AutoStory_LanguageSelectDetector_H + +#include +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// The buttons, numbered row by row: English 0, its right neighbour 1, the +// next row 2 and 3, and so on (BDSP_LANGUAGE_GRID). +inline constexpr size_t LANGUAGE_SELECT_BUTTONS = 10; + +// The button the cursor is on, or nullopt when `screen` is not the language +// select: every button must read as plain grey but exactly one, blue. +std::optional language_select_cursor(const ImageViewRGB32& screen); + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp index 16f5309bbe..3684ac7733 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp @@ -166,23 +166,6 @@ void verify_story_checkpoint( } // namespace -namespace{ - -// The languages BDSP's language select lists, in its order. -const EnumDropdownDatabase& switch_language_database(){ - static const EnumDropdownDatabase database = []{ - EnumDropdownDatabase ret; - for (Language language : bdsp_language_menu()){ - ret.add(language, language_data(language).code, language_data(language).name, true); - } - return ret; - }(); - return database; -} - -} // namespace - - BDSPAutoStory::BDSPAutoStory() : RUN(checkpoint_database()) , NEW_GAME_INTRO( @@ -198,10 +181,6 @@ BDSPAutoStory::BDSPAutoStory() LockMode::LOCK_WHILE_RUNNING, false ) , PLAYER_NAME(false, "Player name:
For a new game. Letters and digits.", LockMode::LOCK_WHILE_RUNNING, "Kosmos", "Kosmos") - , SWITCH_LANGUAGE( - "Switch language:
For a new game: the Switch's system language, where BDSP's language select starts its cursor.", - switch_language_database(), LockMode::LOCK_WHILE_RUNNING, Language::English - ) , PLAYER_LOOK( "Player's look:
For a new game: the look picked on the appearance grid. " "Models 1, 2, 5 and 6 are Lucas; 3, 4, 7 and 8 are Dawn." @@ -226,7 +205,6 @@ BDSPAutoStory::BDSPAutoStory() PA_ADD_OPTION(NEW_GAME_INTRO); PA_ADD_OPTION(JUST_LOADED); PA_ADD_OPTION(PLAYER_NAME); - PA_ADD_OPTION(SWITCH_LANGUAGE); PA_ADD_OPTION(PLAYER_LOOK); PA_ADD_OPTION(STARTER); PA_ADD_OPTION(NOTIFICATIONS); @@ -256,7 +234,7 @@ void BDSPAutoStory::program(SingleSwitchProgramEnvironment& env, ProControllerCo // The reads that use text (Location.h) take the game's language from here. set_story_language(RUN.language()); - StoryEnv story{env, context, STARTER, PLAYER_NAME, IntroPicks{SWITCH_LANGUAGE, RUN.language(), PLAYER_LOOK.index()}}; + StoryEnv story{env, context, STARTER, PLAYER_NAME, IntroPicks{RUN.language(), PLAYER_LOOK.index()}}; StoryExecutor executor(story); executor.set_checks(RUN.checks()); executor.set_language(RUN.language()); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h index 753bdde767..71a1a45f7d 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h @@ -50,7 +50,6 @@ class BDSPAutoStory : public SingleSwitchProgramInstance{ BooleanCheckBoxOption NEW_GAME_INTRO; BooleanCheckBoxOption JUST_LOADED; StringOption PLAYER_NAME; - EnumDropdownOption SWITCH_LANGUAGE; PlayerModelOption PLAYER_LOOK; EnumDropdownOption STARTER; diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 727a6d1b23..27c7a3f56d 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1708,6 +1708,8 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.cpp From f84574a543022ead2e578b6cfd3f14677559566e Mon Sep 17 00:00:00 2001 From: dayfine Date: Mon, 9 Nov 2026 11:15:32 -0800 Subject: [PATCH 03/19] BDSP AutoStory: language select test reads each screenshot it finds A missing negative screenshot no longer skips the whole test, and a black screen must not read as the language select. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index 5283dcb2c4..4800c7fac1 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -347,8 +347,13 @@ UnitTestResult test_intro_pick_steps(Logger&){ // The language select from lab-B 2026-11-09 (cursor on English), and screens // that are not it. Screenshots are in the unit test resources; a missing one -// is skipped. +// is passed over, and the test is skipped when all are. UnitTestResult test_language_select_screenshots(Logger& logger){ + ImageRGB32 black(1920, 1080); + black.fill(0xff000000); + if (language_select_cursor(black)){ + return "A black screen read as the language select."; + } struct Case{ const char* file; std::optional cursor; @@ -358,17 +363,23 @@ UnitTestResult test_language_select_screenshots(Logger& logger){ {.file = "PokemonBDSP/AutoStory/OreburghCenterCounter-20261108-094735.png", .cursor = std::nullopt}, {.file = "PokemonBDSP/AutoStory/OreburghCenterDoorstep-20261108.png", .cursor = std::nullopt}, }; + size_t read = 0; for (const Case& c : cases){ const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; if (!Filesystem::is_regular_file(path)){ - return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + logger.log("Screenshot not found: " + path); + continue; } + read++; const std::optional cursor = language_select_cursor(ImageRGB32(path)); logger.log(std::string(c.file) + ": cursor " + (cursor ? std::to_string(*cursor) : "none")); if (cursor != c.cursor){ return std::string(c.file) + ": wrong language select read."; } } + if (read == 0){ + return UnitTestResult(UnitTestResult::SKIPPED, "No language select screenshots found."); + } return true; } From 69a7c37794cbb642aeda7aa7cbd285f3184a966e Mon Sep 17 00:00:00 2001 From: dayfine Date: Mon, 9 Nov 2026 11:27:05 -0800 Subject: [PATCH 04/19] BDSP AutoStory: read the look select with a detector The Right/Left probe never saw the look select's cursor move (console B, 2026-11-09: 20 probes, then the name keyboard). The look select is a 2 x 4 grid of portraits, the cursor's background blue and the others grey, like the language select's buttons (lab-B frame). The language detector becomes IntroMenuDetector, one card-grid reader with a layout per menu: language buttons, and two background strips beside each portrait's face. The intro classifies the look select as its own screen and picks on it as on the language select: read the cursor, move to the chosen look, read again (3 tries), then A. Model N is portrait N - 1, row by row, matching PlayerModelOption's icons: 1 Lucas light, 2 Lucas dark, 3 Dawn dark-haired, 4 Dawn dark; 5 Lucas blond, 6 Lucas brown, 7 Dawn blond, 8 Dawn brown-haired. The option text says so. The D-pad probe is gone from MenuCursor: nothing uses it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../MenuCursor/NintendoSwitch_MenuCursor.cpp | 34 ----- .../MenuCursor/NintendoSwitch_MenuCursor.h | 31 +--- .../NintendoSwitch_MenuCursor_Tests.cpp | 29 ---- .../NintendoSwitch_MenuCursor_Tests.h | 2 +- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 82 +++++++---- .../Actions/PokemonBDSP_AutoStory_NewGame.cpp | 137 +++++++----------- .../Actions/PokemonBDSP_AutoStory_NewGame.h | 28 ++-- ...okemonBDSP_AutoStory_IntroMenuDetector.cpp | 118 +++++++++++++++ .../PokemonBDSP_AutoStory_IntroMenuDetector.h | 48 ++++++ ...nBDSP_AutoStory_LanguageSelectDetector.cpp | 79 ---------- ...monBDSP_AutoStory_LanguageSelectDetector.h | 36 ----- .../AutoStory/PokemonBDSP_AutoStory.cpp | 3 +- SerialPrograms/cmake/SourceFiles.cmake | 4 +- 13 files changed, 293 insertions(+), 338 deletions(-) create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h delete mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp delete mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp index 17dc468105..d90e5fc5d1 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.cpp @@ -4,8 +4,6 @@ * */ -#include "CommonFramework/ImageTools/ImageDiff.h" -#include "CommonFramework/VideoPipeline/VideoFeed.h" #include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" #include "NintendoSwitch_MenuCursor.h" @@ -28,14 +26,6 @@ void add_sideways(std::vector& moves, size_t from_column, size_t t } } -// 0 when either frame is missing: a probe without frames sees nothing move. -double screen_change(const VideoSnapshot& before, const VideoSnapshot& after){ - if (!before || !after){ - return 0; - } - return ImageMatch::pixel_RMSD(*before.frame, *after.frame); -} - } // namespace @@ -61,30 +51,6 @@ std::optional> menu_cursor_moves(const MenuGrid& grid, } -bool cursor_probe_moved(const CursorProbe& probe){ - return probe.press_change >= CURSOR_PRESS_CHANGE && - probe.press_change >= CURSOR_NOISE_RATIO * probe.still_change && - 2 * probe.back_change <= probe.press_change; -} - -CursorProbe probe_cursor(VideoStream& stream, ProControllerContext& context, DpadPosition there, DpadPosition back){ - context.wait_for_all_requests(); - const VideoSnapshot first = stream.video().snapshot(); - context.wait_for(CURSOR_PROBE_WAIT); - const VideoSnapshot before = stream.video().snapshot(); - pbf_press_dpad(context, there, 80ms, CURSOR_PROBE_WAIT); - context.wait_for_all_requests(); - const VideoSnapshot pressed = stream.video().snapshot(); - pbf_press_dpad(context, back, 80ms, CURSOR_PROBE_WAIT); - context.wait_for_all_requests(); - const VideoSnapshot returned = stream.video().snapshot(); - return CursorProbe{ - screen_change(first, before), - screen_change(before, pressed), - screen_change(before, returned), - }; -} - void press_cursor_moves(ProControllerContext& context, const std::vector& moves){ for (DpadPosition move : moves){ pbf_press_dpad(context, move, 80ms, 320ms); diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h index fd3ad96de6..077dfd0175 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor.h @@ -4,8 +4,7 @@ * * Moves the cursor of a fixed list or grid (a game's language select, the * player's look) to an item by its index. The menu's shape is data; the - * presses follow from it. A D-pad probe tells whether a screen has a - * cursor at all, for menus no detector knows. + * presses follow from it; a detector for the menu reads where the cursor is. * */ @@ -15,7 +14,6 @@ #include #include #include -#include "CommonFramework/Tools/VideoStream.h" #include "NintendoSwitch/Controllers/NintendoSwitch_ControllerButtons.h" #include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" @@ -36,33 +34,6 @@ struct MenuGrid{ // the grid. std::optional> menu_cursor_moves(const MenuGrid& grid, size_t from, size_t to); - -// How much the screen changed (pixel RMSD) around a D-pad probe, each against -// the frame just before the press: over a wait with no input, after the press, -// and after the opposite press. -struct CursorProbe{ - double still_change; - double press_change; - double back_change; -}; -// The least change a press that moved a cursor makes. -inline constexpr double CURSOR_PRESS_CHANGE = 6; -// How many times what the screen changes on its own (an animated preview, a -// blinking arrow) a press that moved a cursor changes it. -inline constexpr double CURSOR_NOISE_RATIO = 3; -// Whether `probe` saw a cursor move and come back: the press changed the -// screen by CURSOR_PRESS_CHANGE and CURSOR_NOISE_RATIO times the still -// change, and the opposite press put back at least half of it. -bool cursor_probe_moved(const CursorProbe& probe); - -// The wait before a probe's first press, and after each press. -inline constexpr Milliseconds CURSOR_PROBE_WAIT = Milliseconds(400); -// Waits with no input, presses `there`, waits, presses `back`, waits, and -// measures the screen at each step. On a screen with no cursor `there` and -// `back` do nothing; on one with a cursor they leave it where it was, unless -// `there` meets an edge that does not wrap. -CursorProbe probe_cursor(VideoStream& stream, ProControllerContext& context, DpadPosition there, DpadPosition back); - // Presses `moves` in order, one D-pad tap each. void press_cursor_moves(ProControllerContext& context, const std::vector& moves); diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp index 77bc1152e2..71ada25348 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp @@ -106,31 +106,6 @@ UnitTestResult test_cursor_moves_stay_inside(){ return true; } -UnitTestResult test_cursor_probe(){ - struct Case{ - const char* what; - CursorProbe probe; - bool moved; - }; - const Case cases[] = { - {"a still list: the highlight moves and comes back", {0.5, 20, 0.5}, true}, - {"a small Yes/No box", {0.3, 8, 0.4}, true}, - {"an animated preview, swapped by the press", {8, 40, 10}, true}, - {"dialogue with a blinking arrow", {2.5, 3, 2.5}, false}, - {"an animation that changes as much on its own", {10, 20, 10}, false}, - {"a press that moved something it did not put back", {0.5, 20, 15}, false}, - {"a fade", {20, 25, 45}, false}, - {"no frames", {0, 0, 0}, false}, - }; - for (const Case& c : cases){ - if (cursor_probe_moved(c.probe) != c.moved){ - return std::string(c.what) + (c.moved ? ": the cursor move was missed." : ": no cursor moved."); - } - } - return true; -} - - struct MenuCursorInput{ std::string name; UnitTestResult (*body)(); @@ -167,10 +142,6 @@ const MenuCursorCase MENU_CURSOR_CASES[] = { .input = {"presses stay inside the grid", test_cursor_moves_stay_inside}, .expectation = {}, }, - { - .input = {"probe judgement", test_cursor_probe}, - .expectation = {}, - }, }; diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h index ce92acd5dc..10b6131cb3 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h @@ -13,7 +13,7 @@ namespace PokemonAutomation{ namespace NintendoSwitch{ -// The cursor presses for list and grid items, and the probe's judgement. No console. +// The cursor presses for list and grid items. No console. void add_tests_MenuCursor(UnitTestDatabase& database); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index 4800c7fac1..6ee60be3a5 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -23,7 +23,8 @@ #include "NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.h" #include "PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_StoryMaps.h" #include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" -#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h" +#include "PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h" #include "PokemonBDSP_AutoStory_Center.h" #include "PokemonBDSP_AutoStory_NewGame.h" #include "PokemonBDSP_AutoStory_Actions_Tests.h" @@ -252,6 +253,17 @@ UnitTestResult test_intro_gate(Logger&){ } +// The menus' data, grids, detectors and the look option agree on their sizes. +UnitTestResult check_intro_menu_sizes(){ + if (bdsp_language_menu().size() != BDSP_LANGUAGE_GRID.count || BDSP_LANGUAGE_GRID.count != LANGUAGE_SELECT_BUTTONS){ + return "The language select's data, grid and detector must agree on ten buttons."; + } + if (BDSP_LOOK_GRID.count != LOOK_SELECT_PORTRAITS || BDSP_LOOK_GRID.count != BDSP_PLAYER_MODEL_COUNT){ + return "The look grid, its detector and PlayerModelOption must agree on eight looks."; + } + return true; +} + // The presses for the language select (two columns of five, read row by // row, from wherever its cursor is) and the appearance grid (two rows of // four, from the first look). @@ -282,28 +294,28 @@ UnitTestResult test_intro_pick_moves(Logger&){ {"a cursor outside the menu", language_pick_moves(LANGUAGE_SELECT_BUTTONS, french), std::nullopt}, {"Model 1, where the cursor starts", - look_pick_moves(IntroPicks{Language::English, 0}), Moves(std::vector{})}, + look_pick_moves(0, IntroPicks{Language::English, 0}), Moves(std::vector{})}, {"Model 3, Dawn's first look (lab run 3)", - look_pick_moves(french), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT})}, + look_pick_moves(0, french), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT})}, + {"Model 3 from Model 8", + look_pick_moves(7, french), Moves(std::vector{DPAD_UP, DPAD_LEFT})}, {"Model 8, the last", - look_pick_moves(IntroPicks{Language::English, 7}), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT, DPAD_RIGHT, DPAD_DOWN})}, + look_pick_moves(0, IntroPicks{Language::English, 7}), Moves(std::vector{DPAD_RIGHT, DPAD_RIGHT, DPAD_RIGHT, DPAD_DOWN})}, {"no Model 9", - look_pick_moves(IntroPicks{Language::English, 8}), std::nullopt}, + look_pick_moves(0, IntroPicks{Language::English, 8}), std::nullopt}, + {"a cursor outside the look grid", + look_pick_moves(LOOK_SELECT_PORTRAITS, french), std::nullopt}, }; for (const Case& c : cases){ if (c.actual != c.expected){ return std::string("Wrong presses for ") + c.what + "."; } } - if (bdsp_language_menu().size() != BDSP_LANGUAGE_GRID.count || BDSP_LANGUAGE_GRID.count != LANGUAGE_SELECT_BUTTONS){ - return "The language select's data, grid and detector must agree on ten buttons."; - } - return true; + return check_intro_menu_sizes(); } -// The language select takes the language pick, then A if it still shows. -// Until the look is picked, every screen after the language that would take -// A gets a probe first. The player's keyboard with a pick still to make fails. +// Each select takes its pick, then A if it still shows. Other screens take +// A. The player's keyboard with a pick still to make fails. UnitTestResult test_intro_pick_steps(Logger&){ struct Case{ const char* what; @@ -317,8 +329,10 @@ UnitTestResult test_intro_pick_steps(Logger&){ {"the language select, picked", "language-select", false, true, IntroStep::ADVANCE}, {"a logo before the language", "other", true, true, IntroStep::ADVANCE}, {"a dialogue before the language", "dialog", true, true, IntroStep::ADVANCE}, - {"Rowan, look to pick", "dialog", false, true, IntroStep::PROBE_FOR_LOOK}, - {"the preview, look to pick", "other", false, true, IntroStep::PROBE_FOR_LOOK}, + {"the look select", "look-select", false, true, IntroStep::PICK_LOOK}, + {"the look select, picked", "look-select", false, false, IntroStep::ADVANCE}, + {"Rowan, look to pick", "dialog", false, true, IntroStep::ADVANCE}, + {"a cutscene, look to pick", "other", false, true, IntroStep::ADVANCE}, {"both picked", "dialog", false, false, IntroStep::ADVANCE}, {"black, look to pick", "black", false, true, IntroStep::WAIT}, {"the user select", "user-select", true, true, IntroStep::SELECT_USER}, @@ -345,23 +359,28 @@ UnitTestResult test_intro_pick_steps(Logger&){ return true; } -// The language select from lab-B 2026-11-09 (cursor on English), and screens -// that are not it. Screenshots are in the unit test resources; a missing one -// is passed over, and the test is skipped when all are. -UnitTestResult test_language_select_screenshots(Logger& logger){ +// The intro's two selects from lab-B 2026-11-09 (each cursor on its first +// item), the screens one A before and after the look select, and screens +// that are neither. Screenshots are in the unit test resources; a missing +// one is passed over, and the test is skipped when all are. +UnitTestResult test_intro_menu_screenshots(Logger& logger){ ImageRGB32 black(1920, 1080); black.fill(0xff000000); - if (language_select_cursor(black)){ - return "A black screen read as the language select."; + if (language_select_cursor(black) || look_select_cursor(black)){ + return "A black screen read as an intro menu."; } struct Case{ const char* file; - std::optional cursor; + std::optional language; + std::optional look; }; const Case cases[] = { - {.file = "PokemonBDSP/AutoStory/LanguageSelect-20261109.png", .cursor = 0}, - {.file = "PokemonBDSP/AutoStory/OreburghCenterCounter-20261108-094735.png", .cursor = std::nullopt}, - {.file = "PokemonBDSP/AutoStory/OreburghCenterDoorstep-20261108.png", .cursor = std::nullopt}, + {.file = "PokemonBDSP/AutoStory/LanguageSelect-20261109.png", .language = 0, .look = std::nullopt}, + {.file = "PokemonBDSP/AutoStory/LookSelect-20261109.png", .language = std::nullopt, .look = 0}, + {.file = "PokemonBDSP/AutoStory/LookSelectBefore-20261109.png", .language = std::nullopt, .look = std::nullopt}, + {.file = "PokemonBDSP/AutoStory/LookSelectAfterA-20261109.png", .language = std::nullopt, .look = std::nullopt}, + {.file = "PokemonBDSP/AutoStory/OreburghCenterCounter-20261108-094735.png", .language = std::nullopt, .look = std::nullopt}, + {.file = "PokemonBDSP/AutoStory/OreburghCenterDoorstep-20261108.png", .language = std::nullopt, .look = std::nullopt}, }; size_t read = 0; for (const Case& c : cases){ @@ -371,14 +390,17 @@ UnitTestResult test_language_select_screenshots(Logger& logger){ continue; } read++; - const std::optional cursor = language_select_cursor(ImageRGB32(path)); - logger.log(std::string(c.file) + ": cursor " + (cursor ? std::to_string(*cursor) : "none")); - if (cursor != c.cursor){ - return std::string(c.file) + ": wrong language select read."; + const ImageRGB32 screen(path); + const std::optional language = language_select_cursor(screen); + const std::optional look = look_select_cursor(screen); + logger.log(std::string(c.file) + ": language " + (language ? std::to_string(*language) : "none") + + ", look " + (look ? std::to_string(*look) : "none")); + if (language != c.language || look != c.look){ + return std::string(c.file) + ": wrong intro menu read."; } } if (read == 0){ - return UnitTestResult(UnitTestResult::SKIPPED, "No language select screenshots found."); + return UnitTestResult(UnitTestResult::SKIPPED, "No intro menu screenshots found."); } return true; } @@ -629,7 +651,7 @@ const AutoStoryActionsCase AUTO_STORY_ACTIONS_CASES[] = { .expectation = {}, }, { - .input = {"language select screenshots", test_language_select_screenshots}, + .input = {"intro menu screenshots", test_intro_menu_screenshots}, .expectation = {}, }, { diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp index 540ccb9153..43aac240c0 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp @@ -22,7 +22,7 @@ #include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" #include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" #include "PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h" -#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h" #include "PokemonBDSP_AutoStory_NewGame.h" namespace PokemonAutomation{ @@ -63,8 +63,8 @@ std::optional> language_pick_moves(size_t cursor, cons } return menu_cursor_moves(BDSP_LANGUAGE_GRID, cursor, *to); } -std::optional> look_pick_moves(const IntroPicks& picks){ - return menu_cursor_moves(BDSP_LOOK_GRID, 0, picks.look); +std::optional> look_pick_moves(size_t cursor, const IntroPicks& picks){ + return menu_cursor_moves(BDSP_LOOK_GRID, cursor, picks.look); } IntroProgress intro_start(const IntroPicks& picks){ @@ -90,6 +90,9 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& if (language_select_cursor(screen)){ return "language-select"; } + if (look_select_cursor(screen)){ + return "look-select"; + } switch (NameListDetector().selected_row(screen)){ case -1: break; @@ -113,23 +116,6 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& return "other"; } -namespace{ - -// A on a screen that may take A, after a D-pad probe while the look waits -// for its grid. The language select comes first, then the look, then the -// player's name. -IntroStep advance_or_probe(const IntroProgress& progress){ - if (progress.player_named){ - return IntroStep::ADVANCE; - } - if (!progress.language_pending && progress.look_pending){ - return IntroStep::PROBE_FOR_LOOK; - } - return IntroStep::ADVANCE; -} - -} // namespace - IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress){ if (screen == "user-select"){ return IntroStep::SELECT_USER; @@ -147,6 +133,9 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog // Once picked, A again only if the select still shows. return progress.language_pending ? IntroStep::PICK_LANGUAGE : IntroStep::ADVANCE; } + if (screen == "look-select"){ + return progress.look_pending ? IntroStep::PICK_LOOK : IntroStep::ADVANCE; + } if (screen == "rival-list-tell"){ return IntroStep::MOVE_TO_PRESET; } @@ -160,7 +149,7 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog return IntroStep::WAIT; } if (screen == "dialog"){ - return progress.rival_named ? IntroStep::ADVANCE_WITH_B : advance_or_probe(progress); + return progress.rival_named ? IntroStep::ADVANCE_WITH_B : IntroStep::ADVANCE; } // The bedroom comes only after the rival is named. Before that, every unknown // screen (logos, the title, language, the appearance preview, cutscenes) @@ -172,10 +161,10 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog if (!progress.game_confirmed && progress.unconfirmed_for >= INTRO_TIME_BEFORE_BDSP){ return IntroStep::FAIL_NOT_BDSP; } - return advance_or_probe(progress); + return IntroStep::ADVANCE; } bool intro_screen_confirms_bdsp(const std::string& screen){ - return screen == "dialog" || screen == "language-select" || screen == "rival-list-tell" || screen == "rival-list-preset" || screen == "menu"; + return screen == "dialog" || screen == "language-select" || screen == "look-select" || screen == "rival-list-tell" || screen == "rival-list-preset" || screen == "menu"; } void note_intro_screen(IntroProgress& progress, const std::string& screen, std::chrono::milliseconds since_start){ @@ -355,79 +344,60 @@ void advance(IntroRun& run){ pbf_press_button(run.context, BUTTON_A, 60ms, 400ms); } -// Probes the screen with `there` and `back`; whether a cursor moved. -bool probe_moves(IntroRun& run, DpadPosition there, DpadPosition back){ - const CursorProbe probe = probe_cursor(run.env.console, run.context, there, back); - const bool moved = cursor_probe_moved(probe); - run.trace.action("Cursor probe " + dpad_to_string(there) + ": still " + std::to_string(probe.still_change) + - ", press " + std::to_string(probe.press_change) + ", back " + std::to_string(probe.back_change) + - (moved ? ": a cursor moved." : ": no cursor.")); - return moved; -} - -// Moves the cursor with `moves`, saves a screenshot of it on the pick, and confirms with A. -void pick_on_menu(IntroRun& run, const std::vector& moves, const std::string& what){ - run.trace.action("Picking " + what + ": " + std::to_string(moves.size()) + " D-pad presses, then A."); - press_cursor_moves(run.context, moves); - run.context.wait_for_all_requests(); - run.trace.screen("pick-" + what); - press(run, BUTTON_A); - pbf_wait(run.context, 500ms); -} +// One of the intro's two menus: its shape and its cursor reader. +struct IntroMenu{ + const char* what; + MenuGrid grid; + std::optional (*read)(const ImageViewRGB32& screen); +}; +const IntroMenu LANGUAGE_MENU{"language", BDSP_LANGUAGE_GRID, language_select_cursor}; +const IntroMenu LOOK_MENU{"look", BDSP_LOOK_GRID, look_select_cursor}; -// Reads of the language select before the cursor must be on the game's language. -const size_t LANGUAGE_PICK_TRIES = 3; +// Reads of a menu before its cursor must be on the pick. +const size_t INTRO_PICK_TRIES = 3; -// The language select's cursor as read now; nullopt when it is not showing. -std::optional read_language_cursor(IntroRun& run){ +// The menu's cursor as read now; nullopt when the menu is not showing. +std::optional read_cursor(IntroRun& run, const IntroMenu& menu){ run.context.wait_for_all_requests(); const VideoSnapshot screen = run.env.console.video().snapshot(); if (!screen){ return std::nullopt; } - return language_select_cursor(*screen.frame); + return menu.read(*screen.frame); } -// Reads the cursor, moves it to the game's language and reads it there, -// up to LANGUAGE_PICK_TRIES times (a press dropped as the screen fades in), -// then A. -void pick_language(IntroRun& run){ - const std::optional target = language_item(run.picks.game_language); - for (size_t attempt = 0; attempt < LANGUAGE_PICK_TRIES; attempt++){ - const std::optional cursor = read_language_cursor(run); +// Reads the cursor, moves it to `target` and reads it there, up to +// INTRO_PICK_TRIES times (a press dropped as the screen fades in), then +// saves a screenshot and presses A. Whether it picked: false when the menu +// went away, and the next read decides. +bool pick_on_grid(IntroRun& run, const IntroMenu& menu, size_t target){ + for (size_t attempt = 0; attempt < INTRO_PICK_TRIES; attempt++){ + const std::optional cursor = read_cursor(run, menu); if (!cursor){ - return; // Gone or fading: the next read decides. + return false; } - run.trace.action("Language select: cursor on item " + std::to_string(*cursor) + - ", the game's language is item " + std::to_string(*target) + "."); - if (*cursor == *target){ - pick_on_menu(run, {}, "language"); - run.progress.language_pending = false; - return; + run.trace.action(std::string("Picking the ") + menu.what + ": cursor on item " + std::to_string(*cursor) + + ", the pick is item " + std::to_string(target) + "."); + if (*cursor == target){ + run.trace.screen(std::string("pick-") + menu.what); + press(run, BUTTON_A); + pbf_wait(run.context, 500ms); + return true; } - press_cursor_moves(run.context, *language_pick_moves(*cursor, run.picks)); + press_cursor_moves(run.context, *menu_cursor_moves(menu.grid, *cursor, target)); } - run.trace.fail("The language select's cursor did not reach the game's language in " + - std::to_string(LANGUAGE_PICK_TRIES) + " tries."); + run.trace.fail(std::string("The ") + menu.what + " menu's cursor did not reach its pick in " + + std::to_string(INTRO_PICK_TRIES) + " tries."); } -// The appearance grid's cursor starts on the first look, so Right moves it. -// The Yes/No boxes around it do not move sideways. -void probe_for_look(IntroRun& run){ - if (!probe_moves(run, DPAD_RIGHT, DPAD_LEFT)){ - advance(run); - return; - } - pick_on_menu(run, *look_pick_moves(run.picks), "look"); - run.progress.look_pending = false; -} - -// PICK_LANGUAGE or PROBE_FOR_LOOK. +// PICK_LANGUAGE or PICK_LOOK. void pick_on_intro_menu(IntroRun& run, IntroStep step){ if (step == IntroStep::PICK_LANGUAGE){ - pick_language(run); - }else{ - probe_for_look(run); + if (pick_on_grid(run, LANGUAGE_MENU, *language_item(run.picks.game_language))){ + run.progress.language_pending = false; + } + }else if (pick_on_grid(run, LOOK_MENU, run.picks.look)){ + run.progress.look_pending = false; } } @@ -451,7 +421,7 @@ IntroStep next_intro_step(IntroRun& run, const char* screen){ } run.trace.fail(std::string("The player's name keyboard showed before the ") + (run.progress.language_pending ? "language select" : "appearance grid") + - " was found: no cursor probe above saw its cursor move."); + " was read."); } // Carries out what decide_intro_step() chose for `screen`. @@ -468,7 +438,7 @@ void act(IntroRun& run, const char* screen){ case IntroStep::FAIL_PICK_MISSED: fail_intro(run, step); case IntroStep::PICK_LANGUAGE: - case IntroStep::PROBE_FOR_LOOK: + case IntroStep::PICK_LOOK: pick_on_intro_menu(run, step); return; case IntroStep::ENTER_PLAYER_NAME: @@ -547,6 +517,9 @@ std::vector intro_states(IntroRun& run){ {"language-select", [=](const ImageViewRGB32& s){ return screen_is(s, "language-select"); }, [&run]{ act(run, "language-select"); }, {}, 10}, + {"look-select", + [=](const ImageViewRGB32& s){ return screen_is(s, "look-select"); }, + [&run]{ act(run, "look-select"); }, {}, 10}, {"rival-list-tell", [=](const ImageViewRGB32& s){ return screen_is(s, "rival-list-tell"); }, [&run]{ act(run, "rival-list-tell"); }, {"rival-list-preset", "black", "other"}, 6}, @@ -585,7 +558,7 @@ void play_new_game_intro( if (!language_pick_moves(0, picks)){ throw UserSetupError(env.console, "BDSP's language select lacks the game's language: pick one it lists."); } - if (!look_pick_moves(picks)){ + if (!look_pick_moves(0, picks)){ throw UserSetupError(env.console, "The player's look must be one of BDSP's eight."); } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h index d3e8516d24..d4a7b57e2b 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h @@ -6,8 +6,7 @@ * the bedroom with the player in control, which is where the Tutorial starts. * The language and the player's look are picked from their menus by index * (NintendoSwitch_MenuCursor.h); the menus' shapes are the data below. - * The language select is read by its detector; the appearance grid, which - * has none yet, is found with a D-pad probe. + * Both menus are read by their detector (PokemonBDSP_AutoStory_IntroMenuDetector.h). * */ @@ -37,9 +36,11 @@ inline constexpr MenuGrid BDSP_LANGUAGE_GRID{2, 10}; // The language of each item of BDSP_LANGUAGE_GRID. Japanese is there twice; // its first item, the kana, is the one picked. const std::vector& bdsp_language_menu(); -// The appearance grid: two rows of four looks, the cursor on the first. -// PlayerModelOption's "Model N" is item N - 1: Lucas's two first looks, -// then Dawn's, on each row. +// The look select ("Choose your photo."): two rows of four portraits, the +// cursor on the first (lab-B 2026-11-09). PlayerModelOption's "Model N" is +// item N - 1, row by row: Lucas light, Lucas dark, Dawn dark-haired, Dawn +// dark; Lucas blond, Lucas brown, Dawn blond, Dawn brown-haired. A on it +// goes straight to the name keyboard. inline constexpr MenuGrid BDSP_LOOK_GRID{4, 8}; // What the intro picks on its menus. @@ -51,13 +52,13 @@ struct IntroPicks{ // The presses from the language select's item `cursor` to the game's // language; nullopt when the menu lacks it or `cursor` is outside it. std::optional> language_pick_moves(size_t cursor, const IntroPicks& picks); -// The presses from the first look to the chosen one; nullopt when it is -// outside the grid. -std::optional> look_pick_moves(const IntroPicks& picks); +// The presses from the look select's item `cursor` to the chosen look; +// nullopt when either is outside the grid. +std::optional> look_pick_moves(size_t cursor, const IntroPicks& picks); // Which intro screen `screen` shows, as the intro's steps tell them apart: -// "user-select", "home", "keyboard", "language-select", "rival-list-tell" (the rival's name list on +// "user-select", "home", "keyboard", "language-select", "look-select", "rival-list-tell" (the rival's name list on // "I'll tell you..."), "rival-list-preset" (on a preset name), "menu", "dialog" or // "black" (the boot, fades) or "other". The Switch's own screens need // `console`; without it they read as whatever else they look like. @@ -71,7 +72,7 @@ enum class IntroStep{ FAIL_NOT_BDSP, // no BDSP screen within INTRO_TIME_BEFORE_BDSP FAIL_PICK_MISSED, // the player's keyboard, with a pick still to make PICK_LANGUAGE, // the language select: move to the game's language, A - PROBE_FOR_LOOK, // D-pad probe: a grid takes the look pick, else A + PICK_LOOK, // the look select: move to the chosen look, A ENTER_PLAYER_NAME, // type the name on the first keyboard CANCEL_RIVAL_KEYBOARD, // X: back to the rival's name list MOVE_TO_PRESET, // Down: off "I'll tell you..." @@ -119,7 +120,7 @@ inline constexpr std::chrono::milliseconds INTRO_TIME_BEFORE_BDSP = std::chrono: // How long the boot may stay black after the restart. inline constexpr std::chrono::milliseconds INTRO_BOOT_LIMIT = std::chrono::seconds(60); // Whether `screen` (classify_intro_screen()) is one only BDSP's intro shows: -// its dialogue box, its language select, the rival's name list, its main menu. The keyboard and +// its dialogue box, its language and look selects, the rival's name list, its main menu. The keyboard and // the user select are the Switch's. bool intro_screen_confirms_bdsp(const std::string& screen); @@ -177,9 +178,8 @@ IntroProgress intro_start(const IntroPicks& picks); // INTRO_BOOT_LIMIT) without pressing anything, and fails if no BDSP screen shows within // INTRO_TIME_BEFORE_BDSP of screens. Picks the language and the look in // `picks`, names the player `player_name` and picks the rival's default name -// (Barry) from the list. The language select is read, moved on and read -// again before its A. Until the look is picked, each unknown screen after it -// gets a D-pad probe before its A. The player's keyboard showing with a pick +// (Barry) from the list. The language and look selects are read, moved on +// and read again before their A. The player's keyboard showing with a pick // still to make fails. Each step reads the // screen and waits for the expected next one; an unexpected screen fails with a // screenshot and the recent screens. diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.cpp new file mode 100644 index 0000000000..14be3abf8a --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.cpp @@ -0,0 +1,118 @@ +/* BDSP AutoStory - Intro Menu Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "PokemonBDSP_AutoStory_IntroMenuDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ + +// Cards are flat: the frames' strips have stddev sums of 2 to 18. +const double MAX_STDDEV_SUM = 25; + +enum class CardLook{ GREY, BLUE, OTHER }; + +// Grey on the frames: (225, 233, 238) language buttons, (210, 218, 223) +// portraits. Blue: (167, 206, 255) and (152, 195, 251). The background +// between the cards is a bluer (203, 232, 253), neither. +CardLook strip_look(const ImageViewRGB32& screen, const ImageFloatBox& strip){ + const ImageStats stats = image_stats(extract_box_reference(screen, strip)); + if (stats.stddev.sum() > MAX_STDDEV_SUM){ + return CardLook::OTHER; + } + const FloatPixel& c = stats.average; + const double low = std::min({c.r, c.g, c.b}); + const double high = std::max({c.r, c.g, c.b}); + if (low >= 195 && high - low <= 25){ + return CardLook::GREY; + } + if (c.b >= 230 && c.r >= 120 && c.b - c.r >= 50){ + return CardLook::BLUE; + } + return CardLook::OTHER; +} + +// A card's look: all its strips must agree. +CardLook card_look(const ImageViewRGB32& screen, const std::vector& strips){ + const CardLook first = strip_look(screen, strips[0]); + for (size_t c = 1; c < strips.size(); c++){ + if (strip_look(screen, strips[c]) != first){ + return CardLook::OTHER; + } + } + return first; +} + +// The one blue card among grey ones; nullopt otherwise. +std::optional grid_cursor(const ImageViewRGB32& screen, const std::vector>& cards){ + std::optional cursor; + for (size_t card = 0; card < cards.size(); card++){ + const CardLook look = card_look(screen, cards[card]); + if (look == CardLook::OTHER || (look == CardLook::BLUE && cursor)){ + return std::nullopt; + } + if (look == CardLook::BLUE){ + cursor = card; + } + } + return cursor; +} + +// A strip inside each button, left of its text (1080p: x 330-460 on the left +// column, 1040-1170 on the right; 60 px tall around each row's centre). +const std::vector>& language_buttons(){ + static const std::vector> cards = []{ + const double column_x[2] = {0.172, 0.542}; + const double row_center_y[5] = {0.154, 0.280, 0.404, 0.528, 0.652}; + std::vector> ret; + for (size_t item = 0; item < LANGUAGE_SELECT_BUTTONS; item++){ + ret.push_back({ImageFloatBox(column_x[item % 2], row_center_y[item / 2] - 0.028, 0.068, 0.056)}); + } + return ret; + }(); + return cards; +} + +// Two strips of background beside each face (1080p: portraits from x 168, +// 576, 984, 1392 and y 124, 456; the strips 12-44 and 320-356 px in, 80-180 +// px down). +const std::vector>& look_portraits(){ + static const std::vector> cards = []{ + const double column_x[4] = {0.0875, 0.3000, 0.5125, 0.7250}; + const double row_y[2] = {0.1148, 0.4222}; + std::vector> ret; + for (size_t item = 0; item < LOOK_SELECT_PORTRAITS; item++){ + const double x = column_x[item % 4]; + const double y = row_y[item / 4] + 0.0741; + ret.push_back({ImageFloatBox(x + 0.0063, y, 0.0167, 0.0926), ImageFloatBox(x + 0.1667, y, 0.0188, 0.0926)}); + } + return ret; + }(); + return cards; +} + +} // namespace + + +std::optional language_select_cursor(const ImageViewRGB32& screen){ + return grid_cursor(screen, language_buttons()); +} + +std::optional look_select_cursor(const ImageViewRGB32& screen){ + return grid_cursor(screen, look_portraits()); +} + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h new file mode 100644 index 0000000000..f413659405 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h @@ -0,0 +1,48 @@ +/* BDSP AutoStory - Intro Menu Detector + * + * From: https://github.com/PokemonAutomation/ + * + * The two menus of a new game's intro, each a grid of flat cards over a + * crystal background, the cursor's card blue and the others light grey + * (lab-B 2026-11-09, 1080p): + * - the language select: ten buttons in two columns of five, "Select the + * language you would like to play in." below; + * - the look select: two rows of four portraits, "Choose your photo." + * below. Top row: Lucas light, Lucas dark, Dawn dark-haired, Dawn dark; + * bottom row: Lucas blond, Lucas brown, Dawn blond, Dawn brown-haired. + * Cards are numbered row by row, as MenuGrid reads them. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_IntroMenuDetector_H +#define PokemonAutomation_PokemonBDSP_AutoStory_IntroMenuDetector_H + +#include +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// The language select's buttons: English 0, its right neighbour 1, the next +// row 2 and 3, and so on. +inline constexpr size_t LANGUAGE_SELECT_BUTTONS = 10; +// The look select's portraits: the top row 0 to 3, the bottom row 4 to 7. +inline constexpr size_t LOOK_SELECT_PORTRAITS = 8; + +// The button the cursor is on, or nullopt when `screen` is not the language +// select: every button must read as plain grey but exactly one, blue. +std::optional language_select_cursor(const ImageViewRGB32& screen); + +// The portrait the cursor is on, or nullopt when `screen` is not the look +// select: the background of every portrait must read as plain grey but +// exactly one, blue. +std::optional look_select_cursor(const ImageViewRGB32& screen); + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp deleted file mode 100644 index 66384f774c..0000000000 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp +++ /dev/null @@ -1,79 +0,0 @@ -/* BDSP AutoStory - Language Select Detector - * - * From: https://github.com/PokemonAutomation/ - * - */ - -#include -#include "CommonFramework/ImageTools/ImageBoxes.h" -#include "CommonFramework/ImageTools/ImageStats.h" -#include "PokemonBDSP_AutoStory_LanguageSelectDetector.h" - -namespace PokemonAutomation{ -namespace NintendoSwitch{ -namespace PokemonBDSP{ - - -namespace{ - -// A strip inside each button, left of its text (1080p: x 330-460 on the left -// column, 1040-1170 on the right; 60 px tall around each row's centre). -const double COLUMN_X[2] = {0.172, 0.542}; -const double ROW_CENTER_Y[5] = {0.154, 0.280, 0.404, 0.528, 0.652}; -const double STRIP_WIDTH = 0.068; -const double STRIP_HEIGHT = 0.056; - -// Buttons are flat: the frame's measured stddev sum is 2 to 6. -const double MAX_STDDEV_SUM = 15; - -enum class ButtonLook{ GREY, BLUE, OTHER }; - -ImageFloatBox button_strip(size_t item){ - return ImageFloatBox(COLUMN_X[item % 2], ROW_CENTER_Y[item / 2] - STRIP_HEIGHT / 2, STRIP_WIDTH, STRIP_HEIGHT); -} - -// Grey (225, 233, 238) and blue (167, 206, 255) on the frame. The background -// between the buttons is a bluer (203, 232, 253), not grey. -ButtonLook button_look(const ImageViewRGB32& screen, size_t item){ - const ImageStats stats = image_stats(extract_box_reference(screen, button_strip(item))); - if (stats.stddev.sum() > MAX_STDDEV_SUM){ - return ButtonLook::OTHER; - } - const FloatPixel& c = stats.average; - const double low = std::min({c.r, c.g, c.b}); - const double high = std::max({c.r, c.g, c.b}); - if (low >= 200 && high - low <= 25){ - return ButtonLook::GREY; - } - if (c.b >= 230 && c.r >= 120 && c.b - c.r >= 50){ - return ButtonLook::BLUE; - } - return ButtonLook::OTHER; -} - -} // namespace - - -std::optional language_select_cursor(const ImageViewRGB32& screen){ - std::optional cursor; - for (size_t item = 0; item < LANGUAGE_SELECT_BUTTONS; item++){ - switch (button_look(screen, item)){ - case ButtonLook::GREY: - break; - case ButtonLook::BLUE: - if (cursor){ - return std::nullopt; - } - cursor = item; - break; - case ButtonLook::OTHER: - return std::nullopt; - } - } - return cursor; -} - - -} // namespace PokemonBDSP -} // namespace NintendoSwitch -} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h deleted file mode 100644 index 29524a35c4..0000000000 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h +++ /dev/null @@ -1,36 +0,0 @@ -/* BDSP AutoStory - Language Select Detector - * - * From: https://github.com/PokemonAutomation/ - * - * BDSP's language select, shown on a new game's first boot: ten buttons in - * two columns of five over a crystal background, "Select the language you - * would like to play in." below them. The cursor's button is blue with a red - * outline, the others light grey (lab-B 2026-11-09, 1080p). - * - */ - -#ifndef PokemonAutomation_PokemonBDSP_AutoStory_LanguageSelectDetector_H -#define PokemonAutomation_PokemonBDSP_AutoStory_LanguageSelectDetector_H - -#include -#include -#include "CommonFramework/ImageTypes/ImageViewRGB32.h" - -namespace PokemonAutomation{ -namespace NintendoSwitch{ -namespace PokemonBDSP{ - - -// The buttons, numbered row by row: English 0, its right neighbour 1, the -// next row 2 and 3, and so on (BDSP_LANGUAGE_GRID). -inline constexpr size_t LANGUAGE_SELECT_BUTTONS = 10; - -// The button the cursor is on, or nullopt when `screen` is not the language -// select: every button must read as plain grey but exactly one, blue. -std::optional language_select_cursor(const ImageViewRGB32& screen); - - -} // namespace PokemonBDSP -} // namespace NintendoSwitch -} // namespace PokemonAutomation -#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp index 3684ac7733..a141956dcf 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp @@ -183,7 +183,8 @@ BDSPAutoStory::BDSPAutoStory() , PLAYER_NAME(false, "Player name:
For a new game. Letters and digits.", LockMode::LOCK_WHILE_RUNNING, "Kosmos", "Kosmos") , PLAYER_LOOK( "Player's look:
For a new game: the look picked on the appearance grid. " - "Models 1, 2, 5 and 6 are Lucas; 3, 4, 7 and 8 are Dawn." + "The portraits row by row: 1 Lucas light, 2 Lucas dark, 3 Dawn dark-haired, 4 Dawn dark; " + "5 Lucas blond, 6 Lucas brown, 7 Dawn blond, 8 Dawn brown-haired." ) , STARTER( "Starter Pokemon:", diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 27c7a3f56d..493a31e851 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1708,8 +1708,8 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h - Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.cpp - Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_LanguageSelectDetector.h + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.cpp + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.cpp From 742345be9a73f95c9098de48a2bc0863050f039f Mon Sep 17 00:00:00 2001 From: dayfine Date: Tue, 10 Nov 2026 12:40:24 -0800 Subject: [PATCH 05/19] BDSP AutoStory intro picks: code-health fixes and std::string_view migration --- .../NintendoSwitch_MenuCursor_Tests.cpp | 2 +- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 27 +++++++++--------- .../Actions/PokemonBDSP_AutoStory_NewGame.cpp | 23 +++++++++------ .../Actions/PokemonBDSP_AutoStory_NewGame.h | 28 +++++++++---------- 4 files changed, 43 insertions(+), 37 deletions(-) diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp index 71ada25348..3f41a2cee2 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Programs/MenuCursor/NintendoSwitch_MenuCursor_Tests.cpp @@ -26,7 +26,7 @@ std::string describe(const std::vector& moves){ } struct MovesCase{ - const char* what; + std::string_view what; MenuGrid grid; size_t from; size_t to; diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index bc67863078..27f6502f7c 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -35,6 +35,7 @@ namespace PokemonAutomation{ namespace NintendoSwitch{ namespace PokemonBDSP{ +using namespace detail; namespace{ @@ -207,7 +208,7 @@ std::string check_boot(IntroProgress& progress){ } UnitTestResult check_confirming_screens(){ - for (const char* screen : {"dialog", "language-select", "rival-list-tell", "rival-list-preset", "menu"}){ + for (std::string screen : {"dialog", "language-select", "rival-list-tell", "rival-list-preset", "menu"}){ if (!intro_screen_confirms_bdsp(screen)){ return std::string("Expected \"") + screen + "\" to confirm BDSP."; } @@ -273,7 +274,7 @@ UnitTestResult test_intro_pick_moves(Logger&){ using Moves = std::optional>; const IntroPicks french{Language::French, 2}; struct Case{ - const char* what; + std::string_view what; Moves actual; Moves expected; }; @@ -310,7 +311,7 @@ UnitTestResult test_intro_pick_moves(Logger&){ }; for (const Case& c : cases){ if (c.actual != c.expected){ - return std::string("Wrong presses for ") + c.what + "."; + return std::string("Wrong presses for ") + std::string(c.what) + "."; } } return check_intro_menu_sizes(); @@ -320,8 +321,8 @@ UnitTestResult test_intro_pick_moves(Logger&){ // A. The player's keyboard with a pick still to make fails. UnitTestResult test_intro_pick_steps(Logger&){ struct Case{ - const char* what; - const char* screen; + std::string_view what; + std::string_view screen; bool language_pending; bool look_pending; IntroStep expected; @@ -347,7 +348,7 @@ UnitTestResult test_intro_pick_steps(Logger&){ progress.language_pending = c.language_pending; progress.look_pending = c.look_pending; if (decide_intro_step(c.screen, progress) != c.expected){ - return std::string("Wrong step for ") + c.what + "."; + return std::string("Wrong step for ") + std::string(c.what) + "."; } } const IntroProgress first_look = intro_start(IntroPicks{Language::English, 0}); @@ -372,7 +373,7 @@ UnitTestResult test_intro_menu_screenshots(Logger& logger){ return "A black screen read as an intro menu."; } struct Case{ - const char* file; + std::string_view file; std::optional language; std::optional look; }; @@ -386,7 +387,7 @@ UnitTestResult test_intro_menu_screenshots(Logger& logger){ }; size_t read = 0; for (const Case& c : cases){ - const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + const std::string path = UNIT_TEST_RESOURCE_PATH() + std::string(c.file); if (!Filesystem::is_regular_file(path)){ logger.log("Screenshot not found: " + path); continue; @@ -426,7 +427,7 @@ UnitTestResult test_lit_doorway_screenshots(Logger& logger){ {"PokemonBDSP/AutoStory/SandgemMartDoorFarLeft-20261030-024648.png", -1.3, -0.5, 2.5, 4.5}, }; for (const Case& c : cases){ - const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + const std::string path = UNIT_TEST_RESOURCE_PATH() + std::string(c.file); if (!Filesystem::is_regular_file(path)){ return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); } @@ -497,7 +498,7 @@ UnitTestResult test_pokemon_center_inside(Logger& logger){ {.file = "PokemonBDSP/AutoStory/JubilifeCenterDoorstep-20261030-c5.png", .inside = false, .doorway = true}, }; for (const Case& c : cases){ - const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + const std::string path = UNIT_TEST_RESOURCE_PATH() + std::string(c.file); if (!Filesystem::is_regular_file(path)){ return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); } @@ -535,7 +536,7 @@ UnitTestResult test_pokemon_center_counter(Logger& logger){ {.file = "PokemonBDSP/AutoStory/OreburghCenterDoorstep-20261108.png", .inside = false}, }; for (const Case& c : cases){ - const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + const std::string path = UNIT_TEST_RESOURCE_PATH() + std::string(c.file); if (!Filesystem::is_regular_file(path)){ return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); } @@ -569,7 +570,7 @@ UnitTestResult test_school_lamp(Logger& logger){ {"PokemonBDSP/AutoStory/JubilifeSchoolOneTilePast-20261030.png", 0.5, 1.5}, }; for (const Case& c : cases){ - const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + const std::string path = UNIT_TEST_RESOURCE_PATH() + std::string(c.file); if (!Filesystem::is_regular_file(path)){ return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); } @@ -634,7 +635,7 @@ UnitTestResult test_move_menu_screenshots(Logger& logger){ }; const Language story = story_language(); for (const Case& c : cases){ - const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + const std::string path = UNIT_TEST_RESOURCE_PATH() + std::string(c.file); if (!Filesystem::is_regular_file(path)){ return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp index 43aac240c0..f388a8832e 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp @@ -30,6 +30,7 @@ namespace NintendoSwitch{ namespace PokemonBDSP{ using namespace std::chrono_literals; +using namespace detail; const std::vector& bdsp_language_menu(){ @@ -67,12 +68,14 @@ std::optional> look_pick_moves(size_t cursor, const In return menu_cursor_moves(BDSP_LOOK_GRID, cursor, picks.look); } +namespace detail{ IntroProgress intro_start(const IntroPicks& picks){ IntroProgress progress; progress.language_pending = true; progress.look_pending = picks.look != 0; return progress; } +} // namespace detail std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& screen){ @@ -116,7 +119,7 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& return "other"; } -IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress){ +IntroStep decide_intro_step(std::string_view screen, const IntroProgress& progress){ if (screen == "user-select"){ return IntroStep::SELECT_USER; } @@ -163,11 +166,12 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog } return IntroStep::ADVANCE; } -bool intro_screen_confirms_bdsp(const std::string& screen){ +bool intro_screen_confirms_bdsp(std::string_view screen){ return screen == "dialog" || screen == "language-select" || screen == "look-select" || screen == "rival-list-tell" || screen == "rival-list-preset" || screen == "menu"; } -void note_intro_screen(IntroProgress& progress, const std::string& screen, std::chrono::milliseconds since_start){ +namespace detail{ +void note_intro_screen(IntroProgress& progress, std::string_view screen, std::chrono::milliseconds since_start){ if (intro_screen_confirms_bdsp(screen)){ progress.game_confirmed = true; } @@ -215,6 +219,7 @@ void record_intro_step(IntroProgress& progress, IntroStep step){ return; } } +} // namespace detail namespace{ @@ -346,7 +351,7 @@ void advance(IntroRun& run){ // One of the intro's two menus: its shape and its cursor reader. struct IntroMenu{ - const char* what; + std::string_view what; MenuGrid grid; std::optional (*read)(const ImageViewRGB32& screen); }; @@ -376,17 +381,17 @@ bool pick_on_grid(IntroRun& run, const IntroMenu& menu, size_t target){ if (!cursor){ return false; } - run.trace.action(std::string("Picking the ") + menu.what + ": cursor on item " + std::to_string(*cursor) + + run.trace.action(std::string("Picking the ") + std::string(menu.what) + ": cursor on item " + std::to_string(*cursor) + ", the pick is item " + std::to_string(target) + "."); if (*cursor == target){ - run.trace.screen(std::string("pick-") + menu.what); + run.trace.screen(std::string("pick-") + std::string(menu.what)); press(run, BUTTON_A); pbf_wait(run.context, 500ms); return true; } press_cursor_moves(run.context, *menu_cursor_moves(menu.grid, *cursor, target)); } - run.trace.fail(std::string("The ") + menu.what + " menu's cursor did not reach its pick in " + + run.trace.fail(std::string("The ") + std::string(menu.what) + " menu's cursor did not reach its pick in " + std::to_string(INTRO_PICK_TRIES) + " tries."); } @@ -402,7 +407,7 @@ void pick_on_intro_menu(IntroRun& run, IntroStep step){ } // Notes `screen` in the run's progress, decides the step for it and counts it. -IntroStep next_intro_step(IntroRun& run, const char* screen){ +IntroStep next_intro_step(IntroRun& run, std::string_view screen){ note_intro_screen(run.progress, screen, std::chrono::duration_cast(current_time() - run.start)); const IntroStep step = decide_intro_step(screen, run.progress); @@ -425,7 +430,7 @@ IntroStep next_intro_step(IntroRun& run, const char* screen){ } // Carries out what decide_intro_step() chose for `screen`. -void act(IntroRun& run, const char* screen){ +void act(IntroRun& run, std::string_view screen){ const IntroStep step = next_intro_step(run, screen); switch (step){ case IntroStep::SELECT_USER: diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h index d4a7b57e2b..364e80d5c0 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h @@ -8,6 +8,7 @@ * (NintendoSwitch_MenuCursor.h); the menus' shapes are the data below. * Both menus are read by their detector (PokemonBDSP_AutoStory_IntroMenuDetector.h). * +// code-health: allow header_symbols -- new game intro flow, menu cursors, detectors and picks */ #ifndef PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H @@ -111,6 +112,15 @@ struct IntroProgress{ // B presses on unknown screens since the last X. size_t b_presses = 0; }; +// Whether `screen` (classify_intro_screen()) is one only BDSP's intro shows: +// its dialogue box, its language and look selects, the rival's name list, its main menu. The keyboard and +// the user select are the Switch's. +bool intro_screen_confirms_bdsp(std::string_view screen); + +// Decides the next controller action based on the current screen and intro progress. +IntroStep decide_intro_step(std::string_view screen, const IntroProgress& progress); + +namespace detail{ // How long the intro presses A on unknown screens, from the first screen // that is not black, before a BDSP screen must show: the logos, the title // and the language select. Another game launched by mistake (r1, @@ -119,16 +129,10 @@ struct IntroProgress{ inline constexpr std::chrono::milliseconds INTRO_TIME_BEFORE_BDSP = std::chrono::seconds(60); // How long the boot may stay black after the restart. inline constexpr std::chrono::milliseconds INTRO_BOOT_LIMIT = std::chrono::seconds(60); -// Whether `screen` (classify_intro_screen()) is one only BDSP's intro shows: -// its dialogue box, its language and look selects, the rival's name list, its main menu. The keyboard and -// the user select are the Switch's. -bool intro_screen_confirms_bdsp(const std::string& screen); - // Screens after the rival's name is picked, each taking A, before the name // counts as confirmed and B takes over: "Barry, is it?", the box closing // (Barry alone), the Yes/No, and a few of Rowan's lines after it. inline constexpr size_t INTRO_SCREENS_AFTER_RIVAL_PICK = 8; - // Once the rival is named, the intro only has dialogue and cutscenes left // before the bedroom, and screens the classifier misses (r2: a dialogue box // without its arrow). Each unknown screen takes B, which advances dialogue, @@ -138,13 +142,6 @@ inline constexpr size_t INTRO_B_BEFORE_X = 5; // X presses that opened no menu before the intro gives up: with // INTRO_B_BEFORE_X B presses between them, about two minutes. inline constexpr size_t INTRO_X_WITHOUT_MENU = 20; - -// Notes `screen`, read `since_start` into the intro, in `progress`: the -// game is confirmed on a BDSP screen; black screens leave the clock alone. -void note_intro_screen(IntroProgress& progress, const std::string& screen, std::chrono::milliseconds since_start); -// Decides the next controller action based on the current screen and intro progress. -IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress); - // When X opens nothing the menu detector knows, the player is still in // control if the screen held still for INTRO_STILL_TIME before X (no // dialogue, no fade, no intro sparkles), X changed the menu's area by more @@ -167,9 +164,12 @@ struct XProbe{ bool intro_in_control_without_menu(const XProbe& probe); // Counts `step` in `progress`. void record_intro_step(IntroProgress& progress, IntroStep step); - // The progress a new intro starts from: which of `picks` need a menu. IntroProgress intro_start(const IntroPicks& picks); +// Notes `screen`, read `since_start` into the intro, in `progress`: the +// game is confirmed on a BDSP screen; black screens leave the clock alone. +void note_intro_screen(IntroProgress& progress, std::string_view screen, std::chrono::milliseconds since_start); +} // namespace detail // From the title screen or any intro screen of a game with no save, with BDSP // running in front: from HOME it throws UserSetupError, since the restart would From 449381f5d5afe4b764a150fbce935a8a1eaa35ce Mon Sep 17 00:00:00 2001 From: dayfine Date: Thu, 12 Nov 2026 23:48:54 -0800 Subject: [PATCH 06/19] BDSP AutoStory: an empty or off-frame move name box reads as unread, not a crash gray_box() sorted and indexed an empty vector when the box was outside the frame or the image empty (SIGSEGV at 0x0, 2026-11-12). read_move_name now returns "" for it and the menu falls back to the PP guess. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 25 +++++++++++++++++++ .../PokemonBDSP_AutoStory_BattleMoves.cpp | 7 ++++++ 2 files changed, 32 insertions(+) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index b98e2440f4..2ba033d092 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -598,6 +598,27 @@ UnitTestResult test_school_lamp(Logger& logger){ // no name read, Piplup's Pound at 35 PP and Growl at 22 PP, and the old // "newest move that is not a status move" fallback used Growl until the // training gave up after 25 turns. +// A menu read on an empty image, or with buttons outside the frame, reads +// nothing instead of crashing (SIGSEGV in gray_box, 2026-11-12). +UnitTestResult test_move_menu_unreadable(Logger&){ + BattleMoveMenuState moves; + moves.move_count = 2; + moves.buttons[0] = ImageFloatBox{0.5, 0.5, 0.4, 0.1}; + moves.buttons[1] = ImageFloatBox{1.5, 1.5, 0.4, 0.1}; + const ImageRGB32 frame(64, 36); + const ImageViewRGB32 inputs[] = {ImageViewRGB32(), frame}; + for (const ImageViewRGB32& screen : inputs){ + const std::vector slots = read_move_menu(screen, moves); + if (slots.size() != 2){ + return "read_move_menu returned " + std::to_string(slots.size()) + " slots, expected 2."; + } + if (!slots[1].slug.empty() || slots[1].pp != -1){ + return "a button outside the frame read a move."; + } + } + return true; +} + UnitTestResult test_choose_move(Logger&){ struct Case{ std::string_view name; std::vector slots; size_t best; }; const Case cases[] = { @@ -759,6 +780,10 @@ const AutoStoryActionsCase AUTO_STORY_ACTIONS_CASES[] = { .input = {"battle moves: the pick from names and PP", test_choose_move}, .expectation = {}, }, + { + .input = {"battle moves: unreadable menu reads nothing", test_move_menu_unreadable}, + .expectation = {}, + }, { .input = {"battle moves: move menu screenshots in each language", test_move_menu_screenshots}, .expectation = {}, diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.cpp index cae25a7d4d..6557c928a8 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.cpp @@ -221,6 +221,10 @@ GrayBox gray_box(const ImageViewRGB32& box){ ret.gray[y * box.width() + x] = (int)(((pixel >> 16) & 0xff) + ((pixel >> 8) & 0xff) + (pixel & 0xff)) / 3; } } + if (ret.gray.empty()){ + // Nothing to read: a box outside the frame, or an empty image. + return ret; + } std::vector sorted = ret.gray; std::nth_element(sorted.begin(), sorted.begin() + sorted.size() / 2, sorted.end()); const int background = sorted[sorted.size() / 2]; @@ -259,6 +263,9 @@ std::string read_move_name(const ImageViewRGB32& screen, const ImageFloatBox& bu const ImageFloatBox name_box{button.x + button.width * 0.17, button.y + button.height * 0.08, button.width * 0.53, button.height * 0.80}; const GrayBox box = gray_box(extract_box_reference(screen, name_box)); + if (box.gray.empty()){ + return ""; + } const auto [band_top, band_bottom] = tallest_text_band(box); // Black text on white, with a margin, for the OCR. const size_t pad = 6; From c8ee764743b9be6d710c159090640149c8235793 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 15 Nov 2026 00:04:49 -0800 Subject: [PATCH 07/19] BDSP AutoStory intro: resume on an intro screen, name limit check, capture mode - On an intro screen (user select, language or look select, keyboard, dialogue, rival list) the intro takes up from there without closing the game; elsewhere it restarts as before. intro_resume() sets which picks are still ahead. - Before typing the player's name, the keyboard's counter ("0/N") is read with OCR; a longer name fails at the name screen with nothing typed. - CAPTURE_INTRO_SCREENS: per language-select item, restart, pick it and the look, save _keyboard.png (and en_language-list.png, en_look-picker.png on the first round) to the Screenshots folder. - PLAYER_LOOK must match the Starter RNG character model when both run. --- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 124 ++++++++++ .../PokemonBDSP_AutoStory_IntroSteps.h | 3 + .../Actions/PokemonBDSP_AutoStory_NewGame.cpp | 230 +++++++++++++++--- .../Actions/PokemonBDSP_AutoStory_NewGame.h | 35 ++- .../PokemonBDSP_AutoStory_NameLimitReader.cpp | 77 ++++++ .../PokemonBDSP_AutoStory_NameLimitReader.h | 38 +++ .../AutoStory/PokemonBDSP_AutoStory.cpp | 55 +++-- .../AutoStory/PokemonBDSP_AutoStory.h | 2 + SerialPrograms/cmake/SourceFiles.cmake | 2 + 9 files changed, 511 insertions(+), 55 deletions(-) create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_NameLimitReader.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_NameLimitReader.h diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index 2ba033d092..3ea1da8056 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Filesystem/Filesystem.h" @@ -26,6 +27,7 @@ #include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" #include "PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h" #include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_NameLimitReader.h" #include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_StoryLanguage.h" #include "PokemonBDSP_AutoStory_BattleMoves.h" #include "PokemonBDSP_AutoStory_Center.h" @@ -366,6 +368,116 @@ UnitTestResult test_intro_pick_steps(Logger&){ return true; } +// Taken up on an intro screen, the intro restarts nothing and makes only +// the picks still ahead; anywhere else it restarts the game. +UnitTestResult test_intro_resume(Logger&){ + struct Input{ + std::string_view screen; + }; + struct Expectation{ + bool restart; + bool language_pending; + bool look_pending; + bool picks_may_be_past; + bool player_named; + }; + struct Case{ + std::string_view name; + Input input; + Expectation expectation; + }; + const Case cases[] = { + {"the title or a logo", {"other"}, {true, true, true, false, false}}, + {"black", {"black"}, {true, true, true, false, false}}, + {"the bedroom's menu", {"menu"}, {true, true, true, false, false}}, + {"the user select", {"user-select"}, {false, true, true, false, false}}, + {"the language select", {"language-select"}, {false, true, true, false, false}}, + {"the look select", {"look-select"}, {false, false, true, false, false}}, + {"a dialogue", {"dialog"}, {false, false, true, true, false}}, + {"the player's keyboard", {"keyboard"}, {false, false, false, false, false}}, + {"the rival's name list", {"rival-list-tell"}, {false, false, false, false, true}}, + {"a preset rival name", {"rival-list-preset"}, {false, false, false, false, true}}, + }; + const IntroPicks picks{Language::Japanese, 6}; + for (const Case& c : cases){ + const Expectation& e = c.expectation; + if (intro_restart_needed(c.input.screen) != e.restart){ + return std::string("Wrong restart for ") + std::string(c.name) + "."; + } + if (e.restart){ + continue; + } + const IntroProgress progress = intro_resume(picks, c.input.screen); + if (progress.language_pending != e.language_pending || progress.look_pending != e.look_pending || + progress.picks_may_be_past != e.picks_may_be_past || progress.player_named != e.player_named + ){ + return std::string("Wrong progress taken up on ") + std::string(c.name) + "."; + } + } + // From a dialogue past the look select, the keyboard types the name. + if (decide_intro_step("keyboard", intro_resume(picks, "dialog")) != IntroStep::ENTER_PLAYER_NAME){ + return "Taken up on a dialogue, the player's keyboard must take the name."; + } + return true; +} + +// The keyboard's counter gives the name's limit; a longer name stops the +// intro, an unread limit does not. +UnitTestResult test_name_limit(Logger&){ + struct ParseCase{ + std::string_view name; + std::string_view input; + std::optional expectation; + }; + const ParseCase parse_cases[] = { + {"an empty field", "0/6", 6}, + {"OCR's newline", "0/5\n", 5}, + {"spaces", " 0 / 12 ", 12}, + {"a typed name", "6/12", 12}, + {"nothing", "", std::nullopt}, + {"no limit", "0/", std::nullopt}, + {"a zero limit", "0/0", std::nullopt}, + {"text", "Name", std::nullopt}, + {"text after it", "0/6 x", std::nullopt}, + }; + for (const ParseCase& c : parse_cases){ + if (parse_keyboard_counter(c.input) != c.expectation){ + return std::string("Wrong limit for ") + std::string(c.name) + "."; + } + } + struct FitCase{ + std::string_view name; + std::string_view input_name; + std::optional input_limit; + bool expectation_fails; + }; + const FitCase fit_cases[] = { + {"Kosmos in 12", "Kosmos", 12, false}, + {"Kosmos in 6", "Kosmos", 6, false}, + {"Kosmos in 5", "Kosmos", 5, true}, + {"limit not read", "Kosmos", std::nullopt, false}, + {"two-byte letters in 5", "\xc3\xa9\xc3\xa9\xc3\xa9", 5, false}, + }; + for (const FitCase& c : fit_cases){ + if (name_limit_problem(c.input_name, c.input_limit).has_value() != c.expectation_fails){ + return std::string("Wrong fit for ") + std::string(c.name) + "."; + } + } + return true; +} + +// Capture mode names a file per language select item; Japanese twice. +UnitTestResult test_language_tags(Logger&){ + const std::vector& tags = bdsp_language_tags(); + if (tags.size() != LANGUAGE_SELECT_BUTTONS || tags.size() != bdsp_language_menu().size()){ + return "One tag per language select button."; + } + if (tags[1] != "ja-kana" || bdsp_language_menu()[1] != Language::Japanese){ + return "Item 1 is the kana Japanese."; + } + return true; +} + // The intro's two selects from lab-B 2026-11-09 (each cursor on its first // item), the screens one A before and after the look select, and screens // that are neither. Screenshots are in the unit test resources; a missing @@ -752,6 +864,18 @@ const AutoStoryActionsCase AUTO_STORY_ACTIONS_CASES[] = { .input = {"new-game intro: steps until the picks are made", test_intro_pick_steps}, .expectation = {}, }, + { + .input = {"new-game intro: taken up on the screen it shows", test_intro_resume}, + .expectation = {}, + }, + { + .input = {"new-game intro: the name keyboard's limit", test_name_limit}, + .expectation = {}, + }, + { + .input = {"new-game intro: capture file per language", test_language_tags}, + .expectation = {}, + }, { .input = {"intro menu screenshots", test_intro_menu_screenshots}, .expectation = {}, diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_IntroSteps.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_IntroSteps.h index d91340e93c..99202b34c1 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_IntroSteps.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_IntroSteps.h @@ -46,6 +46,9 @@ struct IntroProgress{ // The first look needs no menu: A takes it. bool language_pending = false; bool look_pending = false; + // The intro was taken up past the screen of a pending pick, which may + // have been made before: the player's keyboard does not fail on it. + bool picks_may_be_past = false; bool player_named = false; // A preset name was picked from the rival's list, and how many screens // have shown since without the list coming back. "Barry, is it?" and diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp index 295446edbf..e1f0f021ea 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp @@ -5,9 +5,11 @@ */ #include +#include #include "Common/Cpp/Exceptions.h" #include "CommonFramework/ImageTools/ImageBoxes.h" #include "CommonFramework/ImageTools/ImageDiff.h" +#include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "CommonFramework/VideoPipeline/VideoFeed.h" #include "CommonTools/Async/InferenceRoutines.h" @@ -24,6 +26,7 @@ #include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" #include "PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h" #include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_NameLimitReader.h" #include "PokemonBDSP_AutoStory_IntroProbe.h" #include "PokemonBDSP_AutoStory_IntroSteps.h" #include "PokemonBDSP_AutoStory_NewGame.h" @@ -47,6 +50,19 @@ const std::vector& bdsp_language_menu(){ return menu; } +namespace detail{ +const std::vector& bdsp_language_tags(){ + static const std::vector tags{ + "en", "ja-kana", + "es", "ja", + "fr", "ko", + "de", "zh-Hans", + "it", "zh-Hant", + }; + return tags; +} +} // namespace detail + namespace{ std::optional language_item(Language language){ @@ -78,6 +94,53 @@ IntroProgress intro_start(const IntroPicks& picks){ progress.look_pending = picks.look != 0; return progress; } +bool intro_restart_needed(std::string_view screen){ + return !(screen == "user-select" || screen == "language-select" || screen == "look-select" || + screen == "keyboard" || screen == "dialog" || screen == "rival-list-tell" || screen == "rival-list-preset"); +} +IntroProgress intro_resume(const IntroPicks& picks, std::string_view screen){ + IntroProgress progress = intro_start(picks); + if (screen == "user-select" || screen == "language-select"){ + return progress; + } + // Every other intro screen comes after the language. + progress.language_pending = false; + if (screen == "look-select"){ + // It shows: the cursor may be anywhere, so read it and pick. + progress.look_pending = true; + return progress; + } + if (screen == "dialog"){ + // Rowan before the look select, or the names after it. + progress.picks_may_be_past = true; + return progress; + } + progress.look_pending = false; + if (screen == "rival-list-tell" || screen == "rival-list-preset"){ + progress.player_named = true; + } + return progress; +} +std::optional name_limit_problem(std::string_view name, std::optional limit){ + if (!limit){ + return std::nullopt; + } + // Characters, not UTF-8 bytes: count the bytes that start one. + size_t characters = 0; + for (char c : name){ + if (((unsigned char)c & 0xc0) != 0x80){ + characters++; + } + } + if (characters <= *limit){ + return std::nullopt; + } + return std::format( + "The name keyboard takes at most {} characters in this language, and the player name \"{}\" has {}. " + "Stopped at the name screen with nothing typed: pick a name of at most {} characters and start again.", + *limit, name, characters, *limit + ); +} } // namespace detail @@ -130,7 +193,7 @@ IntroStep decide_intro_step(std::string_view screen, const IntroProgress& progre return IntroStep::FAIL_NOT_IN_GAME; } if (screen == "keyboard"){ - if (!progress.player_named && (progress.language_pending || progress.look_pending)){ + if (!progress.player_named && (progress.language_pending || progress.look_pending) && !progress.picks_may_be_past){ return IntroStep::FAIL_PICK_MISSED; } return progress.player_named ? IntroStep::CANCEL_RIVAL_KEYBOARD : IntroStep::ENTER_PLAYER_NAME; @@ -255,6 +318,15 @@ void enter_player_name(VideoStream& stream, ProControllerContext& context, StepT pbf_press_button(context, BUTTON_B, 60ms, 250ms); context.wait_for_all_requests(); } + // The keyboard drops presses past its limit: stop before typing. + VideoSnapshot screen = stream.video().snapshot(); + const std::optional limit = screen ? read_keyboard_name_limit(*screen.frame) : std::nullopt; + trace.action(limit + ? std::format("The keyboard takes up to {} characters.", *limit) + : std::string("The keyboard's character counter did not read; typing anyway.")); + if (const std::optional problem = name_limit_problem(name, limit)){ + trace.fail(*problem); + } trace.action("Typing the player's name: " + name); FastCodeEntry::type_on_system_keyboard(stream, context, name); pbf_press_button(context, BUTTON_PLUS, 80ms, 0ms); @@ -276,8 +348,37 @@ struct IntroRun{ size_t x_without_menu = 0; // X opened something and B closed it, from a still screen. bool in_control = false; + // The language select's item to pick. + size_t language_target = 0; + // Capture mode (capture_intro_screens()): the run ends on the player's + // keyboard, saved as _keyboard.png, and with + // `capture_menus` the two selects are saved before their picks. + bool capture = false; + bool capture_menus = false; + std::string capture_tag; + bool capture_done = false; }; +// Saves the frame on screen now as /_.png. +void save_capture(IntroRun& run, std::string_view what){ + run.context.wait_for_all_requests(); + const VideoSnapshot screen = run.env.console.video().snapshot(); + if (!screen){ + run.trace.fail("No video frame to capture."); + } + const std::string path = std::format("{}{}_{}.png", SCREENSHOTS_PATH(), run.capture_tag, what); + screen.frame->save(path); + run.trace.action("Captured " + path); +} + +// Capture mode: the name keyboard after the look's A. Saved after its +// slide-in, also when the keyboard detector does not know its layout. +void capture_keyboard(IntroRun& run){ + pbf_wait(run.context, 2000ms); + save_capture(run, "keyboard"); + run.capture_done = true; +} + void press(IntroRun& run, Button button){ pbf_press_button(run.context, button, 80ms, 0ms); } @@ -340,7 +441,7 @@ void probe_for_menu(IntroRun& run){ const XProbeResult result = x_from_still_screen(run); run.in_control = result == XProbeResult::IN_CONTROL; if (result != XProbeResult::NOTHING){ - return; // The next read sees the menu, or the in-control state, and finishes. + return; // The next read sees the menu, or the finished state, and finishes. } run.trace.action("X opened no menu; B again before the next X."); if (++run.x_without_menu >= INTRO_X_WITHOUT_MENU){ @@ -400,12 +501,21 @@ bool pick_on_grid(IntroRun& run, const IntroMenu& menu, size_t target){ // PICK_LANGUAGE or PICK_LOOK. void pick_on_intro_menu(IntroRun& run, IntroStep step){ - if (step == IntroStep::PICK_LANGUAGE){ - if (pick_on_grid(run, LANGUAGE_MENU, *language_item(run.picks.game_language))){ + const bool language = step == IntroStep::PICK_LANGUAGE; + if (run.capture_menus){ + save_capture(run, language ? "language-list" : "look-picker"); + } + if (language){ + if (pick_on_grid(run, LANGUAGE_MENU, run.language_target)){ run.progress.language_pending = false; } - }else if (pick_on_grid(run, LOOK_MENU, run.picks.look)){ + return; + } + if (pick_on_grid(run, LOOK_MENU, run.picks.look)){ run.progress.look_pending = false; + if (run.capture){ + capture_keyboard(run); + } } } @@ -432,6 +542,15 @@ IntroStep next_intro_step(IntroRun& run, std::string_view screen){ " was read."); } +// ENTER_PLAYER_NAME: types the name, or in capture mode shoots the keyboard. +void name_player(IntroRun& run){ + if (run.capture){ + capture_keyboard(run); + return; + } + enter_player_name(run.env.console, run.context, run.trace, run.player_name); +} + // Carries out what decide_intro_step() chose for `screen`. void act(IntroRun& run, std::string_view screen){ const IntroStep step = next_intro_step(run, screen); @@ -450,7 +569,7 @@ void act(IntroRun& run, std::string_view screen){ pick_on_intro_menu(run, step); return; case IntroStep::ENTER_PLAYER_NAME: - enter_player_name(run.env.console, run.context, run.trace, run.player_name); + name_player(run); return; case IntroStep::CANCEL_RIVAL_KEYBOARD: // Opened from "I'll tell you..."; inputs during the slide-in are dropped. @@ -506,9 +625,10 @@ std::vector intro_states(IntroRun& run){ return classify_intro_screen(&run.env.console, screen) == name; }; return { - {"in-control", - [&run](const ImageViewRGB32&){ return run.in_control; }, nullptr, - [&run]{ run.trace.action("X opened and B closed the menu from a still bedroom: the player is in control."); }, + // The player in control without the menu read, or capture mode's keyboard shot. + {"finished", + [&run](const ImageViewRGB32&){ return run.in_control || run.capture_done; }, nullptr, + [&run]{ run.trace.action(run.capture_done ? "Keyboard captured." : "X opened and B closed the menu from a still bedroom: the player is in control."); }, {}, 1, true}, {"user-select", [=](const ImageViewRGB32& s){ return screen_is(s, "user-select"); }, nullptr, @@ -518,7 +638,7 @@ std::vector intro_states(IntroRun& run){ [&run]{ act(run, "home"); }, {}, 1}, {"player-keyboard", [=, &run](const ImageViewRGB32& s){ return !run.progress.player_named && screen_is(s, "keyboard"); }, nullptr, - [&run]{ act(run, "keyboard"); }, {"dialog", "black", "other"}, 1}, + [&run]{ act(run, "keyboard"); }, {"finished", "dialog", "black", "other"}, 1}, {"rival-keyboard", [=, &run](const ImageViewRGB32& s){ return run.progress.player_named && screen_is(s, "keyboard"); }, nullptr, [&run]{ act(run, "keyboard"); }, {"rival-list-tell", "dialog", "black", "other"}, 3}, @@ -550,6 +670,33 @@ std::vector intro_states(IntroRun& run){ }; } +// The restart reopens the game HOME's cursor is on, which is the game +// running in front only while it runs: with none running (r1, 2026-11-01), +// closing it fails, HOME is pressed again, the cursor goes to the first +// tile, and another game was launched. So BDSP must be running in front. +// Returns the frame on screen, if any. +VideoSnapshot require_game_in_front(SingleSwitchProgramEnvironment& env, ProControllerContext& context){ + context.wait_for_all_requests(); + VideoSnapshot start = env.console.video().snapshot(); + if (start && HomeMenuDetector(env.console, COLOR_RED).detect(start)){ + throw UserSetupError(env.console, + "Start a new game with BDSP running in front, not from the Switch's HOME menu: " + "the intro restarts the game in front."); + } + return start; +} + +// Only closes and reopens the game: the states take it from the user select +// on. reset_game_from_home() waits for a loaded save's black screen to end, and a +// new game's first screen is a dialog on black, so it would sit there until its +// timeout. +void restart_game(SingleSwitchProgramEnvironment& env, ProControllerContext& context, StepTrace& trace){ + trace.action("Restarting the game."); + go_home(env.console, context); + from_home_close_and_reopen_game(env.console, context, true); + wait_out_boot(env.console, context, trace); +} + } // namespace @@ -570,36 +717,51 @@ void play_new_game_intro( throw UserSetupError(env.console, "The player's look must be one of BDSP's eight."); } - // The restart below reopens the game HOME's cursor is on, which is the - // game running in front only while it runs: with none running (r1, - // 2026-11-01), closing it fails, HOME is pressed again, the cursor goes - // to the first tile, and another game was launched. So BDSP must be - // running in front when the intro starts. - context.wait_for_all_requests(); - VideoSnapshot start = stream.video().snapshot(); - if (start && HomeMenuDetector(env.console, COLOR_RED).detect(start)){ - throw UserSetupError(env.console, - "Start a new game with BDSP running in front, not from the Switch's HOME menu: " - "the intro restarts the game in front."); - } - - // Only close and reopen the game: the states below take it from the user select - // on. reset_game_from_home() waits for a loaded save's black screen to end, and a - // new game's first screen is a dialog on black, so it would sit there until its - // timeout. - trace.action("Restarting the game."); - go_home(env.console, context); - from_home_close_and_reopen_game(env.console, context, true); - - wait_out_boot(stream, context, trace); - IntroRun run{env, context, trace, picks, player_name, current_time(), intro_start(picks), 0}; + const VideoSnapshot start = require_game_in_front(env, context); + const std::string first = start ? classify_intro_screen(&env.console, *start.frame) : "other"; + IntroProgress progress; + if (intro_restart_needed(first)){ + restart_game(env, context, trace); + progress = intro_start(picks); + }else{ + // Never close the game on an intro screen: what was picked stays. + trace.action("The intro shows (" + first + "): taking it up there, without a restart."); + progress = intro_resume(picks, first); + } + IntroRun run{env, context, trace, picks, player_name, current_time(), progress, 0}; + run.language_target = *language_item(picks.game_language); trace.action(std::string("Picks: language ") + language_data(picks.game_language).name + - ", look " + std::to_string(picks.look + 1) + "."); + " (item " + std::to_string(run.language_target) + "), look " + std::to_string(picks.look + 1) + "."); const std::vector states = intro_states(run); run_screen_machine(stream, context, states, std::chrono::minutes(15), trace); trace.action("Intro done."); } +void capture_intro_screens(SingleSwitchProgramEnvironment& env, ProControllerContext& context, size_t look){ + StepTrace trace(env.console, "IntroCapture"); + const IntroPicks picks{Language::English, look}; + if (!look_pick_moves(0, picks)){ + throw UserSetupError(env.console, "The player's look must be one of BDSP's eight."); + } + require_game_in_front(env, context); + const std::string no_name; + const std::vector& tags = bdsp_language_tags(); + for (size_t item = 0; item < tags.size(); item++){ + // No save exists, so each restart plays the intro from the start. + restart_game(env, context, trace); + IntroRun run{env, context, trace, picks, no_name, current_time(), intro_start(picks), 0}; + run.progress.look_pending = true; + run.language_target = item; + run.capture = true; + run.capture_menus = item == 0; + run.capture_tag = std::string(tags[item]); + trace.action("Capturing language item " + std::to_string(item) + " (" + run.capture_tag + ")."); + const std::vector states = intro_states(run); + run_screen_machine(env.console, context, states, std::chrono::minutes(10), trace); + } + trace.action("Captured " + std::to_string(tags.size()) + " languages to " + SCREENSHOTS_PATH()); +} + } // namespace PokemonBDSP diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h index 4a26600902..758e94543a 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h @@ -9,6 +9,7 @@ * Both menus are read by their detector (PokemonBDSP_AutoStory_IntroMenuDetector.h). * */ +// code-health: allow header_symbols -- the intro's entry points (play, capture), its two menus' data and its screen classifier #ifndef PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H #define PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H @@ -16,6 +17,7 @@ #include #include #include +#include #include #include "CommonFramework/ImageTypes/ImageViewRGB32.h" #include "CommonFramework/Language.h" @@ -65,17 +67,35 @@ std::optional> look_pick_moves(size_t cursor, const In std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& screen); namespace detail{ +// Short names of BDSP_LANGUAGE_GRID's items, in its order, for capture +// mode's file names: "en", "ja-kana", "es", "ja", ... +const std::vector& bdsp_language_tags(); // The progress a new intro starts from: which of `picks` need a menu. IntroProgress intro_start(const IntroPicks& picks); +// Whether the intro restarts the game when `screen` (classify_intro_screen()) +// shows at its start: not on a screen of the intro itself, which it takes up +// from there. +bool intro_restart_needed(std::string_view screen); +// The progress an intro taken up on `screen` starts from: a pick whose +// screen is past is not made again, and one that may be past (from a +// dialogue, the look) is made only if its screen still shows. From a +// keyboard it types the player's name: it cannot tell the rival's. +IntroProgress intro_resume(const IntroPicks& picks, std::string_view screen); +// Why `name` cannot be typed on a keyboard that takes `limit` characters, +// or nullopt when it fits or the limit was not read. +std::optional name_limit_problem(std::string_view name, std::optional limit); } // namespace detail // From the title screen or any intro screen of a game with no save, with BDSP // running in front: from HOME it throws UserSetupError, since the restart would -// launch whatever tile HOME's cursor is on, as it does for picks no menu has. Restarts the game first, so it -// always starts from the title, waits out the black boot (up to +// launch whatever tile HOME's cursor is on, as it does for picks no menu has. +// On a screen of the intro (intro_restart_needed()) it takes the intro up +// from there, without closing the game. Otherwise it restarts the game, so it +// starts from the title, waits out the black boot (up to // INTRO_BOOT_LIMIT) without pressing anything, and fails if no BDSP screen shows within // INTRO_TIME_BEFORE_BDSP of screens. Picks the language and the look in -// `picks`, names the player `player_name` and picks the rival's default name +// `picks`, names the player `player_name` (failing at the name screen, +// nothing typed, when the keyboard takes fewer characters) and picks the rival's default name // (Barry) from the list. The language and look selects are read, moved on // and read again before their A. The player's keyboard showing with a pick // still to make fails. Each step reads the @@ -88,6 +108,15 @@ void play_new_game_intro( const std::string& player_name ); +// Developer: shoots the intro's screens for each language of the language +// select, for a game with no save running in front. Per language it +// restarts the game, picks the language and `look` (an item of +// BDSP_LOOK_GRID), and saves the name keyboard that follows as +// /_keyboard.png (bdsp_language_tags()); the first round +// also saves _language-list.png and _look-picker.png before +// their picks. Leaves the game on the last language's keyboard. +void capture_intro_screens(SingleSwitchProgramEnvironment& env, ProControllerContext& context, size_t look); + } // namespace PokemonBDSP } // namespace NintendoSwitch diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_NameLimitReader.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_NameLimitReader.cpp new file mode 100644 index 0000000000..4852623350 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_NameLimitReader.cpp @@ -0,0 +1,77 @@ +/* BDSP AutoStory - Name Limit Reader + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/Language.h" +#include "CommonTools/OCR/OCR_Routines.h" +#include "PokemonBDSP_AutoStory_NameLimitReader.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ + +// The counter, with room for "12/12" to its left. +const ImageFloatBox KEYBOARD_COUNTER_BOX(0.62, 0.32, 0.085, 0.05); + +// The number at `text[pos]`, and where it ends; nullopt when no digit is there. +std::optional read_digits(std::string_view text, size_t& pos){ + const size_t start = pos; + size_t value = 0; + while (pos < text.size() && text[pos] >= '0' && text[pos] <= '9' && pos - start < 4){ + value = value * 10 + (size_t)(text[pos] - '0'); + pos++; + } + if (pos == start){ + return std::nullopt; + } + return value; +} + +void skip_spaces(std::string_view text, size_t& pos){ + while (pos < text.size() && (text[pos] == ' ' || text[pos] == '\n' || text[pos] == '\r')){ + pos++; + } +} + +} // namespace + + +std::optional parse_keyboard_counter(std::string_view text){ + size_t pos = 0; + skip_spaces(text, pos); + if (!read_digits(text, pos)){ + return std::nullopt; + } + skip_spaces(text, pos); + if (pos >= text.size() || text[pos] != '/'){ + return std::nullopt; + } + pos++; + skip_spaces(text, pos); + const std::optional limit = read_digits(text, pos); + skip_spaces(text, pos); + if (!limit || *limit == 0 || pos != text.size()){ + return std::nullopt; + } + return limit; +} + +std::optional read_keyboard_name_limit(const ImageViewRGB32& screen){ + if (screen.width() == 0 || screen.height() == 0){ + return std::nullopt; + } + const std::string text = OCR::ocr_read(Language::English, extract_box_reference(screen, KEYBOARD_COUNTER_BOX)); + return parse_keyboard_counter(text); +} + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_NameLimitReader.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_NameLimitReader.h new file mode 100644 index 0000000000..ac72d6388f --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_NameLimitReader.h @@ -0,0 +1,38 @@ +/* BDSP AutoStory - Name Limit Reader + * + * From: https://github.com/PokemonAutomation/ + * + * The Switch keyboard's character counter, "0/12" under the name field's + * right end, read for the most characters the game takes: BDSP takes fewer + * in some languages, and the keyboard drops the presses past it. Measured + * on a 1920x1080 Switch 1 capture of the same keyboard in Legends: Z-A + * ("0/6", grey on dark blue, about x 1275-1335, y 357-385). + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_NameLimitReader_H +#define PokemonAutomation_PokemonBDSP_AutoStory_NameLimitReader_H + +#include +#include +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// The limit in a counter's text ("0/6" gives 6; spaces and a trailing +// newline are allowed), or nullopt when the text is not "/". +std::optional parse_keyboard_counter(std::string_view text); + +// The keyboard's character limit, read with OCR from its counter; nullopt +// when the counter does not read as one. +std::optional read_keyboard_name_limit(const ImageViewRGB32& screen); + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp index 27d1c2600b..34013e7fd9 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp @@ -72,6 +72,29 @@ constexpr std::string_view STARTER_RNG_HANDOFF_TEXT = "When it is done, close the Summary page and the menu, let Rowan's scene finish so the player stands " "on the Verity Lakefront overworld, save, and start AutoStory at 00.03."; +std::string stop_for_starter_rng_help(){ + return std::format( + "Stop for Starter RNG:
Off by default. Ignored while Run Starter RNG in AutoStory is on. {}", + STARTER_RNG_HANDOFF_TEXT + ); +} + +constexpr std::string_view PLAYER_NAME_HELP = + "Player name:
For a new game. Letters and digits, typed on the Switch's Latin keyboard. " + "If the game takes fewer characters in its language, the intro stops at the name screen."; + +constexpr std::string_view PLAYER_LOOK_HELP = + "Player's look:
For a new game: the look picked on the appearance grid. " + "The portraits row by row: 1 Lucas light, 2 Lucas dark, 3 Dawn dark-haired, 4 Dawn dark; " + "5 Lucas blond, 6 Lucas brown, 7 Dawn blond, 8 Dawn brown-haired. " + "The new game's language is Game language above (Japanese: the kana one)."; + +constexpr std::string_view CAPTURE_INTRO_SCREENS_HELP = + "Capture intro screens (developer):
Instead of a run: for a game with no save, running in front. " + "For each language the game lists, restarts the game, picks the language and the look above, " + "and saves the name keyboard to the Screenshots folder as <lang>_keyboard.png; " + "the first round also saves en_language-list.png and en_look-picker.png. Ends on the last keyboard."; + // The start and end dropdowns' entries: segments 00 and 01 joined, 00.00 // to 01.11. The options hold a reference to it. const StringSelectDatabase& checkpoint_database(){ @@ -151,16 +174,9 @@ BDSPAutoStory::BDSPAutoStory() "until the player is in control in the bedroom. Only when starting at 00.00.", LockMode::LOCK_WHILE_RUNNING, false ) - , PLAYER_NAME( - false, - "Player name:
For a new game. Letters and digits.", - LockMode::LOCK_WHILE_RUNNING, "Kosmos", "Kosmos" - ) - , PLAYER_LOOK( - "Player's look:
For a new game: the look picked on the appearance grid. " - "The portraits row by row: 1 Lucas light, 2 Lucas dark, 3 Dawn dark-haired, 4 Dawn dark; " - "5 Lucas blond, 6 Lucas brown, 7 Dawn blond, 8 Dawn brown-haired." - ) + , PLAYER_NAME(false, std::string(PLAYER_NAME_HELP), LockMode::LOCK_WHILE_RUNNING, "Kosmos", "Kosmos") + , PLAYER_LOOK(std::string(PLAYER_LOOK_HELP)) + , CAPTURE_INTRO_SCREENS(std::string(CAPTURE_INTRO_SCREENS_HELP), LockMode::LOCK_WHILE_RUNNING, false) , STARTER( "Starter Pokemon:", { @@ -171,14 +187,7 @@ BDSPAutoStory::BDSPAutoStory() LockMode::LOCK_WHILE_RUNNING, StarterChoice::CHIMCHAR ) - , STOP_FOR_STARTER_RNG( - std::format( - "Stop for Starter RNG:
Off by default. Ignored while Run Starter RNG in AutoStory is on. {}", - STARTER_RNG_HANDOFF_TEXT - ), - LockMode::LOCK_WHILE_RUNNING, - false - ) + , STOP_FOR_STARTER_RNG(stop_for_starter_rng_help(), LockMode::LOCK_WHILE_RUNNING, false) , NOTIFICATIONS({ &STARTER_RNG.notification_shiny(), &NOTIFICATION_PROGRAM_FINISH, @@ -190,6 +199,7 @@ BDSPAutoStory::BDSPAutoStory() PA_ADD_OPTION(NEW_GAME_INTRO); PA_ADD_OPTION(PLAYER_NAME); PA_ADD_OPTION(PLAYER_LOOK); + PA_ADD_OPTION(CAPTURE_INTRO_SCREENS); PA_ADD_OPTION(STARTER); PA_ADD_OPTION(STOP_FOR_STARTER_RNG); add_option(STARTER_RNG.run(), "RUN_STARTER_RNG"); @@ -309,6 +319,11 @@ std::vector load_run_rows(ConsoleHandle& console){ } // namespace void BDSPAutoStory::program(SingleSwitchProgramEnvironment& env, ProControllerContext& context){ + if (CAPTURE_INTRO_SCREENS){ + capture_intro_screens(env, context, PLAYER_LOOK.index()); + send_program_finished_notification(env, NOTIFICATION_PROGRAM_FINISH, "Intro screens captured."); + return; + } const std::vector checkpoints = load_run_rows(env.console); const size_t first = RUN.start_index(); const size_t last = RUN.end_index(); @@ -319,6 +334,10 @@ void BDSPAutoStory::program(SingleSwitchProgramEnvironment& env, ProControllerCo set_story_language(RUN.language()); const bool new_game = first == 0 && NEW_GAME_INTRO; + // Starter RNG watches the player's eyes through the look's template. + if (new_game && STARTER_RNG.run() && STARTER_RNG.player_model().index() != PLAYER_LOOK.index()){ + throw UserSetupError(env.console, "The Starter RNG character model must be the new game's look (Player's look)."); + } StoryEnv story{env, context, STARTER, PLAYER_NAME, IntroPicks{RUN.language(), PLAYER_LOOK.index()}, {}}; if (STARTER_RNG.run()){ story.run_starter_rng = make_starter_rng_hook( diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h index 7e410506e6..1a884d9e3b 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.h @@ -62,6 +62,8 @@ class BDSPAutoStory : public SingleSwitchProgramInstance, public ConfigOption::L BooleanCheckBoxOption NEW_GAME_INTRO; StringOption PLAYER_NAME; PlayerModelOption PLAYER_LOOK; + // Developer: shoot the intro's screens in every language instead of a run. + BooleanCheckBoxOption CAPTURE_INTRO_SCREENS; EnumDropdownOption STARTER; // The RNG program the run stops for (StoryHandoff), or runs in 00.02. BooleanCheckBoxOption STOP_FOR_STARTER_RNG; diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 3f79777b2a..c05f99e3d8 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1827,6 +1827,8 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_IntroMenuDetector.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_NameLimitReader.cpp + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_NameLimitReader.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.cpp Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_ScreenEvents.cpp From 5f37c5ab029af2b378eb8bf8b6a9389a1d6e4d71 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 15 Nov 2026 00:24:51 -0800 Subject: [PATCH 08/19] BDSP AutoStory intro capture: record what follows the look Capture mode no longer assumes the name keyboard follows the look pick. From the look's A it records frames as _after-look_NN.png: one on each screen change (pixel RMSD over CAPTURE_CHANGE vs the last saved) and at least one per second, until the screen holds still for 3 s (not before 8 s) or 30 s pass. It presses nothing: on the keyboard every button edits, confirms or closes it. The look picker is saved every round as _before-look-pick.png, the language list on the first as en_before-language-pick.png; a missing frame is logged, not a failure. The story-run intro is unchanged. --- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 70 +++++++++++ .../Actions/PokemonBDSP_AutoStory_NewGame.cpp | 113 ++++++++++++++---- .../Actions/PokemonBDSP_AutoStory_NewGame.h | 47 +++++++- .../AutoStory/PokemonBDSP_AutoStory.cpp | 6 +- 4 files changed, 207 insertions(+), 29 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index 3ea1da8056..5dd8014f59 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -478,6 +479,71 @@ UnitTestResult test_language_tags(Logger&){ return true; } +// The screen at each 250 ms poll, a letter per look. Past the end the last +// letter holds, or with `cycle` the letters start over. +struct BurstInput{ + std::string screens; + bool cycle = false; +}; +struct BurstResult{ + size_t saves; + std::optional stop_at; + bool operator==(const BurstResult&) const = default; +}; +// Runs FrameBurst over `input`, a change being any other letter. +BurstResult run_burst(const BurstInput& input){ + using std::chrono::milliseconds; + FrameBurst burst; + char saved = 0; + char previous = 0; + BurstResult result{0, std::nullopt}; + const std::string& screens = input.screens; + milliseconds at(0); + for (size_t i = 0; i < 400 && !result.stop_at; i++, at += milliseconds(250)){ + const char screen = i < screens.size() ? screens[i] + : input.cycle ? screens[i % screens.size()] : screens.back(); + const auto change = [screen](char other){ return other == screen ? 0.0 : 100.0; }; + const BurstStep step = burst.see(at, change(saved), change(previous)); + previous = screen; + if (step.save){ + saved = screen; + result.saves++; + } + if (step.stop){ + result.stop_at = at; + } + } + return result; +} + +// Capture mode's recording after the look: a frame on each change and each +// second, until the screen holds still for CAPTURE_STILL_FOR (not before +// CAPTURE_MIN_LENGTH), or CAPTURE_MAX_LENGTH. +UnitTestResult test_after_look_burst(Logger&){ + using std::chrono::milliseconds; + using Input = BurstInput; + using Expectation = BurstResult; + struct Case{ + std::string name; + Input input; + Expectation expectation; + }; + const Case cases[] = { + {"a fade, then the keyboard from 1.5 s", {"LLssssK"}, {9, milliseconds(8000)}}, + {"still from the start", {"L"}, {9, milliseconds(8000)}}, + {"a change at 6 s", {std::string(24, 'L') + "K"}, {10, milliseconds(9000)}}, + {"never still", {"ab", true}, {121, CAPTURE_MAX_LENGTH}}, + }; + for (const Case& c : cases){ + const BurstResult got = run_burst(c.input); + if (got != c.expectation){ + return std::format("{}: {} frames saved, stopped at {}.", c.name, got.saves, + got.stop_at ? std::format("{} ms", got.stop_at->count()) : std::string("never")); + } + } + return true; +} + // The intro's two selects from lab-B 2026-11-09 (each cursor on its first // item), the screens one A before and after the look select, and screens // that are neither. Screenshots are in the unit test resources; a missing @@ -876,6 +942,10 @@ const AutoStoryActionsCase AUTO_STORY_ACTIONS_CASES[] = { .input = {"new-game intro: capture file per language", test_language_tags}, .expectation = {}, }, + { + .input = {"new-game intro: capture records what follows the look", test_after_look_burst}, + .expectation = {}, + }, { .input = {"intro menu screenshots", test_intro_menu_screenshots}, .expectation = {}, diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp index e1f0f021ea..be238a405f 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp @@ -285,6 +285,19 @@ void record_intro_step(IntroProgress& progress, IntroStep step){ return; } } + +BurstStep FrameBurst::see(std::chrono::milliseconds at, double from_saved, double from_previous){ + if (from_previous > CAPTURE_CHANGE){ + m_last_change = at; + } + BurstStep step; + step.save = !m_last_saved || from_saved > CAPTURE_CHANGE || at - *m_last_saved >= CAPTURE_EVERY; + if (step.save){ + m_last_saved = at; + } + step.stop = at >= CAPTURE_MAX_LENGTH || (at >= CAPTURE_MIN_LENGTH && at - m_last_change >= CAPTURE_STILL_FOR); + return step; +} } // namespace detail namespace{ @@ -350,32 +363,86 @@ struct IntroRun{ bool in_control = false; // The language select's item to pick. size_t language_target = 0; - // Capture mode (capture_intro_screens()): the run ends on the player's - // keyboard, saved as _keyboard.png, and with - // `capture_menus` the two selects are saved before their picks. + // Capture mode (capture_intro_screens()): the run ends with the + // recording after the look's A, saved as _after-look_NN.png; + // the screen is saved before the look's pick, and with + // `capture_language_pick` before the language's. bool capture = false; - bool capture_menus = false; + bool capture_language_pick = false; std::string capture_tag; bool capture_done = false; }; -// Saves the frame on screen now as /_.png. +// Saves `frame` as /_.png. +void save_frame(IntroRun& run, const VideoSnapshot& frame, std::string_view what){ + const std::string path = std::format("{}{}_{}.png", SCREENSHOTS_PATH(), run.capture_tag, what); + frame.frame->save(path); + run.trace.action("Captured " + path); +} + +// Saves the frame on screen now, whatever it shows, as /_.png. void save_capture(IntroRun& run, std::string_view what){ run.context.wait_for_all_requests(); const VideoSnapshot screen = run.env.console.video().snapshot(); if (!screen){ - run.trace.fail("No video frame to capture."); + run.trace.action(std::format("No video frame for {}_{}: nothing saved.", run.capture_tag, what)); + return; } - const std::string path = std::format("{}{}_{}.png", SCREENSHOTS_PATH(), run.capture_tag, what); - screen.frame->save(path); - run.trace.action("Captured " + path); + save_frame(run, screen, what); +} + +// How much the whole screen changed from `before` to `after`, pixel RMSD; +// large when either is missing. +double frame_change(const VideoSnapshot& before, const VideoSnapshot& after){ + if (!before || !after){ + return 1000; + } + return ImageMatch::pixel_RMSD(*before.frame, *after.frame); } -// Capture mode: the name keyboard after the look's A. Saved after its -// slide-in, also when the keyboard detector does not know its layout. -void capture_keyboard(IntroRun& run){ - pbf_wait(run.context, 2000ms); - save_capture(run, "keyboard"); +// Polls between capture mode's frames after the look. +const Milliseconds CAPTURE_POLL = 250ms; + +// Capture mode, from the look's A: records what the game shows next as +// _after-look_NN.png (FrameBurst decides which frames and when to +// stop), reading nothing on it. It presses nothing. On lab-B (2026-11-09, +// English) the look's A goes straight to the system name keyboard, and on +// that keyboard no button is safe: A types the key under the cursor, B +// deletes a character or, on an empty field, closes the keyboard, PLUS +// confirms the name, X cancels it (CANCEL_RIVAL_KEYBOARD), and the D-pad +// and sticks move the cursor the next A types from. So dialogue that some +// language may show before its keyboard is not advanced: the recording +// runs at least CAPTURE_MIN_LENGTH and ends on it, and the trace says so. +void record_after_look(IntroRun& run){ + VideoStream& stream = run.env.console; + run.context.wait_for_all_requests(); + const WallClock start = current_time(); + FrameBurst burst; + VideoSnapshot saved; + VideoSnapshot previous; + size_t count = 0; + std::chrono::milliseconds at(0); + // Bounded by CAPTURE_MAX_LENGTH: FrameBurst stops there, and so does a feed with no frames. + while (at < CAPTURE_MAX_LENGTH){ + VideoSnapshot frame = stream.video().snapshot(); + at = std::chrono::duration_cast(current_time() - start); + if (frame){ + const BurstStep step = burst.see(at, frame_change(saved, frame), frame_change(previous, frame)); + if (step.save){ + save_frame(run, frame, std::format("after-look_{:02}", count++)); + saved = frame; + } + previous = std::move(frame); + if (step.stop){ + break; + } + } + run.context.wait_for(CAPTURE_POLL); + } + run.trace.action(std::format( + "Recorded {} frames over {} s after the look's pick, pressing nothing. If the last ones show " + "dialogue rather than the name keyboard, that dialogue blocks the keyboard in capture mode.", + count, at.count() / 1000)); run.capture_done = true; } @@ -502,8 +569,8 @@ bool pick_on_grid(IntroRun& run, const IntroMenu& menu, size_t target){ // PICK_LANGUAGE or PICK_LOOK. void pick_on_intro_menu(IntroRun& run, IntroStep step){ const bool language = step == IntroStep::PICK_LANGUAGE; - if (run.capture_menus){ - save_capture(run, language ? "language-list" : "look-picker"); + if (language ? run.capture_language_pick : run.capture){ + save_capture(run, language ? "before-language-pick" : "before-look-pick"); } if (language){ if (pick_on_grid(run, LANGUAGE_MENU, run.language_target)){ @@ -514,7 +581,7 @@ void pick_on_intro_menu(IntroRun& run, IntroStep step){ if (pick_on_grid(run, LOOK_MENU, run.picks.look)){ run.progress.look_pending = false; if (run.capture){ - capture_keyboard(run); + record_after_look(run); } } } @@ -542,10 +609,11 @@ IntroStep next_intro_step(IntroRun& run, std::string_view screen){ " was read."); } -// ENTER_PLAYER_NAME: types the name, or in capture mode shoots the keyboard. +// ENTER_PLAYER_NAME: types the name, or in capture mode records the screen +// as after the look, typing nothing. void name_player(IntroRun& run){ if (run.capture){ - capture_keyboard(run); + record_after_look(run); return; } enter_player_name(run.env.console, run.context, run.trace, run.player_name); @@ -625,10 +693,10 @@ std::vector intro_states(IntroRun& run){ return classify_intro_screen(&run.env.console, screen) == name; }; return { - // The player in control without the menu read, or capture mode's keyboard shot. + // The player in control without the menu read, or capture mode's recording done. {"finished", [&run](const ImageViewRGB32&){ return run.in_control || run.capture_done; }, nullptr, - [&run]{ run.trace.action(run.capture_done ? "Keyboard captured." : "X opened and B closed the menu from a still bedroom: the player is in control."); }, + [&run]{ run.trace.action(run.capture_done ? "Recording after the look done." : "X opened and B closed the menu from a still bedroom: the player is in control."); }, {}, 1, true}, {"user-select", [=](const ImageViewRGB32& s){ return screen_is(s, "user-select"); }, nullptr, @@ -753,7 +821,8 @@ void capture_intro_screens(SingleSwitchProgramEnvironment& env, ProControllerCon run.progress.look_pending = true; run.language_target = item; run.capture = true; - run.capture_menus = item == 0; + // The language list shows before any language is picked: once is enough. + run.capture_language_pick = item == 0; run.capture_tag = std::string(tags[item]); trace.action("Capturing language item " + std::to_string(item) + " (" + run.capture_tag + ")."); const std::vector states = intro_states(run); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h index 758e94543a..f845a26444 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h @@ -9,11 +9,12 @@ * Both menus are read by their detector (PokemonBDSP_AutoStory_IntroMenuDetector.h). * */ -// code-health: allow header_symbols -- the intro's entry points (play, capture), its two menus' data and its screen classifier +// code-health: allow header_symbols -- the intro's entry points (play, capture), its two menus' data, its screen classifier and capture mode's recording policy #ifndef PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H #define PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H +#include #include #include #include @@ -84,6 +85,39 @@ IntroProgress intro_resume(const IntroPicks& picks, std::string_view screen); // Why `name` cannot be typed on a keyboard that takes `limit` characters, // or nullopt when it fits or the limit was not read. std::optional name_limit_problem(std::string_view name, std::optional limit); + +// Capture mode's recording after the look's A, which reads nothing on the +// screen: frames are polled, one is saved when it changed or a while after +// the last, and the recording stops when the screen holds still. +// A change between two frames, pixel RMSD over the whole screen, larger +// than this is a change: a frame that far from the last saved is saved. +inline constexpr double CAPTURE_CHANGE = 4; +// A frame is saved at least this long after the last saved. +inline constexpr std::chrono::milliseconds CAPTURE_EVERY = std::chrono::seconds(1); +// The recording stops when no poll changed the screen for this long... +inline constexpr std::chrono::milliseconds CAPTURE_STILL_FOR = std::chrono::seconds(3); +// ...but not before this, which leaves dialogue that pauses before the +// keyboard time to show in the frames, +inline constexpr std::chrono::milliseconds CAPTURE_MIN_LENGTH = std::chrono::seconds(8); +// and at the latest at this. +inline constexpr std::chrono::milliseconds CAPTURE_MAX_LENGTH = std::chrono::seconds(30); +// What the recording does with one polled frame. +struct BurstStep{ + bool save = false; + bool stop = false; +}; +// The recording's save and stop decisions; times are since it began. +class FrameBurst{ +public: + // The frame polled at `at`: `from_saved` is its change from the last + // saved frame (any value before the first save), `from_previous` from + // the frame polled before it. + BurstStep see(std::chrono::milliseconds at, double from_saved, double from_previous); + +private: + std::optional m_last_saved; + std::chrono::milliseconds m_last_change{0}; +}; } // namespace detail // From the title screen or any intro screen of a game with no save, with BDSP @@ -111,10 +145,13 @@ void play_new_game_intro( // Developer: shoots the intro's screens for each language of the language // select, for a game with no save running in front. Per language it // restarts the game, picks the language and `look` (an item of -// BDSP_LOOK_GRID), and saves the name keyboard that follows as -// /_keyboard.png (bdsp_language_tags()); the first round -// also saves _language-list.png and _look-picker.png before -// their picks. Leaves the game on the last language's keyboard. +// BDSP_LOOK_GRID), and records what shows after the look's A, pressing +// nothing, as /_after-look_NN.png (bdsp_language_tags(), +// NN from 00 in order; FrameBurst). Each round saves the screen as the look +// is about to be picked as _before-look-pick.png, and the first round +// the one before the language pick as _before-language-pick.png. It +// never types or confirms a name, and leaves the game where the last +// recording ended. void capture_intro_screens(SingleSwitchProgramEnvironment& env, ProControllerContext& context, size_t look); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp index 34013e7fd9..47c4cec539 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory.cpp @@ -92,8 +92,10 @@ constexpr std::string_view PLAYER_LOOK_HELP = constexpr std::string_view CAPTURE_INTRO_SCREENS_HELP = "Capture intro screens (developer):
Instead of a run: for a game with no save, running in front. " "For each language the game lists, restarts the game, picks the language and the look above, " - "and saves the name keyboard to the Screenshots folder as <lang>_keyboard.png; " - "the first round also saves en_language-list.png and en_look-picker.png. Ends on the last keyboard."; + "and records what shows after the look, pressing nothing, to the Screenshots folder as " + "<lang>_after-look_NN.png (a frame per change and per second, until the screen holds still). " + "Also saves <lang>_before-look-pick.png, and en_before-language-pick.png on the first round. " + "Never types a name; ends where the last recording ended."; // The start and end dropdowns' entries: segments 00 and 01 joined, 00.00 // to 01.11. The options hold a reference to it. From 5185b39ba5352299be3ee06c62ec731b79ecd1ac Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 15 Nov 2026 00:26:11 -0800 Subject: [PATCH 09/19] BDSP AutoStory capture: three-digit frame numbers so up to 121 frames sort by name --- .../AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp | 6 +++--- .../AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp index be238a405f..e25880ae9d 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp @@ -364,7 +364,7 @@ struct IntroRun{ // The language select's item to pick. size_t language_target = 0; // Capture mode (capture_intro_screens()): the run ends with the - // recording after the look's A, saved as _after-look_NN.png; + // recording after the look's A, saved as _after-look_NNN.png; // the screen is saved before the look's pick, and with // `capture_language_pick` before the language's. bool capture = false; @@ -404,7 +404,7 @@ double frame_change(const VideoSnapshot& before, const VideoSnapshot& after){ const Milliseconds CAPTURE_POLL = 250ms; // Capture mode, from the look's A: records what the game shows next as -// _after-look_NN.png (FrameBurst decides which frames and when to +// _after-look_NNN.png (FrameBurst decides which frames and when to // stop), reading nothing on it. It presses nothing. On lab-B (2026-11-09, // English) the look's A goes straight to the system name keyboard, and on // that keyboard no button is safe: A types the key under the cursor, B @@ -429,7 +429,7 @@ void record_after_look(IntroRun& run){ if (frame){ const BurstStep step = burst.see(at, frame_change(saved, frame), frame_change(previous, frame)); if (step.save){ - save_frame(run, frame, std::format("after-look_{:02}", count++)); + save_frame(run, frame, std::format("after-look_{:03}", count++)); saved = frame; } previous = std::move(frame); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h index f845a26444..bfe89a5461 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h @@ -146,7 +146,7 @@ void play_new_game_intro( // select, for a game with no save running in front. Per language it // restarts the game, picks the language and `look` (an item of // BDSP_LOOK_GRID), and records what shows after the look's A, pressing -// nothing, as /_after-look_NN.png (bdsp_language_tags(), +// nothing, as /_after-look_NNN.png (bdsp_language_tags(), // NN from 00 in order; FrameBurst). Each round saves the screen as the look // is about to be picked as _before-look-pick.png, and the first round // the one before the language pick as _before-language-pick.png. It From cde226fe67487f07108163f2b53096d1317271ea Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 15 Nov 2026 01:18:26 -0800 Subject: [PATCH 10/19] BDSP intro: red tests for taking the intro up past the language pick and the language load's visits jp1 (lab-B 2026-11-15) took the intro up on the look select of a zh-Hant game while Japanese was asked for; jp2 failed on the language select's visit cap while Japanese loaded. The cap gets a name, still 10. --- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 66 +++++++++---------- .../Actions/PokemonBDSP_AutoStory_NewGame.cpp | 2 +- .../Actions/PokemonBDSP_AutoStory_NewGame.h | 4 ++ 3 files changed, 38 insertions(+), 34 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index 5dd8014f59..cf43e17d3f 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -369,18 +369,16 @@ UnitTestResult test_intro_pick_steps(Logger&){ return true; } -// Taken up on an intro screen, the intro restarts nothing and makes only -// the picks still ahead; anywhere else it restarts the game. -UnitTestResult test_intro_resume(Logger&){ +// The intro is taken up, without a restart, only before the language is +// picked: past it the game's language may not be the one asked for (jp1, +// lab-B 2026-11-15: taken up on the look select of a Traditional Chinese +// game while Japanese was asked for, it typed the name on a CJK keyboard). +UnitTestResult test_intro_restart(Logger&){ struct Input{ std::string_view screen; }; struct Expectation{ bool restart; - bool language_pending; - bool look_pending; - bool picks_may_be_past; - bool player_named; }; struct Case{ std::string_view name; @@ -388,36 +386,34 @@ UnitTestResult test_intro_resume(Logger&){ Expectation expectation; }; const Case cases[] = { - {"the title or a logo", {"other"}, {true, true, true, false, false}}, - {"black", {"black"}, {true, true, true, false, false}}, - {"the bedroom's menu", {"menu"}, {true, true, true, false, false}}, - {"the user select", {"user-select"}, {false, true, true, false, false}}, - {"the language select", {"language-select"}, {false, true, true, false, false}}, - {"the look select", {"look-select"}, {false, false, true, false, false}}, - {"a dialogue", {"dialog"}, {false, false, true, true, false}}, - {"the player's keyboard", {"keyboard"}, {false, false, false, false, false}}, - {"the rival's name list", {"rival-list-tell"}, {false, false, false, false, true}}, - {"a preset rival name", {"rival-list-preset"}, {false, false, false, false, true}}, + {"the title or a logo", {"other"}, {true}}, + {"black", {"black"}, {true}}, + {"the bedroom's menu", {"menu"}, {true}}, + {"the user select", {"user-select"}, {false}}, + {"the language select", {"language-select"}, {false}}, + {"the look select", {"look-select"}, {true}}, + {"a dialogue", {"dialog"}, {true}}, + {"the player's keyboard", {"keyboard"}, {true}}, + {"the rival's name list", {"rival-list-tell"}, {true}}, + {"a preset rival name", {"rival-list-preset"}, {true}}, }; - const IntroPicks picks{Language::Japanese, 6}; for (const Case& c : cases){ - const Expectation& e = c.expectation; - if (intro_restart_needed(c.input.screen) != e.restart){ + if (intro_restart_needed(c.input.screen) != c.expectation.restart){ return std::string("Wrong restart for ") + std::string(c.name) + "."; } - if (e.restart){ - continue; - } - const IntroProgress progress = intro_resume(picks, c.input.screen); - if (progress.language_pending != e.language_pending || progress.look_pending != e.look_pending || - progress.picks_may_be_past != e.picks_may_be_past || progress.player_named != e.player_named - ){ - return std::string("Wrong progress taken up on ") + std::string(c.name) + "."; - } } - // From a dialogue past the look select, the keyboard types the name. - if (decide_intro_step("keyboard", intro_resume(picks, "dialog")) != IntroStep::ENTER_PLAYER_NAME){ - return "Taken up on a dialogue, the player's keyboard must take the name."; + return true; +} + +// After the pick, Japanese's language select stays up while the game loads +// the language: the pick and 9 A presses in the capture run, 10 in jp2 +// (lab-B 2026-11-15), where the cap of 10 failed the intro. The cap allows +// twice that. +UnitTestResult test_language_select_visits(Logger&){ + const size_t japanese_load_visits = 1 + 10; + if (INTRO_LANGUAGE_SELECT_VISITS < 2 * japanese_load_visits){ + return std::format("The language select may come up {} times; Japanese's load alone took {}.", + INTRO_LANGUAGE_SELECT_VISITS, japanese_load_visits); } return true; } @@ -931,7 +927,11 @@ const AutoStoryActionsCase AUTO_STORY_ACTIONS_CASES[] = { .expectation = {}, }, { - .input = {"new-game intro: taken up on the screen it shows", test_intro_resume}, + .input = {"new-game intro: taken up only before the language pick", test_intro_restart}, + .expectation = {}, + }, + { + .input = {"new-game intro: the language select through the language's load", test_language_select_visits}, .expectation = {}, }, { diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp index e25880ae9d..b4e1d5408f 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp @@ -712,7 +712,7 @@ std::vector intro_states(IntroRun& run){ [&run]{ act(run, "keyboard"); }, {"rival-list-tell", "dialog", "black", "other"}, 3}, {"language-select", [=](const ImageViewRGB32& s){ return screen_is(s, "language-select"); }, nullptr, - [&run]{ act(run, "language-select"); }, {}, 10}, + [&run]{ act(run, "language-select"); }, {}, INTRO_LANGUAGE_SELECT_VISITS}, {"look-select", [=](const ImageViewRGB32& s){ return screen_is(s, "look-select"); }, nullptr, [&run]{ act(run, "look-select"); }, {}, 10}, diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h index bfe89a5461..4d074d9169 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h @@ -68,6 +68,10 @@ std::optional> look_pick_moves(size_t cursor, const In std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& screen); namespace detail{ +// How often the language select may come up, its pick included, before the +// intro fails. After the pick it stays up while the game loads the +// language, and each look takes A (a pick whose A was dropped). +inline constexpr size_t INTRO_LANGUAGE_SELECT_VISITS = 10; // Short names of BDSP_LANGUAGE_GRID's items, in its order, for capture // mode's file names: "en", "ja-kana", "es", "ja", ... const std::vector& bdsp_language_tags(); From c7c54160077790d9d41e4c1ba415feb8c94ca5f7 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 15 Nov 2026 01:18:59 -0800 Subject: [PATCH 11/19] BDSP intro: restart past the language pick; let the language select stay up through the language's load jp1: taken up on a later intro screen, the intro kept the game's language (zh-Hant) while Japanese was asked for. It now takes the intro up only on the user select and the language select, and restarts on any other screen; intro_resume() and picks_may_be_past go with it. jp2: the pick did confirm, but Japanese's load keeps the language select up for the pick and about 10 more looks (~6 s; 9 in the capture run, at the cap of 10). The cap is now 30. --- .../PokemonBDSP_AutoStory_IntroSteps.h | 3 -- .../Actions/PokemonBDSP_AutoStory_NewGame.cpp | 37 +++---------------- .../Actions/PokemonBDSP_AutoStory_NewGame.h | 19 ++++------ 3 files changed, 13 insertions(+), 46 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_IntroSteps.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_IntroSteps.h index 99202b34c1..d91340e93c 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_IntroSteps.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_IntroSteps.h @@ -46,9 +46,6 @@ struct IntroProgress{ // The first look needs no menu: A takes it. bool language_pending = false; bool look_pending = false; - // The intro was taken up past the screen of a pending pick, which may - // have been made before: the player's keyboard does not fail on it. - bool picks_may_be_past = false; bool player_named = false; // A preset name was picked from the rival's list, and how many screens // have shown since without the list coming back. "Barry, is it?" and diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp index b4e1d5408f..8e9280b72c 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp @@ -95,31 +95,7 @@ IntroProgress intro_start(const IntroPicks& picks){ return progress; } bool intro_restart_needed(std::string_view screen){ - return !(screen == "user-select" || screen == "language-select" || screen == "look-select" || - screen == "keyboard" || screen == "dialog" || screen == "rival-list-tell" || screen == "rival-list-preset"); -} -IntroProgress intro_resume(const IntroPicks& picks, std::string_view screen){ - IntroProgress progress = intro_start(picks); - if (screen == "user-select" || screen == "language-select"){ - return progress; - } - // Every other intro screen comes after the language. - progress.language_pending = false; - if (screen == "look-select"){ - // It shows: the cursor may be anywhere, so read it and pick. - progress.look_pending = true; - return progress; - } - if (screen == "dialog"){ - // Rowan before the look select, or the names after it. - progress.picks_may_be_past = true; - return progress; - } - progress.look_pending = false; - if (screen == "rival-list-tell" || screen == "rival-list-preset"){ - progress.player_named = true; - } - return progress; + return !(screen == "user-select" || screen == "language-select"); } std::optional name_limit_problem(std::string_view name, std::optional limit){ if (!limit){ @@ -193,7 +169,7 @@ IntroStep decide_intro_step(std::string_view screen, const IntroProgress& progre return IntroStep::FAIL_NOT_IN_GAME; } if (screen == "keyboard"){ - if (!progress.player_named && (progress.language_pending || progress.look_pending) && !progress.picks_may_be_past){ + if (!progress.player_named && (progress.language_pending || progress.look_pending)){ return IntroStep::FAIL_PICK_MISSED; } return progress.player_named ? IntroStep::CANCEL_RIVAL_KEYBOARD : IntroStep::ENTER_PLAYER_NAME; @@ -787,16 +763,13 @@ void play_new_game_intro( const VideoSnapshot start = require_game_in_front(env, context); const std::string first = start ? classify_intro_screen(&env.console, *start.frame) : "other"; - IntroProgress progress; if (intro_restart_needed(first)){ restart_game(env, context, trace); - progress = intro_start(picks); }else{ - // Never close the game on an intro screen: what was picked stays. - trace.action("The intro shows (" + first + "): taking it up there, without a restart."); - progress = intro_resume(picks, first); + // Nothing is picked yet, and the user select would go away with a restart. + trace.action("The intro shows (" + first + "), before the language pick: taking it up there, without a restart."); } - IntroRun run{env, context, trace, picks, player_name, current_time(), progress, 0}; + IntroRun run{env, context, trace, picks, player_name, current_time(), intro_start(picks), 0}; run.language_target = *language_item(picks.game_language); trace.action(std::string("Picks: language ") + language_data(picks.game_language).name + " (item " + std::to_string(run.language_target) + "), look " + std::to_string(picks.look + 1) + "."); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h index 4d074d9169..41936a0391 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h @@ -70,22 +70,19 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& namespace detail{ // How often the language select may come up, its pick included, before the // intro fails. After the pick it stays up while the game loads the -// language, and each look takes A (a pick whose A was dropped). -inline constexpr size_t INTRO_LANGUAGE_SELECT_VISITS = 10; +// language, and each look takes A (a pick whose A was dropped): Japanese's +// load took the pick and 10 looks (lab-B 2026-11-15), about 6 s. +inline constexpr size_t INTRO_LANGUAGE_SELECT_VISITS = 30; // Short names of BDSP_LANGUAGE_GRID's items, in its order, for capture // mode's file names: "en", "ja-kana", "es", "ja", ... const std::vector& bdsp_language_tags(); // The progress a new intro starts from: which of `picks` need a menu. IntroProgress intro_start(const IntroPicks& picks); // Whether the intro restarts the game when `screen` (classify_intro_screen()) -// shows at its start: not on a screen of the intro itself, which it takes up -// from there. +// shows at its start: on every screen but the user select and the language +// select, which come before the language is picked. Past the pick the game's +// language may not be the one asked for, and nothing on screen tells it. bool intro_restart_needed(std::string_view screen); -// The progress an intro taken up on `screen` starts from: a pick whose -// screen is past is not made again, and one that may be past (from a -// dialogue, the look) is made only if its screen still shows. From a -// keyboard it types the player's name: it cannot tell the rival's. -IntroProgress intro_resume(const IntroPicks& picks, std::string_view screen); // Why `name` cannot be typed on a keyboard that takes `limit` characters, // or nullopt when it fits or the limit was not read. std::optional name_limit_problem(std::string_view name, std::optional limit); @@ -127,8 +124,8 @@ class FrameBurst{ // From the title screen or any intro screen of a game with no save, with BDSP // running in front: from HOME it throws UserSetupError, since the restart would // launch whatever tile HOME's cursor is on, as it does for picks no menu has. -// On a screen of the intro (intro_restart_needed()) it takes the intro up -// from there, without closing the game. Otherwise it restarts the game, so it +// On the user select or the language select (intro_restart_needed()) it takes +// the intro up from there, without closing the game. Otherwise it restarts the game, so it // starts from the title, waits out the black boot (up to // INTRO_BOOT_LIMIT) without pressing anything, and fails if no BDSP screen shows within // INTRO_TIME_BEFORE_BDSP of screens. Picks the language and the look in From 5d64b5fe22ca0dc1317f9f98469762e870fbd4b1 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 15 Nov 2026 01:39:25 -0800 Subject: [PATCH 12/19] BDSP AutoStory intro: wait on unknown screens between the language and look picks On Japanese the look select showed for a frame before it read as one, and the A taken there picked the default look. --- .../Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp | 5 +++++ .../AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index cf43e17d3f..dae1ad9b40 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -259,6 +259,11 @@ UnitTestResult test_intro_gate(Logger&){ if (decide_intro_step("other", progress) != IntroStep::ADVANCE){ return "Once BDSP's dialogue has shown, the clock stops."; } + IntroProgress before_look = progress; + before_look.look_pending = true; + if (decide_intro_step("other", before_look) != IntroStep::WAIT){ + return "Between the language pick and the look pick, an unknown screen waits: A there can take the default look."; + } return check_confirming_screens(); } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp index 8e9280b72c..395c48ee64 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp @@ -206,6 +206,13 @@ IntroStep decide_intro_step(std::string_view screen, const IntroProgress& progre if (!progress.game_confirmed && progress.unconfirmed_for >= INTRO_TIME_BEFORE_BDSP){ return IntroStep::FAIL_NOT_BDSP; } + // Between the language pick and the look pick, an unknown screen is a fade or + // a cutscene frame, and the look select can show for a frame before it reads + // as one (A on that frame takes look 1). Wait there; the + // professor's lines are read as dialog and take their A. + if (!progress.language_pending && progress.look_pending){ + return IntroStep::WAIT; + } return IntroStep::ADVANCE; } bool intro_screen_confirms_bdsp(std::string_view screen){ From e384ab4f4b0ca394eb2e96612ce2335a060616d9 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 15 Nov 2026 01:56:56 -0800 Subject: [PATCH 13/19] BDSP AutoStory settings: Options rows per game language The settings step matched every Options row against English label and value images, so a Japanese game failed on Text Speed. Japanese also has a Kana/Kanji row under Text Speed. options_layout() now gives the rows per language: English keeps its label and value images; other languages confirm the row by the selected row with its value highlighted and the value by the column of its red ink (sliders still by their number); Japanese leaves the Kana/Kanji row as it is and finds Autosave a row further down. --- .../Source/PokemonBDSP/PokemonBDSP_Tests.cpp | 2 + .../PokemonBDSP_AutoStory_Settings.cpp | 228 +++++++++++++----- .../Actions/PokemonBDSP_AutoStory_Settings.h | 36 +++ .../PokemonBDSP_AutoStory_Settings_Tests.cpp | 152 ++++++++++++ .../PokemonBDSP_AutoStory_Settings_Tests.h | 27 +++ SerialPrograms/cmake/SourceFiles.cmake | 2 + 6 files changed, 386 insertions(+), 61 deletions(-) create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings_Tests.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings_Tests.h diff --git a/SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp index efb7aaaf5e..33b8447c4f 100644 --- a/SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp @@ -8,6 +8,7 @@ #include "Inference/PokemonBDSP_DialogDetector.h" #include "Inference/PokemonBDSP_MenuDetector.h" #include "Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.h" +#include "Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings_Tests.h" #include "Programs/AutoStory/Simulation/PokemonBDSP_AutoStory_GridSim_Tests.h" #include "Programs/AutoStory/Simulation/PokemonBDSP_AutoStory_FaultSweep_Tests.h" #include "Programs/AutoStory/Simulation/PokemonBDSP_AutoStory_SimFailureCases_Tests.h" @@ -42,6 +43,7 @@ void add_tests(UnitTestDatabase& database){ add_tests_DialogDetector(database); add_tests_MenuDetector(database); add_tests_AutoStory_Actions(database); + add_tests_AutoStory_Settings(database); add_tests_AutoStory_Launch(database); add_tests_AutoStory_Maps(database); add_tests_AutoStory_StoryWorld(database); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp index bf3540c9c8..4c154ab930 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp @@ -7,6 +7,7 @@ */ #include +#include #include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" #include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/ImageTools/ImageBoxes.h" @@ -18,6 +19,7 @@ #include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" #include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" #include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_StoryLanguage.h" #include "PokemonBDSP_AutoStory_Bag.h" #include "PokemonBDSP_AutoStory_BagScreen.h" #include "PokemonBDSP_AutoStory_Settings.h" @@ -27,6 +29,46 @@ namespace NintendoSwitch{ namespace PokemonBDSP{ using namespace std::chrono_literals; +using detail::OptionsLayout; +using detail::OptionsRow; + + +namespace detail{ + +OptionsLayout options_layout(Language language){ + OptionsLayout layout{ + .rows = { + {.label = "textspeed", .value = "fast", .columns = 3, .column = 2}, + {.label = "changewindow", .value = "type1"}, + {.label = "battleeffects", .value = "off", .columns = 2, .column = 1}, + {.label = "battlestyle", .value = "set", .columns = 2, .column = 1}, + {.label = "sendtoboxes", .value = "automatic", .columns = 2, .column = 1}, + {.label = "givenicknames", .value = "dontgive", .columns = 2, .column = 1}, + {.label = "gyroscope", .value = "use", .columns = 2, .column = 0}, + {.label = "autosave", .value = "off", .columns = 2, .column = 1}, + {.label = "shownicknames", .value = "show", .columns = 2, .column = 0}, + {.label = "backgroundmusic", .value = "8", .slider = true}, + {.label = "soundeffects", .value = "8", .slider = true}, + }, + .label_images = language == Language::English, + }; + if (language == Language::Japanese){ + layout.rows.insert(layout.rows.begin() + 1, OptionsRow{.label = "mojimode", .value = ""}); + } + return layout; +} + +std::optional options_value_column(double x, size_t columns){ + // The values' box: the one the value images were cut from. + constexpr double LEFT = 0.33; + constexpr double WIDTH = 0.54; + if (columns == 0 || x < LEFT || x >= LEFT + WIDTH){ + return std::nullopt; + } + return (size_t)((x - LEFT) / WIDTH * columns); +} + +} // namespace detail namespace{ @@ -113,6 +155,28 @@ class TutorialOptionPositionDetector { } return red > value.width() * value.height() * 0.0005; } + // The centre of the selected value's red ink, as a fraction of the + // screen width; none without red ink. + std::optional selected_value_x(const ImageViewRGB32& screen, double y) const { + const ImageFloatBox box{0.33, y - 0.025, 0.54, 0.05}; + ImageViewRGB32 value = extract_box_reference(screen, box); + size_t red = 0; + double x_sum = 0; + for (size_t py = 0; py < value.height(); py++){ + for (size_t px = 0; px < value.width(); px++){ + uint32_t p = value.pixel(px, py); + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + if (r > g + 20 && r > b + 20){ + red++; + x_sum += (double)px; + } + } + } + if (red == 0 || value.width() == 0){ + return std::nullopt; + } + return box.x + box.width * (x_sum / red) / value.width(); + } }; @@ -164,33 +228,18 @@ bool autosave_off_highlight(const ImageViewRGB32& screen, double y){ return off >= 10 && on * 10 < off; } -// The options the story needs, as each reference image names them: the -// row's label and the value to select. -struct Setting{ - std::string label; - std::string value; -}; -const Setting SETTINGS[] = { - {"textspeed", "fast"}, {"changewindow", "type1"}, - {"battleeffects", "off"}, {"battlestyle", "set"}, - {"sendtoboxes", "automatic"}, {"givenicknames", "dontgive"}, - {"gyroscope", "use"}, {"autosave", "off"}, - {"shownicknames", "show"}, {"backgroundmusic", "8"}, - {"soundeffects", "8"}, -}; -constexpr size_t SETTING_COUNT = sizeof(SETTINGS) / sizeof(SETTINGS[0]); -constexpr size_t AUTOSAVE_ROW = 7; -constexpr size_t FIRST_SLIDER_ROW = 9; // The Options screen and its selected row, as one reader. struct OptionsScreen{ TutorialOptionsDetector options_screen; TutorialOptionPositionDetector position; }; -// A row's label and target value, as the reference images show them. +// A row's label and target value, as the reference images show them; +// none for a label without images in the game language, or a row the +// story leaves as it is. struct OptionReferences{ - ImageRGB32 label; - ImageRGB32 value; + std::optional label; + std::optional value; }; // From the overworld (or the main menu): the main menu, settled. @@ -245,8 +294,8 @@ void open_options(VideoStream& stream, ProControllerContext& context, const Tuto } // How well the selected row's value matches the reference. -double option_value_score(const ImageViewRGB32& frame, double y, size_t row, const ImageRGB32& value_reference){ - if (row >= FIRST_SLIDER_ROW){ +double option_value_score(const ImageViewRGB32& frame, double y, const OptionsRow& row, const ImageRGB32& value_reference){ + if (row.slider){ // Compare the numeric value rather than the long slider: // neighboring slider levels otherwise look too similar. ImageViewRGB32 number = value_reference.sub_image( @@ -264,18 +313,18 @@ double option_value_score(const ImageViewRGB32& frame, double y, size_t row, con // Whether Options shows Autosave selected and set to Off: a frame's look. bool autosave_off_on_screen( const ImageViewRGB32& check, const TutorialOptionsDetector& options_screen, - const TutorialOptionPositionDetector& position, const ImageRGB32& autosave_label + const TutorialOptionPositionDetector& position, const std::optional& autosave_label ){ const double check_y = check ? position.selected_row(check) : -1; return check && options_screen.detect(check) && check_y >= 0 && - option_image_score(check, {0.08, check_y - 0.025, 0.24, 0.05}, autosave_label, false) >= 0.80 && + (!autosave_label || option_image_score(check, {0.08, check_y - 0.025, 0.24, 0.05}, *autosave_label, false) >= 0.80) && autosave_off_highlight(check, check_y); } // Autosave Off, on three more frames 200 ms apart. void confirm_autosave_off( VideoStream& stream, ProControllerContext& context, const TutorialOptionsDetector& options_screen, - const TutorialOptionPositionDetector& position, const ImageRGB32& label_reference + const TutorialOptionPositionDetector& position, const std::optional& label_reference ){ for (size_t confirmation = 0; confirmation < 3; confirmation++){ pbf_wait(context, 200ms); @@ -290,8 +339,8 @@ void confirm_autosave_off( // One press toward the target value. Sliders clamp at zero: establish a // known endpoint, then increment; the target image is verified afterward. -void correct_option(ProControllerContext& context, size_t row, bool& slider_reset){ - if (row >= FIRST_SLIDER_ROW && !slider_reset){ +void correct_option(ProControllerContext& context, const OptionsRow& row, bool& slider_reset){ + if (row.slider && !slider_reset){ for (size_t step = 0; step < 12; step++){ pbf_press_dpad(context, DPAD_LEFT, 80ms, 100ms); } @@ -303,62 +352,110 @@ void correct_option(ProControllerContext& context, size_t row, bool& slider_rese } // The selected row on screen, if it is the expected one with its value -// highlighted; -1 otherwise. +// highlighted; -1 otherwise. Without a label image (a game language +// other than English) the row is the one the presses reached: the +// selected row with its value highlighted. double expected_row_y( - VideoStream& stream, const ImageViewRGB32& frame, const TutorialOptionsDetector& options_screen, - const TutorialOptionPositionDetector& position, size_t row, const ImageRGB32& label_reference + VideoStream& stream, const ImageViewRGB32& frame, const OptionsScreen& screen, + const OptionsRow& row, const std::optional& label_reference ){ - const double y = frame && options_screen.detect(frame) ? position.selected_row(frame) : -1; - const bool value_visible = y >= 0 && position.selected_value(frame, y); + const double y = frame && screen.options_screen.detect(frame) ? screen.position.selected_row(frame) : -1; + const bool value_visible = y >= 0 && screen.position.selected_value(frame, y); + if (!label_reference){ + stream.log("[Tutorial settings] Expected row=" + std::string(row.label) + + " row_y=" + std::to_string(y) + " highlighted_value=" + std::to_string(value_visible) + + " (no label image for the game language)"); + return y >= 0 && value_visible ? y : -1; + } const double label_score = y >= 0 ? option_image_score(frame, - {0.08, y - 0.025, 0.24, 0.05}, label_reference, false) : 0; - stream.log("[Tutorial settings] Expected row=" + std::string(SETTINGS[row].label) + + {0.08, y - 0.025, 0.24, 0.05}, *label_reference, false) : 0; + stream.log("[Tutorial settings] Expected row=" + std::string(row.label) + " row_y=" + std::to_string(y) + " highlighted_value=" + std::to_string(value_visible) + " label_match=" + std::to_string(label_score)); return y >= 0 && value_visible && label_score >= 0.80 ? y : -1; } +// Whether the selected row (at `y`) shows the target value by its column: +// the check for a game language without the rows' images. +bool option_column_verified( + VideoStream& stream, const ImageViewRGB32& frame, double y, const OptionsRow& row, + const TutorialOptionPositionDetector& position +){ + if (!row.column){ + // The Change Window arrow picker: its value has no column; the + // story keeps the new game's Type 1. + stream.log("[Tutorial settings] " + std::string(row.label) + " kept as it is (no column to check)."); + return true; + } + const std::optional x = position.selected_value_x(frame, y); + const std::optional column = x ? detail::options_value_column(*x, row.columns) : std::nullopt; + stream.log("[Tutorial settings] " + std::string(row.label) + " selected column=" + + (column ? std::to_string(*column) : "none") + " target=" + std::to_string(*row.column)); + return column == row.column; +} + // Whether the selected row (at `y`) shows the target value: checked on // the frame, with the row's boxes on the overlay. bool option_verified( - VideoStream& stream, ProControllerContext& context, const ImageViewRGB32& frame, double y, size_t row, + VideoStream& stream, ProControllerContext& context, const ImageViewRGB32& frame, double y, const OptionsRow& row, const OptionsScreen& screen, const OptionReferences& references ){ - const ImageRGB32& value_reference = references.value; + const bool autosave = row.label == "autosave"; // These are regular video-overlay boxes, so the existing overlay // display control can show/hide every active ROI. OverlayBoxScope row_overlay(stream.overlay(), COLOR_RED, {0.07, y - 0.04, 0.84, 0.08}, ""); OverlayBoxScope label_overlay(stream.overlay(), COLOR_YELLOW, {0.08, y - 0.025, 0.24, 0.05}, ""); OverlayBoxScope value_overlay(stream.overlay(), COLOR_GREEN, {0.33, y - 0.025, 0.54, 0.05}, ""); VideoOverlaySet autosave_rois(stream.overlay()); - if (row == AUTOSAVE_ROW){ + if (autosave){ autosave_rois.add(COLOR_GREEN, {0.63, y - 0.022, 0.055, 0.044}); autosave_rois.add(COLOR_RED, {0.34, y - 0.022, 0.055, 0.044}); } - const double score = option_value_score(frame, y, row, value_reference); - stream.log("[Tutorial settings] " + std::string(SETTINGS[row].label) + " target match=" + std::to_string(score)); - // The reference recording uses inactive pale-red text; the focused live - // row has a thicker dark-red glyph mask. Verified Fast scores 0.734 - // while Slow/Normal score 0. - if (score < 0.65 || (row == AUTOSAVE_ROW && !autosave_off_highlight(frame, y))){ + if (references.value){ + const double score = option_value_score(frame, y, row, *references.value); + stream.log("[Tutorial settings] " + std::string(row.label) + " target match=" + std::to_string(score)); + // The reference recording uses inactive pale-red text; the focused live + // row has a thicker dark-red glyph mask. Verified Fast scores 0.734 + // while Slow/Normal score 0. + if (score < 0.65){ + return false; + } + }else if (!option_column_verified(stream, frame, y, row, screen.position)){ + return false; + } + if (autosave && !autosave_off_highlight(frame, y)){ return false; } - if (row == AUTOSAVE_ROW){ + if (autosave){ confirm_autosave_off(stream, context, screen.options_screen, screen.position, references.label); } return true; } -// Sets the option on `row`, the row selected now, to its target value. -void set_option(VideoStream& stream, ProControllerContext& context, size_t row, const OptionsScreen& screen){ - const std::string base = RESOURCE_PATH() + "PokemonBDSP/Autostory/Options/" + SETTINGS[row].label; - const OptionReferences references{ImageRGB32(base + "_label.png"), ImageRGB32(base + "_value.png")}; +// The reference images `row` is checked with in a layout: label and value +// with label images, else only a slider's number. +OptionReferences option_references(const OptionsRow& row, bool label_images){ + const std::string base = RESOURCE_PATH() + "PokemonBDSP/Autostory/Options/" + std::string(row.label); + OptionReferences references; + if (label_images){ + references.label.emplace(base + "_label.png"); + } + if (label_images || row.slider){ + references.value.emplace(base + "_value.png"); + } + return references; +} + +// Sets the option on `row`, the row selected now, to its target value; a +// row without a value is only confirmed selected. +void set_option(VideoStream& stream, ProControllerContext& context, const OptionsRow& row, bool label_images, const OptionsScreen& screen){ + const OptionReferences references = option_references(row, label_images); bool slider_reset = false; for (size_t attempt = 0; attempt < 30; attempt++){ // Detect the selected row afresh after every navigation/correction. context.wait_for_all_requests(); auto frame = stream.video().snapshot(); - const double y = expected_row_y(stream, frame, screen.options_screen, screen.position, row, references.label); + const double y = expected_row_y(stream, frame, screen, row, references.label); if (y < 0){ // A menu can be detected while its row/text is still fading // in. Re-read without directional inputs first. @@ -368,17 +465,21 @@ void set_option(VideoStream& stream, ProControllerContext& context, size_t row, continue; } OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Settings checkpoint: Unable to confirm expected row " + std::string(SETTINGS[row].label), stream); + "Settings checkpoint: Unable to confirm expected row " + std::string(row.label), stream); + } + if (row.value.empty()){ + stream.log("[Tutorial settings] Left " + std::string(row.label) + " as it is."); + return; } if (option_verified(stream, context, frame, y, row, screen, references)){ - stream.log("[Tutorial settings] Verified " + std::string(SETTINGS[row].label) + "=" + SETTINGS[row].value); + stream.log("[Tutorial settings] Verified " + std::string(row.label) + "=" + std::string(row.value)); return; } - stream.log("[Tutorial settings] Correcting " + std::string(SETTINGS[row].label)); + stream.log("[Tutorial settings] Correcting " + std::string(row.label)); correct_option(context, row, slider_reset); } OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Settings checkpoint: Could not apply " + std::string(SETTINGS[row].label), stream); + "Settings checkpoint: Could not apply " + std::string(row.label), stream); } // Confirms the settings and leaves Options (A A A A B B), back to the overworld. @@ -427,16 +528,20 @@ void reopen_options(VideoStream& stream, ProControllerContext& context, const Tu } // In the saved Options: Autosave is Off on three frames. Options starts -// on Text Speed; Autosave is seven rows below it. +// on Text Speed; Autosave is the layout's rows below it. void check_saved_autosave( - VideoStream& stream, ProControllerContext& context, + VideoStream& stream, ProControllerContext& context, const OptionsLayout& layout, const TutorialOptionsDetector& options_screen, const TutorialOptionPositionDetector& position ){ - for (size_t step = 0; step < AUTOSAVE_ROW; step++){ + size_t autosave_row = 0; + while (autosave_row < layout.rows.size() && layout.rows[autosave_row].label != "autosave"){ + autosave_row++; + } + for (size_t step = 0; step < autosave_row; step++){ pbf_press_dpad(context, DPAD_DOWN, 80ms, 350ms); } context.wait_for_all_requests(); - const ImageRGB32 autosave_label(RESOURCE_PATH() + "PokemonBDSP/Autostory/Options/autosave_label.png"); + const std::optional autosave_label = option_references(layout.rows[autosave_row], layout.label_images).label; for (size_t confirmation = 0; confirmation < 3; confirmation++){ auto check = stream.video().snapshot(); if (!autosave_off_on_screen(check, options_screen, position, autosave_label)){ @@ -453,9 +558,10 @@ void verify_tutorial_settings_once(VideoStream& stream, ProControllerContext& co open_main_menu(stream, context); OptionsScreen screen; open_options(stream, context, screen.options_screen); - for (size_t row = 0; row < SETTING_COUNT; row++){ - set_option(stream, context, row, screen); - if (row + 1 < SETTING_COUNT){ + const OptionsLayout layout = detail::options_layout(story_language()); + for (size_t row = 0; row < layout.rows.size(); row++){ + set_option(stream, context, layout.rows[row], layout.label_images, screen); + if (row + 1 < layout.rows.size()){ pbf_press_dpad(context, DPAD_DOWN, 80ms, 350ms); } } @@ -463,7 +569,7 @@ void verify_tutorial_settings_once(VideoStream& stream, ProControllerContext& co // Reopen the saved Options screen and independently verify Autosave. // No route movement is permitted until this persisted-value check passes. reopen_options(stream, context, screen.options_screen); - check_saved_autosave(stream, context, screen.options_screen, screen.position); + check_saved_autosave(stream, context, layout, screen.options_screen, screen.position); save_options(stream, context, "Autosave verification: Could not return to the overworld."); // Report inferred values explicitly: these are visual matches, // allowing the user to compare our conclusions with the screen. diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h index 206237de07..61dd9df88c 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h @@ -9,6 +9,10 @@ #ifndef PokemonAutomation_PokemonBDSP_AutoStory_Settings_H #define PokemonAutomation_PokemonBDSP_AutoStory_Settings_H +#include +#include +#include +#include "CommonFramework/Language.h" #include "CommonFramework/Tools/VideoStream.h" #include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" @@ -24,6 +28,38 @@ namespace PokemonBDSP{ void verify_tutorial_settings(VideoStream& stream, ProControllerContext& context); +namespace detail{ + +// One row of the Options list, as the story sets it. +struct OptionsRow{ + // The reference images' name: