From 9805a053c5f21ca04b017761e325c88cd29e0dda Mon Sep 17 00:00:00 2001 From: dayfine Date: Mon, 9 Nov 2026 11:48:22 -0800 Subject: [PATCH 1/2] BDSP AutoStory: set Options by position, in any language The settings step matched each row's label and value against English template PNGs that only the lab had, so a French game failed on the first row. Rows are now counted from Text Speed (Options opens on it), each press down confirmed by the outline moving or the list scrolling under it; values are read by which column holds the red highlight. No templates. The Change Window selector and the two volume sliders are left as they are. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 42 ++ .../PokemonBDSP_AutoStory_Settings.cpp | 546 ++++++++---------- .../Actions/PokemonBDSP_AutoStory_Settings.h | 18 + 3 files changed, 314 insertions(+), 292 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index 49a0dd2313..d95b796149 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -24,6 +24,7 @@ #include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" #include "PokemonBDSP_AutoStory_Center.h" #include "PokemonBDSP_AutoStory_NewGame.h" +#include "PokemonBDSP_AutoStory_Settings.h" #include "PokemonBDSP_AutoStory_Actions_Tests.h" namespace PokemonAutomation{ @@ -431,6 +432,43 @@ UnitTestResult test_school_lamp(Logger& logger){ } +// The Options screen of a French game (console B, 2026-11-09; in +// CommandLineTests under PokemonBDSP/AutoStory): read by position and +// colour only, the cursor on Text Speed with Normal (the middle of three) +// selected, and the first value of the two-value rows below it. +UnitTestResult test_options_french(Logger& logger){ + const std::string path = UNIT_TEST_RESOURCE_PATH() + "PokemonBDSP/AutoStory/OptionsFrench-20261109.png"; + if (!Filesystem::is_regular_file(path)){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + } + const ImageRGB32 screen(path); + if (!options_screen_shown(screen)){ + return "Options screen not recognised."; + } + const std::optional y = options_selected_row_y(screen); + if (!y || *y < 0.137 || *y > 0.177){ + return "Selected row not found on the first row."; + } + logger.log("Selected row at y=" + std::to_string(*y)); + if (options_highlighted_column(screen, *y, 3) != size_t(1)){ + return "Text Speed: Normal (column 1 of 3) not read."; + } + // Battle Effects to Gyroscope: rows 2 to 6, the first value highlighted. + for (size_t row = 2; row <= 6; row++){ + const double row_y = *y + 0.0858 * row; + if (options_highlighted_column(screen, row_y, 2) != size_t(0)){ + return "Row " + std::to_string(row) + ": the first of two values not read."; + } + } + ImageRGB32 black(1920, 1080); + black.fill(0xff000000); + if (options_screen_shown(black)){ + return "A black frame read as Options."; + } + return true; +} + + struct AutoStoryActionsInput{ std::string name; std::function body; @@ -506,6 +544,10 @@ const AutoStoryActionsCase AUTO_STORY_ACTIONS_CASES[] = { .input = {"Jubilife School lamp", test_school_lamp}, .expectation = {}, }, + { + .input = {"Options read by position, French", test_options_french}, + .expectation = {}, + }, }; void add_tests_AutoStory_Actions(UnitTestDatabase& database){ diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp index e1f71ad647..82995d0335 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp @@ -7,10 +7,10 @@ */ #include +#include +#include #include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" -#include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/ImageTools/ImageBoxes.h" -#include "CommonFramework/ImageTypes/ImageRGB32.h" #include "CommonFramework/VideoPipeline/VideoFeed.h" #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" #include "CommonTools/Async/InferenceRoutines.h" @@ -30,167 +30,163 @@ using namespace std::chrono_literals; namespace{ -// Options chrome and position detectors are independent of setting values. -class TutorialOptionsDetector { -public: - bool detect(const ImageViewRGB32& screen) const { - ImageViewRGB32 panel = extract_box_reference(screen, ImageFloatBox{0.04, 0.065, 0.93, 0.70}); - size_t white = 0; - for (size_t y = 0; y < panel.height(); y += 4) - for (size_t x = 0; x < panel.width(); x += 4){ - uint32_t p = panel.pixel(x, y); - if (((p >> 16) & 255) > 225 && ((p >> 8) & 255) > 225 && (p & 255) > 225){ - white++; - } - } - double samples = (double)(((panel.width() + 3) / 4) * ((panel.height() + 3) / 4)); - // Distinguish Options from the icon menu and bag using its - // blue heading and full-width white settings panel. - ImageViewRGB32 heading = extract_box_reference(screen, ImageFloatBox{0.01, 0.005, 0.20, 0.025}); - size_t blue = 0, total = 0; - for (size_t y = 0; y < heading.height(); y++) - for (size_t x = 0; x < heading.width(); x++){ - uint32_t p = heading.pixel(x, y); - int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; - blue += b > r + 20 && g > r + 10; - total++; - } - return samples > 0 && white / samples > 0.70 && total > 0 && blue > total / 4; - } -}; +// How red a pixel must be to count as highlighted text or the row outline. +bool is_red(uint32_t pixel, int margin){ + const int r = (pixel >> 16) & 255, g = (pixel >> 8) & 255, b = pixel & 255; + return r > g + margin && r > b + margin; +} -// Options fades in after A on its icon: poll instead of judging one frame. -bool wait_for_options_screen(VideoStream& stream, ProControllerContext& context, - const TutorialOptionsDetector& options_screen, Milliseconds timeout){ - context.wait_for_all_requests(); - const WallClock deadline = current_time() + timeout; - while (current_time() < deadline){ - auto frame = stream.video().snapshot(); - if (frame && options_screen.detect(frame)){ - return true; +// The red pixels in `box` of the screen. +size_t red_pixels(const ImageViewRGB32& screen, const ImageFloatBox& box){ + const ImageViewRGB32 image = extract_box_reference(screen, box); + size_t red = 0; + for (size_t y = 0; y < image.height(); y++){ + for (size_t x = 0; x < image.width(); x++){ + red += is_red(image.pixel(x, y), 30); } - context.wait_for(100ms); } - return false; + return red; } -class TutorialOptionPositionDetector { -public: - // Locate the two long red horizontal lines surrounding the - // selected row. Scrolling changes their position automatically. - double selected_row(const ImageViewRGB32& screen) const { - int first = -1, last = -1; - for (size_t y = (size_t)(screen.height() * 0.08); y < screen.height() * 0.75; y++){ - size_t red = 0, total = 0; - for (size_t x = (size_t)(screen.width() * 0.08); x < screen.width() * 0.90; x += 2){ - uint32_t p = screen.pixel(x, y); - int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; - red += r > 150 && r > g + 40 && r > b + 40; - total++; - } - if (total && red > total * 0.70){ - if (first < 0){ - first = (int)y; - } - last = (int)y; - } +// The share of near-white pixels on the settings panel, sampled every 4th pixel. +double white_share(const ImageViewRGB32& screen){ + const ImageViewRGB32 panel = extract_box_reference(screen, ImageFloatBox{0.04, 0.065, 0.93, 0.70}); + size_t white = 0, samples = 0; + for (size_t y = 0; y < panel.height(); y += 4){ + for (size_t x = 0; x < panel.width(); x += 4){ + const uint32_t p = panel.pixel(x, y); + white += ((p >> 16) & 255) > 225 && ((p >> 8) & 255) > 225 && (p & 255) > 225; + samples++; } - if (first < 0 || last - first < screen.height() * 0.04 || last - first > screen.height() * 0.12){ - return -1; - } - return (first + last) / (2.0 * screen.height()); } - // Locate selected red value text separately from the row border. - bool selected_value(const ImageViewRGB32& screen, double y) const { - ImageViewRGB32 value = extract_box_reference(screen, ImageFloatBox{0.33, y - 0.025, 0.54, 0.05}); - size_t red = 0; - for (size_t py = 0; py < value.height(); py++) - for (size_t px = 0; px < value.width(); px++){ - uint32_t p = value.pixel(px, py); - int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; - red += r > g + 20 && r > b + 20; - } - return red > value.width() * value.height() * 0.0005; + return samples == 0 ? 0 : (double)white / samples; +} + +// The share of blue pixels in the heading bar, which the icon menu and the Bag lack. +double heading_blue_share(const ImageViewRGB32& screen){ + const ImageViewRGB32 heading = extract_box_reference(screen, ImageFloatBox{0.01, 0.005, 0.20, 0.025}); + size_t blue = 0, total = 0; + for (size_t y = 0; y < heading.height(); y++){ + for (size_t x = 0; x < heading.width(); x++){ + const uint32_t p = heading.pixel(x, y); + const int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + blue += b > r + 20 && g > r + 10; + total++; + } } + return total == 0 ? 0 : (double)blue / total; +} + +// Where each row's value columns start and end, as fractions of the screen +// width: the same in every language (values are left-aligned in them). +struct ValueColumn{ + double left; + double right; +}; +const ValueColumn TWO_COLUMNS[] = {{0.335, 0.62}, {0.62, 0.88}}; +const ValueColumn THREE_COLUMNS[] = {{0.335, 0.52}, {0.52, 0.70}, {0.70, 0.88}}; + +// The first row's outline centre when Options opens, and how far it may be off. +constexpr double FIRST_ROW_Y = 0.157; +constexpr double ROW_Y_TOLERANCE = 0.02; +// How far apart rows are on screen. +constexpr double ROW_PITCH = 0.086; + +// The rows the story sets, top to bottom, in the game's fixed order (the +// same in every language). `columns` is the number of value columns, 0 for +// the Change Window selector, which is left as the new game has it (Type 1); +// `target` is the column to select, counted from the left. The two volume +// sliders below Show Nicknames are left as they are. +struct OptionRow{ + const char* name; + size_t columns; + size_t target; }; +const OptionRow OPTION_ROWS[] = { + {"Text Speed", 3, 2}, // Slow, Normal, Fast + {"Change Window", 0, 0}, // < Type 1 > + {"Battle Effects", 2, 1}, // On, Off + {"Battle Style", 2, 1}, // Switch, Set + {"Send to Boxes", 2, 1}, // Manual, Automatic + {"Give Nicknames", 2, 1}, // Give, Don't give + {"Gyroscope", 2, 0}, // Use, Don't use + {"Autosave", 2, 1}, // On, Off + {"Show Nicknames", 2, 0}, // Show, Don't show +}; +constexpr size_t OPTION_ROW_COUNT = sizeof(OPTION_ROWS) / sizeof(OPTION_ROWS[0]); +constexpr size_t AUTOSAVE_ROW = 7; + +} // namespace -// Compare binary visual shapes with small alignment tolerance. -// No text recognition is needed for fixed English option labels. -double option_image_score(const ImageViewRGB32& screen, - const ImageFloatBox& box, const ImageRGB32& reference, bool red_only){ - ImageRGB32 actual = extract_box_reference(screen, box).scale_to(384, 32); - ImageRGB32 expected = reference.scale_to(384, 32); - auto ink = [red_only](uint32_t p){ - int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; - return red_only ? r > g + 20 && r > b + 20 : r + g + b < 500; - }; - double best = 0; - for (int dy = -2; dy <= 2; dy++){ - for (int dx = -2; dx <= 2; dx++){ - size_t overlap = 0, count = 0; - for (int y = 2; y < 30; y++){ - for (int x = 2; x < 382; x++){ - bool a = ink(actual.pixel(x + dx, y + dy)); - bool b = ink(expected.pixel(x, y)); - overlap += a && b; - count += a + b; - } - } - if (count){ - best = std::max(best, 2.0 * overlap / count); +bool options_screen_shown(const ImageViewRGB32& screen){ + return screen && white_share(screen) > 0.70 && heading_blue_share(screen) > 0.25; +} + +std::optional options_selected_row_y(const ImageViewRGB32& screen){ + if (!screen){ + return std::nullopt; + } + // The two long red horizontal lines of the outline around the selected row. + int first = -1, last = -1; + for (size_t y = (size_t)(screen.height() * 0.08); y < screen.height() * 0.75; y++){ + size_t red = 0, total = 0; + for (size_t x = (size_t)(screen.width() * 0.08); x < screen.width() * 0.90; x += 2){ + const uint32_t p = screen.pixel(x, y); + red += ((p >> 16) & 255) > 150 && is_red(p, 40); + total++; + } + if (total && red > total * 0.70){ + if (first < 0){ + first = (int)y; } + last = (int)y; } } - return best; + if (first < 0 || last - first < screen.height() * 0.04 || last - first > screen.height() * 0.12){ + return std::nullopt; + } + return (first + last) / (2.0 * screen.height()); } -bool autosave_off_highlight(const ImageViewRGB32& screen, double y){ - auto red_count = [&](const ImageFloatBox& box){ - ImageViewRGB32 image = extract_box_reference(screen, box); - size_t red = 0; - for (size_t py = 0; py < image.height(); py++) - for (size_t px = 0; px < image.width(); px++){ - uint32_t p = image.pixel(px, py); - int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; - red += r > g + 30 && r > b + 30; - } - return red; - }; - size_t off = red_count({0.63, y - 0.022, 0.055, 0.044}); - size_t on = red_count({0.34, y - 0.022, 0.055, 0.044}); - // Require a clear Off highlight and reject any competing On highlight. - return off >= 10 && on * 10 < off; +std::optional options_highlighted_column(const ImageViewRGB32& screen, double y, size_t columns){ + const ValueColumn* layout = columns == 2 ? TWO_COLUMNS : columns == 3 ? THREE_COLUMNS : nullptr; + if (!screen || layout == nullptr){ + return std::nullopt; + } + // The band stays inside the outline, whose lines are red too. + const double band = 0.022; + std::optional highlighted; + for (size_t c = 0; c < columns; c++){ + const ImageFloatBox box{layout[c].left, y - band, layout[c].right - layout[c].left, 2 * band}; + // A short word such as "Off" has a few hundred red pixels at 1080p. + if (red_pixels(screen, box) * 10000 < screen.width() * screen.height() * 0.5){ + continue; + } + if (highlighted){ + return std::nullopt; + } + highlighted = c; + } + return highlighted; } -// The options the story needs, as each reference image names them: the -// row's label and the value to select. -struct Setting{ - const char* label; - const char* value; -}; -const Setting SETTINGS[] = { - {"textspeed", "fast"}, {"changewindow", "type1"}, - {"battleeffects", "off"}, {"battlestyle", "set"}, - {"sendtoboxes", "automatic"}, {"givenicknames", "dontgive"}, - {"gyroscope", "use"}, {"autosave", "off"}, - {"shownicknames", "show"}, {"backgroundmusic", "8"}, - {"soundeffects", "8"}, -}; -constexpr size_t SETTING_COUNT = sizeof(SETTINGS) / sizeof(SETTINGS[0]); -constexpr size_t AUTOSAVE_ROW = 7; -constexpr size_t FIRST_SLIDER_ROW = 9; -// The Options screen and its selected row, as one reader. -struct OptionsScreen{ - TutorialOptionsDetector options_screen; - TutorialOptionPositionDetector position; -}; -// A row's label and target value, as the reference images show them. -struct OptionReferences{ - ImageRGB32 label; - ImageRGB32 value; -}; +namespace{ + +// Options fades in after A on its icon: poll instead of judging one frame. +bool wait_for_options_screen(VideoStream& stream, ProControllerContext& context, Milliseconds timeout){ + context.wait_for_all_requests(); + const WallClock deadline = current_time() + timeout; + while (current_time() < deadline){ + auto frame = stream.video().snapshot(); + if (frame && options_screen_shown(frame)){ + return true; + } + context.wait_for(100ms); + } + return false; +} // From the overworld (or the main menu): the main menu, settled. void open_main_menu(VideoStream& stream, ProControllerContext& context){ @@ -218,12 +214,12 @@ void open_main_menu(VideoStream& stream, ProControllerContext& context){ // From the main menu: Options. Only two icons are available at this // initial checkpoint; verify the main menu before trying the other. -void open_options(VideoStream& stream, ProControllerContext& context, const TutorialOptionsDetector& options_screen){ +void open_options(VideoStream& stream, ProControllerContext& context){ bool options_open = false; for (size_t attempt = 0; attempt < 2; attempt++){ pbf_press_dpad(context, attempt == 0 ? DPAD_RIGHT : DPAD_LEFT, 80ms, 350ms); pbf_press_button(context, BUTTON_A, 80ms, 0ms); - if (wait_for_options_screen(stream, context, options_screen, 5s)){ + if (wait_for_options_screen(stream, context, 5s)){ options_open = true; break; } @@ -243,141 +239,118 @@ void open_options(VideoStream& stream, ProControllerContext& context, const Tuto context.wait_for_all_requests(); } -// How well the selected row's value matches the reference. -double option_value_score(const ImageViewRGB32& frame, double y, size_t row, const ImageRGB32& value_reference){ - if (row >= FIRST_SLIDER_ROW){ - // Compare the numeric value rather than the long slider: - // neighboring slider levels otherwise look too similar. - ImageViewRGB32 number = value_reference.sub_image( - value_reference.width() * 82 / 100, 0, - value_reference.width() - value_reference.width() * 82 / 100, - value_reference.height()); - ImageRGB32 number_reference = number.scale_to(96, 32); - return option_image_score(frame, - {0.7728, y - 0.025, 0.0972, 0.05}, number_reference, true); +// The selected row's centre on a fresh frame of Options, retried while the +// screen fades in; fails with a screenshot after about a second. +double read_selected_row(VideoStream& stream, ProControllerContext& context, size_t row){ + for (size_t attempt = 0; attempt < 6; attempt++){ + context.wait_for_all_requests(); + const auto frame = stream.video().snapshot(); + const std::optional y = frame && options_screen_shown(frame) ? options_selected_row_y(frame) : std::nullopt; + if (y){ + return *y; + } + pbf_wait(context, 200ms); } - return option_image_score(frame, - {0.33, y - 0.025, 0.54, 0.05}, value_reference, true); + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Settings checkpoint: No selected row on Options for " + std::string(OPTION_ROWS[row].name), stream); } -// Whether Options shows Autosave selected and set to Off: a frame's look. -bool autosave_off_on_screen( - const ImageViewRGB32& check, const TutorialOptionsDetector& options_screen, - const TutorialOptionPositionDetector& position, const ImageRGB32& autosave_label -){ - const double check_y = check ? position.selected_row(check) : -1; - return check && options_screen.detect(check) && check_y >= 0 && - option_image_score(check, {0.08, check_y - 0.025, 0.24, 0.05}, autosave_label, false) >= 0.80 && - autosave_off_highlight(check, check_y); +// Options opens on its first row (Text Speed), at the top of the list: +// that row's centre. +double check_first_row(VideoStream& stream, ProControllerContext& context){ + const double y = read_selected_row(stream, context, 0); + stream.log("[Tutorial settings] First row at y=" + std::to_string(y)); + if (y < FIRST_ROW_Y - ROW_Y_TOLERANCE || y > FIRST_ROW_Y + ROW_Y_TOLERANCE){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Settings checkpoint: Options did not open on its first row.", stream); + } + return y; } -// Autosave Off, on three more frames 200 ms apart. -void confirm_autosave_off( - VideoStream& stream, ProControllerContext& context, const TutorialOptionsDetector& options_screen, - const TutorialOptionPositionDetector& position, const ImageRGB32& label_reference -){ - for (size_t confirmation = 0; confirmation < 3; confirmation++){ - pbf_wait(context, 200ms); - context.wait_for_all_requests(); - auto check = stream.video().snapshot(); - if (!autosave_off_on_screen(check, options_screen, position, label_reference)){ - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Autosave Off could not be confirmed. Stopping before movement.", stream); +// The mean grey-level difference of two frames over `box`, sampled every 2nd pixel. +double box_difference(const ImageViewRGB32& a, const ImageViewRGB32& b, const ImageFloatBox& box){ + const ImageViewRGB32 x = extract_box_reference(a, box); + const ImageViewRGB32 z = extract_box_reference(b, box); + const size_t width = std::min(x.width(), z.width()), height = std::min(x.height(), z.height()); + double sum = 0; + size_t count = 0; + for (size_t py = 0; py < height; py += 2){ + for (size_t px = 0; px < width; px += 2){ + const uint32_t p = x.pixel(px, py), q = z.pixel(px, py); + const int gp = ((p >> 16) & 255) + ((p >> 8) & 255) + (p & 255); + const int gq = ((q >> 16) & 255) + ((q >> 8) & 255) + (q & 255); + sum += std::abs(gp - gq) / 3.0; + count++; } } + return count == 0 ? 0 : sum / count; } -// One press toward the target value. Sliders clamp at zero: establish a -// known endpoint, then increment; the target image is verified afterward. -void correct_option(ProControllerContext& context, size_t row, bool& slider_reset){ - if (row >= FIRST_SLIDER_ROW && !slider_reset){ - for (size_t step = 0; step < 12; step++){ - pbf_press_dpad(context, DPAD_LEFT, 80ms, 100ms); +// From row `row` (outline at `y`): one press down, confirmed by the outline +// moving a row down, or, at the bottom of the view, by the list scrolling +// under it (the label under the outline changes). Never presses twice, so +// a row is never skipped. Returns the new row's centre. +double move_down(VideoStream& stream, ProControllerContext& context, size_t row, double y){ + context.wait_for_all_requests(); + const auto before = stream.video().snapshot(); + pbf_press_dpad(context, DPAD_DOWN, 80ms, 350ms); + context.wait_for_all_requests(); + const ImageFloatBox label{0.08, y - 0.025, 0.24, 0.05}; + for (size_t check = 0; check < 15; check++){ + const auto after = stream.video().snapshot(); + const std::optional now = after && options_screen_shown(after) ? options_selected_row_y(after) : std::nullopt; + if (now && *now > y + ROW_PITCH / 2 && *now < y + ROW_PITCH * 1.5){ + return *now; } - slider_reset = true; - }else{ - pbf_press_dpad(context, DPAD_RIGHT, 80ms, 350ms); + if (now && std::abs(*now - y) < ROW_PITCH / 4 && before && box_difference(before, after, label) > 8){ + return *now; + } + context.wait_for(100ms); } - context.wait_for_all_requests(); + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Settings checkpoint: Cursor did not move down from " + std::string(OPTION_ROWS[row].name), stream); } -// The selected row on screen, if it is the expected one with its value -// highlighted; -1 otherwise. -double expected_row_y( - VideoStream& stream, const ImageViewRGB32& frame, const TutorialOptionsDetector& options_screen, - const TutorialOptionPositionDetector& position, size_t row, const ImageRGB32& label_reference -){ - const double y = frame && options_screen.detect(frame) ? position.selected_row(frame) : -1; - const bool value_visible = y >= 0 && position.selected_value(frame, y); - const double label_score = y >= 0 ? option_image_score(frame, - {0.08, y - 0.025, 0.24, 0.05}, label_reference, false) : 0; - stream.log("[Tutorial settings] Expected row=" + std::string(SETTINGS[row].label) + - " row_y=" + std::to_string(y) + " highlighted_value=" + std::to_string(value_visible) + - " label_match=" + std::to_string(label_score)); - return y >= 0 && value_visible && label_score >= 0.80 ? y : -1; +// The highlighted column of `row` (outline at `y`) on a fresh frame, or none. +std::optional read_column(VideoStream& stream, ProControllerContext& context, size_t row, double y){ + context.wait_for_all_requests(); + const auto frame = stream.video().snapshot(); + const std::optional column = frame ? options_highlighted_column(frame, y, OPTION_ROWS[row].columns) : std::nullopt; + stream.log("[Tutorial settings] Row " + std::to_string(row) + " (" + OPTION_ROWS[row].name + ") y=" + + std::to_string(y) + " column=" + (column ? std::to_string(*column) : "none") + + " target=" + std::to_string(OPTION_ROWS[row].target)); + return column; } -// Whether the selected row (at `y`) shows the target value: checked on -// the frame, with the row's boxes on the overlay. -bool option_verified( - VideoStream& stream, ProControllerContext& context, const ImageViewRGB32& frame, double y, size_t row, - const OptionsScreen& screen, const OptionReferences& references -){ - const ImageRGB32& value_reference = references.value; - // These are regular video-overlay boxes, so the existing overlay - // display control can show/hide every active ROI. - OverlayBoxScope row_overlay(stream.overlay(), COLOR_RED, {0.07, y - 0.04, 0.84, 0.08}, ""); - OverlayBoxScope label_overlay(stream.overlay(), COLOR_YELLOW, {0.08, y - 0.025, 0.24, 0.05}, ""); - OverlayBoxScope value_overlay(stream.overlay(), COLOR_GREEN, {0.33, y - 0.025, 0.54, 0.05}, ""); - VideoOverlaySet autosave_rois(stream.overlay()); - if (row == AUTOSAVE_ROW){ - autosave_rois.add(COLOR_GREEN, {0.63, y - 0.022, 0.055, 0.044}); - autosave_rois.add(COLOR_RED, {0.34, y - 0.022, 0.055, 0.044}); - } - const double score = option_value_score(frame, y, row, value_reference); - stream.log("[Tutorial settings] " + std::string(SETTINGS[row].label) + " target match=" + std::to_string(score)); - // The reference recording uses inactive pale-red text; the focused live - // row has a thicker dark-red glyph mask. Verified Fast scores 0.734 - // while Slow/Normal score 0. - if (score < 0.65 || (row == AUTOSAVE_ROW && !autosave_off_highlight(frame, y))){ - return false; - } - if (row == AUTOSAVE_ROW){ - confirm_autosave_off(stream, context, screen.options_screen, screen.position, references.label); +// Sets the selected row `row` (outline at `y`) to its target column, a +// press at a time toward it, reading the highlight after each press. +void set_option(VideoStream& stream, ProControllerContext& context, size_t row, double y){ + const OptionRow& option = OPTION_ROWS[row]; + if (option.columns == 0){ + stream.log("[Tutorial settings] Row " + std::to_string(row) + " (" + option.name + ") left as it is."); + return; } - return true; -} - -// Sets the option on `row`, the row selected now, to its target value. -void set_option(VideoStream& stream, ProControllerContext& context, size_t row, const OptionsScreen& screen){ - const std::string base = RESOURCE_PATH() + "PokemonBDSP/Autostory/Options/" + SETTINGS[row].label; - const OptionReferences references{ImageRGB32(base + "_label.png"), ImageRGB32(base + "_value.png")}; - bool slider_reset = false; - for (size_t attempt = 0; attempt < 30; attempt++){ - // Detect the selected row afresh after every navigation/correction. - context.wait_for_all_requests(); - auto frame = stream.video().snapshot(); - const double y = expected_row_y(stream, frame, screen.options_screen, screen.position, row, references.label); - if (y < 0){ - // A menu can be detected while its row/text is still fading - // in. Re-read without directional inputs first. - if (attempt < 4){ - pbf_wait(context, 200ms); - context.wait_for_all_requests(); - continue; + OverlayBoxScope row_overlay(stream.overlay(), COLOR_RED, {0.07, y - 0.04, 0.84, 0.08}, ""); + OverlayBoxScope value_overlay(stream.overlay(), COLOR_GREEN, {0.335, y - 0.022, 0.545, 0.044}, ""); + size_t unread = 0; + for (size_t attempt = 0; attempt < 12; attempt++){ + const std::optional column = read_column(stream, context, row, y); + if (!column){ + // The highlight can still be fading in after a press. + if (++unread > 4){ + break; } - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Settings checkpoint: Unable to confirm expected row " + std::string(SETTINGS[row].label), stream); + pbf_wait(context, 200ms); + continue; } - if (option_verified(stream, context, frame, y, row, screen, references)){ - stream.log("[Tutorial settings] Verified " + std::string(SETTINGS[row].label) + "=" + SETTINGS[row].value); + if (*column == option.target){ + stream.log("[Tutorial settings] Verified " + std::string(option.name)); return; } - stream.log("[Tutorial settings] Correcting " + std::string(SETTINGS[row].label)); - correct_option(context, row, slider_reset); + pbf_press_dpad(context, *column < option.target ? DPAD_RIGHT : DPAD_LEFT, 80ms, 350ms); } OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Settings checkpoint: Could not apply " + std::string(SETTINGS[row].label), stream); + "Settings checkpoint: Could not apply " + std::string(option.name), stream); } // Confirms the settings and leaves Options (A A A A B B), back to the overworld. @@ -396,7 +369,7 @@ void save_options(VideoStream& stream, ProControllerContext& context, const std: } // From the overworld: the saved Options screen again. -void reopen_options(VideoStream& stream, ProControllerContext& context, const TutorialOptionsDetector& options_screen){ +void reopen_options(VideoStream& stream, ProControllerContext& context){ pbf_press_button(context, BUTTON_X, 80ms, 1000ms); context.wait_for_all_requests(); MenuWatcher verify_menu(COLOR_RED, /*skip_new_banner*/ true); @@ -407,7 +380,7 @@ void reopen_options(VideoStream& stream, ProControllerContext& context, const Tu bool reopened = false; for (size_t attempt = 0; attempt < 2; attempt++){ pbf_press_button(context, BUTTON_A, 80ms, 0ms); - if (wait_for_options_screen(stream, context, options_screen, 5s)){ + if (wait_for_options_screen(stream, context, 5s)){ reopened = true; break; } @@ -425,51 +398,41 @@ void reopen_options(VideoStream& stream, ProControllerContext& context, const Tu } } -// In the saved Options: Autosave is Off on three frames. Options starts -// on Text Speed; Autosave is seven rows below it. -void check_saved_autosave( - VideoStream& stream, ProControllerContext& context, - const TutorialOptionsDetector& options_screen, const TutorialOptionPositionDetector& position -){ - for (size_t step = 0; step < AUTOSAVE_ROW; step++){ - pbf_press_dpad(context, DPAD_DOWN, 80ms, 350ms); +// In the saved Options: down to Autosave, a confirmed row at a time, and +// Off highlighted on three frames 200 ms apart. +void check_saved_autosave(VideoStream& stream, ProControllerContext& context){ + double y = check_first_row(stream, context); + for (size_t row = 0; row < AUTOSAVE_ROW; row++){ + y = move_down(stream, context, row, y); } - context.wait_for_all_requests(); - const ImageRGB32 autosave_label(RESOURCE_PATH() + "PokemonBDSP/Autostory/Options/autosave_label.png"); for (size_t confirmation = 0; confirmation < 3; confirmation++){ - auto check = stream.video().snapshot(); - if (!autosave_off_on_screen(check, options_screen, position, autosave_label)){ + if (read_column(stream, context, AUTOSAVE_ROW, y) != OPTION_ROWS[AUTOSAVE_ROW].target){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Saved Autosave setting is not verified Off. Stopping before movement.", stream); } pbf_wait(context, 200ms); - context.wait_for_all_requests(); } stream.log("[Tutorial settings] Autosave = Off: confirmed after reopening saved Options (3 frames)."); -} // namespace +} void verify_tutorial_settings_once(VideoStream& stream, ProControllerContext& context){ open_main_menu(stream, context); - OptionsScreen screen; - open_options(stream, context, screen.options_screen); - for (size_t row = 0; row < SETTING_COUNT; row++){ - set_option(stream, context, row, screen); - if (row + 1 < SETTING_COUNT){ - pbf_press_dpad(context, DPAD_DOWN, 80ms, 350ms); + open_options(stream, context); + double y = check_first_row(stream, context); + for (size_t row = 0; row < OPTION_ROW_COUNT; row++){ + set_option(stream, context, row, y); + if (row + 1 < OPTION_ROW_COUNT){ + y = move_down(stream, context, row, y); } } save_options(stream, context, "Settings checkpoint: Did not return to the overworld after saving options."); // Reopen the saved Options screen and independently verify Autosave. // No route movement is permitted until this persisted-value check passes. - reopen_options(stream, context, screen.options_screen); - check_saved_autosave(stream, context, screen.options_screen, screen.position); + reopen_options(stream, context); + check_saved_autosave(stream, context); save_options(stream, context, "Autosave verification: Could not return to the overworld."); - // Report inferred values explicitly: these are visual matches, - // allowing the user to compare our conclusions with the screen. - stream.log("[Tutorial settings] Completed. Values believed applied (visual matches before saving):"); - stream.log("[Tutorial settings] Text Speed = Fast; Change Window = Type 1; Battle Effects = Off; Battle Style = Set."); - stream.log("[Tutorial settings] Send to Boxes = Automatic; Give Nicknames = Don't give; Gyroscope = Use."); - stream.log("[Tutorial settings] Autosave = Off; Show Nicknames = Show; Background Music = 8; Sound Effects = 8."); + stream.log("[Tutorial settings] Completed: Text Speed Fast, Battle Effects Off, Battle Style Set, " + "Send to Boxes Automatic, Give Nicknames Don't give, Gyroscope Use, Autosave Off, Show Nicknames Show."); } // After a failed attempt: back out of Options, save prompts, the Bag or @@ -477,9 +440,8 @@ void verify_tutorial_settings_once(VideoStream& stream, ProControllerContext& co void back_to_overworld(VideoStream& stream, ProControllerContext& context){ stream.log("[Tutorial settings] Returning to overworld before retry."); context.wait_for_all_requests(); - TutorialOptionsDetector options_screen; const auto failed_frame = stream.video().snapshot(); - if (failed_frame && options_screen.detect(failed_frame)){ + if (failed_frame && options_screen_shown(failed_frame)){ // Finish the known Options save/exit sequence before observing // state; never cancel it on an Options frame misclassified as // overworld by the generic detector. @@ -496,7 +458,7 @@ void back_to_overworld(VideoStream& stream, ProControllerContext& context){ size_t confirmations = 0; for (size_t check = 0; check < 15 && !recovered; check++){ const auto frame = stream.video().snapshot(); - const bool clear = frame && !options_screen.detect(frame) && + const bool clear = frame && !options_screen_shown(frame) && !menu.detect(frame) && !black.detect(frame) && bag_pocket_position(frame) == 0 && overworld.detect(frame); if (clear && ++confirmations >= 3){ recovered = true; diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h index 206237de07..7aa388b7e6 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h @@ -9,6 +9,8 @@ #ifndef PokemonAutomation_PokemonBDSP_AutoStory_Settings_H #define PokemonAutomation_PokemonBDSP_AutoStory_Settings_H +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" #include "CommonFramework/Tools/VideoStream.h" #include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" @@ -17,6 +19,22 @@ namespace NintendoSwitch{ namespace PokemonBDSP{ +// The Options screen, read from its layout and colours only, so that it +// reads the same in every game language. + +// Whether the frame shows Options: the blue heading bar over the white panel. +bool options_screen_shown(const ImageViewRGB32& screen); + +// The centre of the selected row (its red outline), as a fraction of the +// screen height; none without exactly one outlined row. +std::optional options_selected_row_y(const ImageViewRGB32& screen); + +// Which of the `columns` value columns (2 or 3) of the row centred at `y` +// holds the highlighted (red) value, counted from the left; none when no +// column or more than one is highlighted. +std::optional options_highlighted_column(const ImageViewRGB32& screen, double y, size_t columns); + + // From the overworld: opens Options and sets what the story needs (fast // text, Send to Boxes automatic, no nickname prompt, Autosave off), saves // them and returns to the overworld. Tries four times, backing out to the From c35fbfc6f84baec604be5189ec5a72ca19b7866f Mon Sep 17 00:00:00 2001 From: dayfine Date: Tue, 10 Nov 2026 12:44:58 -0800 Subject: [PATCH 2/2] AutoStory (BDSP): adopt std::string_view for option row names --- .../AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp index 82995d0335..61a1517fa3 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp @@ -98,7 +98,7 @@ constexpr double ROW_PITCH = 0.086; // `target` is the column to select, counted from the left. The two volume // sliders below Show Nicknames are left as they are. struct OptionRow{ - const char* name; + std::string_view name; size_t columns; size_t target; }; @@ -316,7 +316,7 @@ std::optional read_column(VideoStream& stream, ProControllerContext& con context.wait_for_all_requests(); const auto frame = stream.video().snapshot(); const std::optional column = frame ? options_highlighted_column(frame, y, OPTION_ROWS[row].columns) : std::nullopt; - stream.log("[Tutorial settings] Row " + std::to_string(row) + " (" + OPTION_ROWS[row].name + ") y=" + + stream.log("[Tutorial settings] Row " + std::to_string(row) + " (" + std::string(OPTION_ROWS[row].name) + ") y=" + std::to_string(y) + " column=" + (column ? std::to_string(*column) : "none") + " target=" + std::to_string(OPTION_ROWS[row].target)); return column; @@ -327,7 +327,7 @@ std::optional read_column(VideoStream& stream, ProControllerContext& con void set_option(VideoStream& stream, ProControllerContext& context, size_t row, double y){ const OptionRow& option = OPTION_ROWS[row]; if (option.columns == 0){ - stream.log("[Tutorial settings] Row " + std::to_string(row) + " (" + option.name + ") left as it is."); + stream.log("[Tutorial settings] Row " + std::to_string(row) + " (" + std::string(option.name) + ") left as it is."); return; } OverlayBoxScope row_overlay(stream.overlay(), COLOR_RED, {0.07, y - 0.04, 0.84, 0.08}, "");