diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index 446a771ed8..6f3459c3bc 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -30,6 +30,7 @@ #include "PokemonBDSP_AutoStory_IntroSteps.h" #include "PokemonBDSP_AutoStory_LitDoorway.h" #include "PokemonBDSP_AutoStory_NewGame.h" +#include "PokemonBDSP_AutoStory_Settings.h" #include "PokemonBDSP_AutoStory_Actions_Tests.h" namespace PokemonAutomation{ @@ -437,6 +438,42 @@ UnitTestResult test_school_lamp(Logger& logger){ } +// The Options screen of a French game (console B, 2026-11-09; in +// CommandLineTests under PokemonBDSP/AutoStory): read by position and +// colour only, the cursor on Text Speed with Normal (the middle of three) +// selected, and the first value of the two-value rows below it. +UnitTestResult test_options_french(Logger& logger){ + const std::string path = UNIT_TEST_RESOURCE_PATH() + "PokemonBDSP/AutoStory/OptionsFrench-20261109.png"; + if (!Filesystem::is_regular_file(path)){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + } + const ImageRGB32 screen(path); + if (!options_screen_shown(screen)){ + return "Options screen not recognised."; + } + const std::optional y = options_selected_row_y(screen); + if (!y || *y < 0.137 || *y > 0.177){ + return "Selected row not found on the first row."; + } + logger.log("Selected row at y=" + std::to_string(*y)); + if (options_highlighted_column(screen, *y, 3) != size_t(1)){ + return "Text Speed: Normal (column 1 of 3) not read."; + } + // Battle Effects to Gyroscope: rows 2 to 6, the first value highlighted. + for (size_t row = 2; row <= 6; row++){ + const double row_y = *y + 0.0858 * row; + if (options_highlighted_column(screen, row_y, 2) != size_t(0)){ + return "Row " + std::to_string(row) + ": the first of two values not read."; + } + } + ImageRGB32 black(1920, 1080); + black.fill(0xff000000); + if (options_screen_shown(black)){ + return "A black frame read as Options."; + } + return true; +} + // The move picked from names and PP as read. lab-B 2026-11-09, French: // no name read, Piplup's Pound at 35 PP and Growl at 22 PP, and the old // "newest move that is not a status move" fallback used Growl until the @@ -586,6 +623,10 @@ const AutoStoryActionsCase AUTO_STORY_ACTIONS_CASES[] = { .input = {"Jubilife School lamp", test_school_lamp}, .expectation = {}, }, + { + .input = {"Options read by position, French", test_options_french}, + .expectation = {}, + }, { .input = {"battle moves: the pick from names and PP", test_choose_move}, .expectation = {}, diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp index bf3540c9c8..ec14e2a0d8 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp @@ -7,10 +7,10 @@ */ #include +#include +#include #include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" -#include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/ImageTools/ImageBoxes.h" -#include "CommonFramework/ImageTypes/ImageRGB32.h" #include "CommonFramework/VideoPipeline/VideoFeed.h" #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" #include "CommonTools/Async/InferenceRoutines.h" @@ -31,167 +31,163 @@ using namespace std::chrono_literals; namespace{ -// Options chrome and position detectors are independent of setting values. -class TutorialOptionsDetector { -public: - bool detect(const ImageViewRGB32& screen) const { - ImageViewRGB32 panel = extract_box_reference(screen, ImageFloatBox{0.04, 0.065, 0.93, 0.70}); - size_t white = 0; - for (size_t y = 0; y < panel.height(); y += 4) - for (size_t x = 0; x < panel.width(); x += 4){ - uint32_t p = panel.pixel(x, y); - if (((p >> 16) & 255) > 225 && ((p >> 8) & 255) > 225 && (p & 255) > 225){ - white++; - } - } - double samples = (double)(((panel.width() + 3) / 4) * ((panel.height() + 3) / 4)); - // Distinguish Options from the icon menu and bag using its - // blue heading and full-width white settings panel. - ImageViewRGB32 heading = extract_box_reference(screen, ImageFloatBox{0.01, 0.005, 0.20, 0.025}); - size_t blue = 0, total = 0; - for (size_t y = 0; y < heading.height(); y++) - for (size_t x = 0; x < heading.width(); x++){ - uint32_t p = heading.pixel(x, y); - int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; - blue += b > r + 20 && g > r + 10; - total++; - } - return samples > 0 && white / samples > 0.70 && total > 0 && blue > total / 4; - } -}; +// How red a pixel must be to count as highlighted text or the row outline. +bool is_red(uint32_t pixel, int margin){ + const int r = (pixel >> 16) & 255, g = (pixel >> 8) & 255, b = pixel & 255; + return r > g + margin && r > b + margin; +} -// Options fades in after A on its icon: poll instead of judging one frame. -bool wait_for_options_screen(VideoStream& stream, ProControllerContext& context, - const TutorialOptionsDetector& options_screen, Milliseconds timeout){ - context.wait_for_all_requests(); - const WallClock deadline = current_time() + timeout; - while (current_time() < deadline){ - auto frame = stream.video().snapshot(); - if (frame && options_screen.detect(frame)){ - return true; +// The red pixels in `box` of the screen. +size_t red_pixels(const ImageViewRGB32& screen, const ImageFloatBox& box){ + const ImageViewRGB32 image = extract_box_reference(screen, box); + size_t red = 0; + for (size_t y = 0; y < image.height(); y++){ + for (size_t x = 0; x < image.width(); x++){ + red += is_red(image.pixel(x, y), 30); } - context.wait_for(100ms); } - return false; + return red; } -class TutorialOptionPositionDetector { -public: - // Locate the two long red horizontal lines surrounding the - // selected row. Scrolling changes their position automatically. - double selected_row(const ImageViewRGB32& screen) const { - int first = -1, last = -1; - for (size_t y = (size_t)(screen.height() * 0.08); y < screen.height() * 0.75; y++){ - size_t red = 0, total = 0; - for (size_t x = (size_t)(screen.width() * 0.08); x < screen.width() * 0.90; x += 2){ - uint32_t p = screen.pixel(x, y); - int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; - red += r > 150 && r > g + 40 && r > b + 40; - total++; - } - if (total && red > total * 0.70){ - if (first < 0){ - first = (int)y; - } - last = (int)y; - } +// The share of near-white pixels on the settings panel, sampled every 4th pixel. +double white_share(const ImageViewRGB32& screen){ + const ImageViewRGB32 panel = extract_box_reference(screen, ImageFloatBox{0.04, 0.065, 0.93, 0.70}); + size_t white = 0, samples = 0; + for (size_t y = 0; y < panel.height(); y += 4){ + for (size_t x = 0; x < panel.width(); x += 4){ + const uint32_t p = panel.pixel(x, y); + white += ((p >> 16) & 255) > 225 && ((p >> 8) & 255) > 225 && (p & 255) > 225; + samples++; } - if (first < 0 || last - first < screen.height() * 0.04 || last - first > screen.height() * 0.12){ - return -1; - } - return (first + last) / (2.0 * screen.height()); } - // Locate selected red value text separately from the row border. - bool selected_value(const ImageViewRGB32& screen, double y) const { - ImageViewRGB32 value = extract_box_reference(screen, ImageFloatBox{0.33, y - 0.025, 0.54, 0.05}); - size_t red = 0; - for (size_t py = 0; py < value.height(); py++) - for (size_t px = 0; px < value.width(); px++){ - uint32_t p = value.pixel(px, py); - int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; - red += r > g + 20 && r > b + 20; - } - return red > value.width() * value.height() * 0.0005; + return samples == 0 ? 0 : (double)white / samples; +} + +// The share of blue pixels in the heading bar, which the icon menu and the Bag lack. +double heading_blue_share(const ImageViewRGB32& screen){ + const ImageViewRGB32 heading = extract_box_reference(screen, ImageFloatBox{0.01, 0.005, 0.20, 0.025}); + size_t blue = 0, total = 0; + for (size_t y = 0; y < heading.height(); y++){ + for (size_t x = 0; x < heading.width(); x++){ + const uint32_t p = heading.pixel(x, y); + const int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + blue += b > r + 20 && g > r + 10; + total++; + } } + return total == 0 ? 0 : (double)blue / total; +} + +// Where each row's value columns start and end, as fractions of the screen +// width: the same in every language (values are left-aligned in them). +struct ValueColumn{ + double left; + double right; +}; +const ValueColumn TWO_COLUMNS[] = {{0.335, 0.62}, {0.62, 0.88}}; +const ValueColumn THREE_COLUMNS[] = {{0.335, 0.52}, {0.52, 0.70}, {0.70, 0.88}}; + +// The first row's outline centre when Options opens, and how far it may be off. +constexpr double FIRST_ROW_Y = 0.157; +constexpr double ROW_Y_TOLERANCE = 0.02; +// How far apart rows are on screen. +constexpr double ROW_PITCH = 0.086; + +// The rows the story sets, top to bottom, in the game's fixed order (the +// same in every language). `columns` is the number of value columns, 0 for +// the Change Window selector, which is left as the new game has it (Type 1); +// `target` is the column to select, counted from the left. The two volume +// sliders below Show Nicknames are left as they are. +struct OptionRow{ + std::string_view name; + size_t columns; + size_t target; }; +const OptionRow OPTION_ROWS[] = { + {"Text Speed", 3, 2}, // Slow, Normal, Fast + {"Change Window", 0, 0}, // < Type 1 > + {"Battle Effects", 2, 1}, // On, Off + {"Battle Style", 2, 1}, // Switch, Set + {"Send to Boxes", 2, 1}, // Manual, Automatic + {"Give Nicknames", 2, 1}, // Give, Don't give + {"Gyroscope", 2, 0}, // Use, Don't use + {"Autosave", 2, 1}, // On, Off + {"Show Nicknames", 2, 0}, // Show, Don't show +}; +constexpr size_t OPTION_ROW_COUNT = sizeof(OPTION_ROWS) / sizeof(OPTION_ROWS[0]); +constexpr size_t AUTOSAVE_ROW = 7; + +} // namespace -// Compare binary visual shapes with small alignment tolerance. -// No text recognition is needed for fixed English option labels. -double option_image_score(const ImageViewRGB32& screen, - const ImageFloatBox& box, const ImageRGB32& reference, bool red_only){ - ImageRGB32 actual = extract_box_reference(screen, box).scale_to(384, 32); - ImageRGB32 expected = reference.scale_to(384, 32); - auto ink = [red_only](uint32_t p){ - int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; - return red_only ? r > g + 20 && r > b + 20 : r + g + b < 500; - }; - double best = 0; - for (int dy = -2; dy <= 2; dy++){ - for (int dx = -2; dx <= 2; dx++){ - size_t overlap = 0, count = 0; - for (int y = 2; y < 30; y++){ - for (int x = 2; x < 382; x++){ - bool a = ink(actual.pixel(x + dx, y + dy)); - bool b = ink(expected.pixel(x, y)); - overlap += a && b; - count += a + b; - } - } - if (count){ - best = std::max(best, 2.0 * overlap / count); +bool options_screen_shown(const ImageViewRGB32& screen){ + return screen && white_share(screen) > 0.70 && heading_blue_share(screen) > 0.25; +} + +std::optional options_selected_row_y(const ImageViewRGB32& screen){ + if (!screen){ + return std::nullopt; + } + // The two long red horizontal lines of the outline around the selected row. + int first = -1, last = -1; + for (size_t y = (size_t)(screen.height() * 0.08); y < screen.height() * 0.75; y++){ + size_t red = 0, total = 0; + for (size_t x = (size_t)(screen.width() * 0.08); x < screen.width() * 0.90; x += 2){ + const uint32_t p = screen.pixel(x, y); + red += ((p >> 16) & 255) > 150 && is_red(p, 40); + total++; + } + if (total && red > total * 0.70){ + if (first < 0){ + first = (int)y; } + last = (int)y; } } - return best; + if (first < 0 || last - first < screen.height() * 0.04 || last - first > screen.height() * 0.12){ + return std::nullopt; + } + return (first + last) / (2.0 * screen.height()); } -bool autosave_off_highlight(const ImageViewRGB32& screen, double y){ - auto red_count = [&](const ImageFloatBox& box){ - ImageViewRGB32 image = extract_box_reference(screen, box); - size_t red = 0; - for (size_t py = 0; py < image.height(); py++) - for (size_t px = 0; px < image.width(); px++){ - uint32_t p = image.pixel(px, py); - int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; - red += r > g + 30 && r > b + 30; - } - return red; - }; - size_t off = red_count({0.63, y - 0.022, 0.055, 0.044}); - size_t on = red_count({0.34, y - 0.022, 0.055, 0.044}); - // Require a clear Off highlight and reject any competing On highlight. - return off >= 10 && on * 10 < off; +std::optional options_highlighted_column(const ImageViewRGB32& screen, double y, size_t columns){ + const ValueColumn* layout = columns == 2 ? TWO_COLUMNS : columns == 3 ? THREE_COLUMNS : nullptr; + if (!screen || layout == nullptr){ + return std::nullopt; + } + // The band stays inside the outline, whose lines are red too. + const double band = 0.022; + std::optional highlighted; + for (size_t c = 0; c < columns; c++){ + const ImageFloatBox box{layout[c].left, y - band, layout[c].right - layout[c].left, 2 * band}; + // A short word such as "Off" has a few hundred red pixels at 1080p. + if (red_pixels(screen, box) * 10000 < screen.width() * screen.height() * 0.5){ + continue; + } + if (highlighted){ + return std::nullopt; + } + highlighted = c; + } + return highlighted; } -// The options the story needs, as each reference image names them: the -// row's label and the value to select. -struct Setting{ - std::string label; - std::string value; -}; -const Setting SETTINGS[] = { - {"textspeed", "fast"}, {"changewindow", "type1"}, - {"battleeffects", "off"}, {"battlestyle", "set"}, - {"sendtoboxes", "automatic"}, {"givenicknames", "dontgive"}, - {"gyroscope", "use"}, {"autosave", "off"}, - {"shownicknames", "show"}, {"backgroundmusic", "8"}, - {"soundeffects", "8"}, -}; -constexpr size_t SETTING_COUNT = sizeof(SETTINGS) / sizeof(SETTINGS[0]); -constexpr size_t AUTOSAVE_ROW = 7; -constexpr size_t FIRST_SLIDER_ROW = 9; -// The Options screen and its selected row, as one reader. -struct OptionsScreen{ - TutorialOptionsDetector options_screen; - TutorialOptionPositionDetector position; -}; -// A row's label and target value, as the reference images show them. -struct OptionReferences{ - ImageRGB32 label; - ImageRGB32 value; -}; +namespace{ + +// Options fades in after A on its icon: poll instead of judging one frame. +bool wait_for_options_screen(VideoStream& stream, ProControllerContext& context, Milliseconds timeout){ + context.wait_for_all_requests(); + const WallClock deadline = current_time() + timeout; + while (current_time() < deadline){ + auto frame = stream.video().snapshot(); + if (frame && options_screen_shown(frame)){ + return true; + } + context.wait_for(100ms); + } + return false; +} // From the overworld (or the main menu): the main menu, settled. void open_main_menu(VideoStream& stream, ProControllerContext& context){ @@ -219,12 +215,12 @@ void open_main_menu(VideoStream& stream, ProControllerContext& context){ // From the main menu: Options. Only two icons are available at this // initial checkpoint; verify the main menu before trying the other. -void open_options(VideoStream& stream, ProControllerContext& context, const TutorialOptionsDetector& options_screen){ +void open_options(VideoStream& stream, ProControllerContext& context){ bool options_open = false; for (size_t attempt = 0; attempt < 2; attempt++){ pbf_press_dpad(context, attempt == 0 ? DPAD_RIGHT : DPAD_LEFT, 80ms, 350ms); pbf_press_button(context, BUTTON_A, 80ms, 0ms); - if (wait_for_options_screen(stream, context, options_screen, 5s)){ + if (wait_for_options_screen(stream, context, 5s)){ options_open = true; break; } @@ -244,141 +240,118 @@ void open_options(VideoStream& stream, ProControllerContext& context, const Tuto context.wait_for_all_requests(); } -// How well the selected row's value matches the reference. -double option_value_score(const ImageViewRGB32& frame, double y, size_t row, const ImageRGB32& value_reference){ - if (row >= FIRST_SLIDER_ROW){ - // Compare the numeric value rather than the long slider: - // neighboring slider levels otherwise look too similar. - ImageViewRGB32 number = value_reference.sub_image( - value_reference.width() * 82 / 100, 0, - value_reference.width() - value_reference.width() * 82 / 100, - value_reference.height()); - ImageRGB32 number_reference = number.scale_to(96, 32); - return option_image_score(frame, - {0.7728, y - 0.025, 0.0972, 0.05}, number_reference, true); +// The selected row's centre on a fresh frame of Options, retried while the +// screen fades in; fails with a screenshot after about a second. +double read_selected_row(VideoStream& stream, ProControllerContext& context, size_t row){ + for (size_t attempt = 0; attempt < 6; attempt++){ + context.wait_for_all_requests(); + const auto frame = stream.video().snapshot(); + const std::optional y = frame && options_screen_shown(frame) ? options_selected_row_y(frame) : std::nullopt; + if (y){ + return *y; + } + pbf_wait(context, 200ms); } - return option_image_score(frame, - {0.33, y - 0.025, 0.54, 0.05}, value_reference, true); + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Settings checkpoint: No selected row on Options for " + std::string(OPTION_ROWS[row].name), stream); } -// Whether Options shows Autosave selected and set to Off: a frame's look. -bool autosave_off_on_screen( - const ImageViewRGB32& check, const TutorialOptionsDetector& options_screen, - const TutorialOptionPositionDetector& position, const ImageRGB32& autosave_label -){ - const double check_y = check ? position.selected_row(check) : -1; - return check && options_screen.detect(check) && check_y >= 0 && - option_image_score(check, {0.08, check_y - 0.025, 0.24, 0.05}, autosave_label, false) >= 0.80 && - autosave_off_highlight(check, check_y); +// Options opens on its first row (Text Speed), at the top of the list: +// that row's centre. +double check_first_row(VideoStream& stream, ProControllerContext& context){ + const double y = read_selected_row(stream, context, 0); + stream.log("[Tutorial settings] First row at y=" + std::to_string(y)); + if (y < FIRST_ROW_Y - ROW_Y_TOLERANCE || y > FIRST_ROW_Y + ROW_Y_TOLERANCE){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Settings checkpoint: Options did not open on its first row.", stream); + } + return y; } -// Autosave Off, on three more frames 200 ms apart. -void confirm_autosave_off( - VideoStream& stream, ProControllerContext& context, const TutorialOptionsDetector& options_screen, - const TutorialOptionPositionDetector& position, const ImageRGB32& label_reference -){ - for (size_t confirmation = 0; confirmation < 3; confirmation++){ - pbf_wait(context, 200ms); - context.wait_for_all_requests(); - auto check = stream.video().snapshot(); - if (!autosave_off_on_screen(check, options_screen, position, label_reference)){ - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Autosave Off could not be confirmed. Stopping before movement.", stream); +// The mean grey-level difference of two frames over `box`, sampled every 2nd pixel. +double box_difference(const ImageViewRGB32& a, const ImageViewRGB32& b, const ImageFloatBox& box){ + const ImageViewRGB32 x = extract_box_reference(a, box); + const ImageViewRGB32 z = extract_box_reference(b, box); + const size_t width = std::min(x.width(), z.width()), height = std::min(x.height(), z.height()); + double sum = 0; + size_t count = 0; + for (size_t py = 0; py < height; py += 2){ + for (size_t px = 0; px < width; px += 2){ + const uint32_t p = x.pixel(px, py), q = z.pixel(px, py); + const int gp = ((p >> 16) & 255) + ((p >> 8) & 255) + (p & 255); + const int gq = ((q >> 16) & 255) + ((q >> 8) & 255) + (q & 255); + sum += std::abs(gp - gq) / 3.0; + count++; } } + return count == 0 ? 0 : sum / count; } -// One press toward the target value. Sliders clamp at zero: establish a -// known endpoint, then increment; the target image is verified afterward. -void correct_option(ProControllerContext& context, size_t row, bool& slider_reset){ - if (row >= FIRST_SLIDER_ROW && !slider_reset){ - for (size_t step = 0; step < 12; step++){ - pbf_press_dpad(context, DPAD_LEFT, 80ms, 100ms); +// From row `row` (outline at `y`): one press down, confirmed by the outline +// moving a row down, or, at the bottom of the view, by the list scrolling +// under it (the label under the outline changes). Never presses twice, so +// a row is never skipped. Returns the new row's centre. +double move_down(VideoStream& stream, ProControllerContext& context, size_t row, double y){ + context.wait_for_all_requests(); + const auto before = stream.video().snapshot(); + pbf_press_dpad(context, DPAD_DOWN, 80ms, 350ms); + context.wait_for_all_requests(); + const ImageFloatBox label{0.08, y - 0.025, 0.24, 0.05}; + for (size_t check = 0; check < 15; check++){ + const auto after = stream.video().snapshot(); + const std::optional now = after && options_screen_shown(after) ? options_selected_row_y(after) : std::nullopt; + if (now && *now > y + ROW_PITCH / 2 && *now < y + ROW_PITCH * 1.5){ + return *now; } - slider_reset = true; - }else{ - pbf_press_dpad(context, DPAD_RIGHT, 80ms, 350ms); + if (now && std::abs(*now - y) < ROW_PITCH / 4 && before && box_difference(before, after, label) > 8){ + return *now; + } + context.wait_for(100ms); } - context.wait_for_all_requests(); + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Settings checkpoint: Cursor did not move down from " + std::string(OPTION_ROWS[row].name), stream); } -// The selected row on screen, if it is the expected one with its value -// highlighted; -1 otherwise. -double expected_row_y( - VideoStream& stream, const ImageViewRGB32& frame, const TutorialOptionsDetector& options_screen, - const TutorialOptionPositionDetector& position, size_t row, const ImageRGB32& label_reference -){ - const double y = frame && options_screen.detect(frame) ? position.selected_row(frame) : -1; - const bool value_visible = y >= 0 && position.selected_value(frame, y); - const double label_score = y >= 0 ? option_image_score(frame, - {0.08, y - 0.025, 0.24, 0.05}, label_reference, false) : 0; - stream.log("[Tutorial settings] Expected row=" + std::string(SETTINGS[row].label) + - " row_y=" + std::to_string(y) + " highlighted_value=" + std::to_string(value_visible) + - " label_match=" + std::to_string(label_score)); - return y >= 0 && value_visible && label_score >= 0.80 ? y : -1; +// The highlighted column of `row` (outline at `y`) on a fresh frame, or none. +std::optional read_column(VideoStream& stream, ProControllerContext& context, size_t row, double y){ + context.wait_for_all_requests(); + const auto frame = stream.video().snapshot(); + const std::optional column = frame ? options_highlighted_column(frame, y, OPTION_ROWS[row].columns) : std::nullopt; + stream.log("[Tutorial settings] Row " + std::to_string(row) + " (" + std::string(OPTION_ROWS[row].name) + ") y=" + + std::to_string(y) + " column=" + (column ? std::to_string(*column) : "none") + + " target=" + std::to_string(OPTION_ROWS[row].target)); + return column; } -// Whether the selected row (at `y`) shows the target value: checked on -// the frame, with the row's boxes on the overlay. -bool option_verified( - VideoStream& stream, ProControllerContext& context, const ImageViewRGB32& frame, double y, size_t row, - const OptionsScreen& screen, const OptionReferences& references -){ - const ImageRGB32& value_reference = references.value; - // These are regular video-overlay boxes, so the existing overlay - // display control can show/hide every active ROI. - OverlayBoxScope row_overlay(stream.overlay(), COLOR_RED, {0.07, y - 0.04, 0.84, 0.08}, ""); - OverlayBoxScope label_overlay(stream.overlay(), COLOR_YELLOW, {0.08, y - 0.025, 0.24, 0.05}, ""); - OverlayBoxScope value_overlay(stream.overlay(), COLOR_GREEN, {0.33, y - 0.025, 0.54, 0.05}, ""); - VideoOverlaySet autosave_rois(stream.overlay()); - if (row == AUTOSAVE_ROW){ - autosave_rois.add(COLOR_GREEN, {0.63, y - 0.022, 0.055, 0.044}); - autosave_rois.add(COLOR_RED, {0.34, y - 0.022, 0.055, 0.044}); - } - const double score = option_value_score(frame, y, row, value_reference); - stream.log("[Tutorial settings] " + std::string(SETTINGS[row].label) + " target match=" + std::to_string(score)); - // The reference recording uses inactive pale-red text; the focused live - // row has a thicker dark-red glyph mask. Verified Fast scores 0.734 - // while Slow/Normal score 0. - if (score < 0.65 || (row == AUTOSAVE_ROW && !autosave_off_highlight(frame, y))){ - return false; - } - if (row == AUTOSAVE_ROW){ - confirm_autosave_off(stream, context, screen.options_screen, screen.position, references.label); +// Sets the selected row `row` (outline at `y`) to its target column, a +// press at a time toward it, reading the highlight after each press. +void set_option(VideoStream& stream, ProControllerContext& context, size_t row, double y){ + const OptionRow& option = OPTION_ROWS[row]; + if (option.columns == 0){ + stream.log("[Tutorial settings] Row " + std::to_string(row) + " (" + std::string(option.name) + ") left as it is."); + return; } - return true; -} - -// Sets the option on `row`, the row selected now, to its target value. -void set_option(VideoStream& stream, ProControllerContext& context, size_t row, const OptionsScreen& screen){ - const std::string base = RESOURCE_PATH() + "PokemonBDSP/Autostory/Options/" + SETTINGS[row].label; - const OptionReferences references{ImageRGB32(base + "_label.png"), ImageRGB32(base + "_value.png")}; - bool slider_reset = false; - for (size_t attempt = 0; attempt < 30; attempt++){ - // Detect the selected row afresh after every navigation/correction. - context.wait_for_all_requests(); - auto frame = stream.video().snapshot(); - const double y = expected_row_y(stream, frame, screen.options_screen, screen.position, row, references.label); - if (y < 0){ - // A menu can be detected while its row/text is still fading - // in. Re-read without directional inputs first. - if (attempt < 4){ - pbf_wait(context, 200ms); - context.wait_for_all_requests(); - continue; + OverlayBoxScope row_overlay(stream.overlay(), COLOR_RED, {0.07, y - 0.04, 0.84, 0.08}, ""); + OverlayBoxScope value_overlay(stream.overlay(), COLOR_GREEN, {0.335, y - 0.022, 0.545, 0.044}, ""); + size_t unread = 0; + for (size_t attempt = 0; attempt < 12; attempt++){ + const std::optional column = read_column(stream, context, row, y); + if (!column){ + // The highlight can still be fading in after a press. + if (++unread > 4){ + break; } - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Settings checkpoint: Unable to confirm expected row " + std::string(SETTINGS[row].label), stream); + pbf_wait(context, 200ms); + continue; } - if (option_verified(stream, context, frame, y, row, screen, references)){ - stream.log("[Tutorial settings] Verified " + std::string(SETTINGS[row].label) + "=" + SETTINGS[row].value); + if (*column == option.target){ + stream.log("[Tutorial settings] Verified " + std::string(option.name)); return; } - stream.log("[Tutorial settings] Correcting " + std::string(SETTINGS[row].label)); - correct_option(context, row, slider_reset); + pbf_press_dpad(context, *column < option.target ? DPAD_RIGHT : DPAD_LEFT, 80ms, 350ms); } OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Settings checkpoint: Could not apply " + std::string(SETTINGS[row].label), stream); + "Settings checkpoint: Could not apply " + std::string(option.name), stream); } // Confirms the settings and leaves Options (A A A A B B), back to the overworld. @@ -397,7 +370,7 @@ void save_options(VideoStream& stream, ProControllerContext& context, const std: } // From the overworld: the saved Options screen again. -void reopen_options(VideoStream& stream, ProControllerContext& context, const TutorialOptionsDetector& options_screen){ +void reopen_options(VideoStream& stream, ProControllerContext& context){ pbf_press_button(context, BUTTON_X, 80ms, 1000ms); context.wait_for_all_requests(); MenuWatcher verify_menu(COLOR_RED, /*skip_new_banner*/ true); @@ -408,7 +381,7 @@ void reopen_options(VideoStream& stream, ProControllerContext& context, const Tu bool reopened = false; for (size_t attempt = 0; attempt < 2; attempt++){ pbf_press_button(context, BUTTON_A, 80ms, 0ms); - if (wait_for_options_screen(stream, context, options_screen, 5s)){ + if (wait_for_options_screen(stream, context, 5s)){ reopened = true; break; } @@ -426,51 +399,41 @@ void reopen_options(VideoStream& stream, ProControllerContext& context, const Tu } } -// In the saved Options: Autosave is Off on three frames. Options starts -// on Text Speed; Autosave is seven rows below it. -void check_saved_autosave( - VideoStream& stream, ProControllerContext& context, - const TutorialOptionsDetector& options_screen, const TutorialOptionPositionDetector& position -){ - for (size_t step = 0; step < AUTOSAVE_ROW; step++){ - pbf_press_dpad(context, DPAD_DOWN, 80ms, 350ms); +// In the saved Options: down to Autosave, a confirmed row at a time, and +// Off highlighted on three frames 200 ms apart. +void check_saved_autosave(VideoStream& stream, ProControllerContext& context){ + double y = check_first_row(stream, context); + for (size_t row = 0; row < AUTOSAVE_ROW; row++){ + y = move_down(stream, context, row, y); } - context.wait_for_all_requests(); - const ImageRGB32 autosave_label(RESOURCE_PATH() + "PokemonBDSP/Autostory/Options/autosave_label.png"); for (size_t confirmation = 0; confirmation < 3; confirmation++){ - auto check = stream.video().snapshot(); - if (!autosave_off_on_screen(check, options_screen, position, autosave_label)){ + if (read_column(stream, context, AUTOSAVE_ROW, y) != OPTION_ROWS[AUTOSAVE_ROW].target){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Saved Autosave setting is not verified Off. Stopping before movement.", stream); } pbf_wait(context, 200ms); - context.wait_for_all_requests(); } stream.log("[Tutorial settings] Autosave = Off: confirmed after reopening saved Options (3 frames)."); -} // namespace +} void verify_tutorial_settings_once(VideoStream& stream, ProControllerContext& context){ open_main_menu(stream, context); - OptionsScreen screen; - open_options(stream, context, screen.options_screen); - for (size_t row = 0; row < SETTING_COUNT; row++){ - set_option(stream, context, row, screen); - if (row + 1 < SETTING_COUNT){ - pbf_press_dpad(context, DPAD_DOWN, 80ms, 350ms); + open_options(stream, context); + double y = check_first_row(stream, context); + for (size_t row = 0; row < OPTION_ROW_COUNT; row++){ + set_option(stream, context, row, y); + if (row + 1 < OPTION_ROW_COUNT){ + y = move_down(stream, context, row, y); } } save_options(stream, context, "Settings checkpoint: Did not return to the overworld after saving options."); // Reopen the saved Options screen and independently verify Autosave. // No route movement is permitted until this persisted-value check passes. - reopen_options(stream, context, screen.options_screen); - check_saved_autosave(stream, context, screen.options_screen, screen.position); + reopen_options(stream, context); + check_saved_autosave(stream, context); save_options(stream, context, "Autosave verification: Could not return to the overworld."); - // Report inferred values explicitly: these are visual matches, - // allowing the user to compare our conclusions with the screen. - stream.log("[Tutorial settings] Completed. Values believed applied (visual matches before saving):"); - stream.log("[Tutorial settings] Text Speed = Fast; Change Window = Type 1; Battle Effects = Off; Battle Style = Set."); - stream.log("[Tutorial settings] Send to Boxes = Automatic; Give Nicknames = Don't give; Gyroscope = Use."); - stream.log("[Tutorial settings] Autosave = Off; Show Nicknames = Show; Background Music = 8; Sound Effects = 8."); + stream.log("[Tutorial settings] Completed: Text Speed Fast, Battle Effects Off, Battle Style Set, " + "Send to Boxes Automatic, Give Nicknames Don't give, Gyroscope Use, Autosave Off, Show Nicknames Show."); } // After a failed attempt: back out of Options, save prompts, the Bag or @@ -478,9 +441,8 @@ void verify_tutorial_settings_once(VideoStream& stream, ProControllerContext& co void back_to_overworld(VideoStream& stream, ProControllerContext& context){ stream.log("[Tutorial settings] Returning to overworld before retry."); context.wait_for_all_requests(); - TutorialOptionsDetector options_screen; const auto failed_frame = stream.video().snapshot(); - if (failed_frame && options_screen.detect(failed_frame)){ + if (failed_frame && options_screen_shown(failed_frame)){ // Finish the known Options save/exit sequence before observing // state; never cancel it on an Options frame misclassified as // overworld by the generic detector. @@ -497,7 +459,7 @@ void back_to_overworld(VideoStream& stream, ProControllerContext& context){ size_t confirmations = 0; for (size_t check = 0; check < 15 && !recovered; check++){ const auto frame = stream.video().snapshot(); - const bool clear = frame && !options_screen.detect(frame) && + const bool clear = frame && !options_screen_shown(frame) && !menu.detect(frame) && !black.detect(frame) && bag_pocket_position(frame) == 0 && overworld.detect(frame); if (clear && ++confirmations >= 3){ recovered = true; diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h index 206237de07..7aa388b7e6 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h @@ -9,6 +9,8 @@ #ifndef PokemonAutomation_PokemonBDSP_AutoStory_Settings_H #define PokemonAutomation_PokemonBDSP_AutoStory_Settings_H +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" #include "CommonFramework/Tools/VideoStream.h" #include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" @@ -17,6 +19,22 @@ namespace NintendoSwitch{ namespace PokemonBDSP{ +// The Options screen, read from its layout and colours only, so that it +// reads the same in every game language. + +// Whether the frame shows Options: the blue heading bar over the white panel. +bool options_screen_shown(const ImageViewRGB32& screen); + +// The centre of the selected row (its red outline), as a fraction of the +// screen height; none without exactly one outlined row. +std::optional options_selected_row_y(const ImageViewRGB32& screen); + +// Which of the `columns` value columns (2 or 3) of the row centred at `y` +// holds the highlighted (red) value, counted from the left; none when no +// column or more than one is highlighted. +std::optional options_highlighted_column(const ImageViewRGB32& screen, double y, size_t columns); + + // From the overworld: opens Options and sets what the story needs (fast // text, Send to Boxes automatic, no nickname prompt, Autosave off), saves // them and returns to the overworld. Tries four times, backing out to the