From d1b5bdf0f97bae85c1e90c7faf78a3eaf7ca33ae Mon Sep 17 00:00:00 2001 From: dayfine Date: Thu, 29 Oct 2026 12:55:16 -0700 Subject: [PATCH 1/5] Switch: read the on-screen keyboard and type on it with feedback Games open the Switch's own keyboard for names (e.g. BDSP's player name). keyboard_enter_code() types on it blind from an assumed starting key, so it can't recover from a press the keyboard ignored while sliding in, or start from wherever the keyboard left its cursor. SystemKeyboardDetector recognizes the keyboard (OK blue, or grey while the field is empty), find_system_keyboard_cursor() reads the key under the cursor, and system_keyboard_field_has_text() says whether the field is empty. Unit tests cover all three on six reference screenshots (CommandLineTests, NintendoSwitch/SystemKeyboardDetector). type_on_system_keyboard() in FastCodeEntry reads the cursor before each key and moves it with keyboard_get_path(), now declared in the header, over KEYBOARD_POSITIONS, so it shares the key grid and the wrap-around moves with blind code entry and also supports AZERTY. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../ComputerPrograms_UnitTestRunner.cpp | 2 + .../NintendoSwitch_SystemKeyboardDetector.cpp | 141 ++++++++++++++++++ .../NintendoSwitch_SystemKeyboardDetector.h | 66 ++++++++ .../NintendoSwitch_KeyboardCodeEntry.h | 9 ++ .../NintendoSwitch_SystemKeyboardTyping.cpp | 108 ++++++++++++++ .../NintendoSwitch_SystemKeyboardTyping.h | 39 +++++ SerialPrograms/cmake/SourceFiles.cmake | 4 + 7 files changed, 369 insertions(+) create mode 100644 SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp create mode 100644 SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h create mode 100644 SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.cpp create mode 100644 SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index 75af30e3e6..e4cc0a7488 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -25,6 +25,7 @@ #include "NintendoSwitch/Inference/NintendoSwitch_CheckOnlineDetector.h" #include "NintendoSwitch/Inference/NintendoSwitch_FailedToConnectDetector.h" +#include "NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h" #include "NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.h" #include "NintendoSwitch/Programs/Typewriter/NintendoSwitch_Typewriter_Tests.h" @@ -59,6 +60,7 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){ Kernels::add_tests(ret); NintendoSwitch::add_tests_CheckOnlineDetector(ret); NintendoSwitch::add_tests_FailedToConnectDetector(ret); + NintendoSwitch::add_tests_SystemKeyboardDetector(ret); NintendoSwitch::add_tests_UpdatePopupDetector(ret); NintendoSwitch::Typewriter::add_tests_TypewriterTextMapping(ret); NintendoSwitch::PokemonFRLG::add_tests(ret); diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp new file mode 100644 index 0000000000..428abbafa3 --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp @@ -0,0 +1,141 @@ +/* System Keyboard Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Images/SolidColorTest.h" +#include "NintendoSwitch_SystemKeyboardDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + +// Centres of the four rows of character keys, and of the first column and the +// spacing between columns, as fractions of the screen. +static constexpr double KEY_ROW_Y[] = {0.500, 0.593, 0.681, 0.770}; +static constexpr double KEY_COL_X0 = 0.077; +static constexpr double KEY_COL_STEP = 0.075; + + +SystemKeyboardDetector::SystemKeyboardDetector(Color color) + : m_color(color) + , m_key_panel(0.15, 0.555, 0.65, 0.020) + , m_bottom_bar(0.10, 0.935, 0.35, 0.030) + , m_ok_button(0.875, 0.760, 0.045, 0.020) +{} +void SystemKeyboardDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_key_panel); + items.add(m_color, m_bottom_bar); + items.add(m_color, m_ok_button); +} +bool SystemKeyboardDetector::detect(const ImageViewRGB32& screen){ + if (!is_white(extract_box_reference(screen, m_key_panel)) || + !is_white(extract_box_reference(screen, m_bottom_bar)) + ){ + return false; + } + // OK is blue, or flat grey while the field is empty (disabled). + const ImageStats ok = image_stats(extract_box_reference(screen, m_ok_button)); + return is_solid(ok, {0.172, 0.276, 0.552}) || is_grey(ok, 600, 690); +} + + +bool find_system_keyboard_cursor(const ImageViewRGB32& screen, int& row, int& col){ + // The cursor is a cyan border around one key. Neighbouring keys overlap that border, + // so take the centroid of all cyan pixels and map it to the nearest key. + const ImageFloatBox area(0.02, 0.44, 0.84, 0.38); + ImageViewRGB32 keys = extract_box_reference(screen, area); + double count = 0, sum_x = 0, sum_y = 0; + for (size_t y = 0; y < keys.height(); y++){ + for (size_t x = 0; x < keys.width(); x++){ + const uint32_t pixel = keys.pixel(x, y); + const int red = (pixel >> 16) & 0xff; + const int green = (pixel >> 8) & 0xff; + const int blue = pixel & 0xff; + if (red < 150 && green > 200 && blue > 190){ + count++; + sum_x += (double)x; + sum_y += (double)y; + } + } + } + // The border is about 0.15% of the screen; a few stray pixels are not a cursor. + if (count < 0.0005 * screen.width() * screen.height()){ + return false; + } + const double cx = area.x + sum_x / count / keys.width() * area.width; + const double cy = area.y + sum_y / count / keys.height() * area.height; + col = (int)std::lround((cx - KEY_COL_X0) / KEY_COL_STEP); + row = 0; + for (int r = 1; r < 4; r++){ + if (std::abs(KEY_ROW_Y[r] - cy) < std::abs(KEY_ROW_Y[row] - cy)){ + row = r; + } + } + return 0 <= col && col < 11; +} + +bool system_keyboard_field_has_text(const ImageViewRGB32& screen){ + // Left part of the field, after the caret's position on an empty field. + const ImageStats stats = image_stats(extract_box_reference(screen, ImageFloatBox(0.11, 0.235, 0.19, 0.030))); + return stats.stddev.sum() > 30; +} + + +// One screenshot and what each function should read from it. +class Test_SystemKeyboardDetector : public UnitTest{ +public: + // cursor_row/col: -1 for a screen without the keyboard. + Test_SystemKeyboardDetector(const std::string& image, bool keyboard, int cursor_row, int cursor_col, bool has_text) + : UnitTest("NintendoSwitch::SystemKeyboardDetector - " + image) + , m_image(UNIT_TEST_RESOURCE_PATH() + image) + , m_keyboard(keyboard) + , m_cursor_row(cursor_row) + , m_cursor_col(cursor_col) + , m_has_text(has_text) + {} + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + ImageRGB32 image(m_image); + if (SystemKeyboardDetector().detect(image) != m_keyboard){ + return false; + } + if (!m_keyboard){ + return true; + } + int row = -1; + int col = -1; + return find_system_keyboard_cursor(image, row, col) + && row == m_cursor_row && col == m_cursor_col + && system_keyboard_field_has_text(image) == m_has_text; + } + +private: + std::string m_image; + bool m_keyboard; + int m_cursor_row; + int m_cursor_col; + bool m_has_text; +}; + + +void add_tests_SystemKeyboardDetector(UnitTestDatabase& database){ + const std::string folder = "NintendoSwitch/SystemKeyboardDetector/"; + // The OK button is blue with text in the field and grey without. + database.add(folder + "BDSP-PlayerName-Empty.png", true, 0, 0, false); + database.add(folder + "BDSP-FriendName-Typed.png", true, 0, 0, true); + database.add(folder + "BDSP-Dialog.png", false, -1, -1, false); + database.add(folder + "HomeMenu.png", false, -1, -1, false); + database.add(folder + "HomeMenu-OnlineLicensePopup.png", false, -1, -1, false); + database.add(folder + "UserSelect.png", false, -1, -1, false); +} + + + +} +} diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h new file mode 100644 index 0000000000..7a2952a2ce --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h @@ -0,0 +1,66 @@ +/* System Keyboard Detector + * + * From: https://github.com/PokemonAutomation/ + * + * The Switch's own on-screen keyboard, which games open for names (e.g. the + * player's name in BDSP): detect it, find the key under its cursor, and tell + * whether its text field is empty. Typing with it is in + * Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h. + * + * Measured on 1920x1080 captures of the QWERTY layout. + * + */ + +#ifndef PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H +#define PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H + +#include +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + + +// White key panel and bottom bar, and the OK button (blue, or grey while the +// field is empty). +class SystemKeyboardDetector : public StaticScreenDetector{ +public: + SystemKeyboardDetector(Color color = COLOR_CYAN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_key_panel; + ImageFloatBox m_bottom_bar; + ImageFloatBox m_ok_button; +}; +class SystemKeyboardWatcher : public DetectorToFinder{ +public: + SystemKeyboardWatcher(Color color = COLOR_CYAN) + : DetectorToFinder("SystemKeyboardWatcher", std::chrono::milliseconds(250), color) + {} +}; + + +// The key under the cursor, as row (0: digits .. 3: "zxcv") and column (0..10). +// The keyboard remembers the cursor between uses, so read it, don't assume it. +// Returns false if no cursor border is visible. +bool find_system_keyboard_cursor(const ImageViewRGB32& screen, int& row, int& col); + +// Whether the text field holds any text. Pressing B on an empty field closes the +// keyboard, so delete only while this is true. +bool system_keyboard_field_has_text(const ImageViewRGB32& screen); + + + +void add_tests_SystemKeyboardDetector(UnitTestDatabase& database); + + + +} +} +#endif diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.h b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.h index 7048582ea0..38f4f4d841 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.h +++ b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.h @@ -8,10 +8,12 @@ #define PokemonAutomation_NintendoSwitch_KeyboardCodeEntry_H #include +#include #include "Controllers/StandardHid/StandardHid_Keyboard.h" #include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" #include "NintendoSwitch/NintendoSwitch_ConsoleHandle.h" #include "NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardEntryMappings.h" +#include "NintendoSwitch_CodeEntryTools.h" namespace PokemonAutomation{ namespace NintendoSwitch{ @@ -19,6 +21,13 @@ namespace FastCodeEntry{ +// The D-pad moves from "source" to "destination" on the on-screen keyboard, +// wrapping around the sides when that is shorter, then ENTER_CHAR. +std::vector keyboard_get_path( + KeyboardEntryPosition source, + KeyboardEntryPosition destination +); + // Automatically select between keyboard or pro controller depending on the // controller type. void keyboard_enter_code( diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.cpp new file mode 100644 index 0000000000..43a3329df5 --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.cpp @@ -0,0 +1,108 @@ +/* System Keyboard Typing + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h" +#include "NintendoSwitch_KeyboardCodeEntry.h" +#include "NintendoSwitch_SystemKeyboardTyping.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace FastCodeEntry{ + +using namespace std::chrono_literals; + + +namespace{ + +constexpr Milliseconds MOVE_HOLD = 100ms; // One key per press, without auto-repeat. +constexpr Milliseconds MOVE_RELEASE = 200ms; +constexpr Milliseconds WRAP_RELEASE = 400ms; // Wrapping around a side animates for longer. +constexpr Milliseconds SETTLE = 300ms; // Before reading the cursor after moving it. +constexpr Milliseconds KEY_HOLD = 80ms; +constexpr Milliseconds KEY_RELEASE = 300ms; + +// A press made while the keyboard is still sliding in is ignored, so the +// cursor is read again after each try until it is on the target. +constexpr size_t MAX_MOVE_TRIES = 10; + + +bool read_cursor(VideoStream& stream, ProControllerContext& context, KeyboardEntryPosition& position){ + context.wait_for_all_requests(); + VideoSnapshot screen = stream.video().snapshot(); + int row = -1; + int col = -1; + if (!screen || !find_system_keyboard_cursor(screen, row, col)){ + return false; + } + position = {(uint8_t)row, (uint8_t)col}; + return true; +} + +void move_cursor(VideoStream& stream, ProControllerContext& context, KeyboardEntryPosition target, char ch){ + for (size_t attempt = 0; attempt < MAX_MOVE_TRIES; attempt++){ + KeyboardEntryPosition cursor; + if (!read_cursor(stream, context, cursor)){ + pbf_wait(context, SETTLE); + continue; + } + if (cursor.row == target.row && cursor.col == target.col){ + return; + } + for (CodeEntryAction action : keyboard_get_path(cursor, target)){ + if (!is_move(action)){ + continue; + } + const bool wrap = ((uint8_t)action & 0x80) != 0; + const DpadPosition direction = (DpadPosition)((uint8_t)action & 0x7f); + pbf_press_dpad(context, direction, MOVE_HOLD, wrap ? WRAP_RELEASE : MOVE_RELEASE); + } + pbf_wait(context, SETTLE); + } + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::string("System keyboard: the cursor did not reach '") + ch + "'.", + stream + ); +} + +} + + + +void type_on_system_keyboard( + VideoStream& stream, ProControllerContext& context, + const std::string& text, + KeyboardLayout layout +){ + const std::map& positions = KEYBOARD_POSITIONS(layout); + for (char ch : text){ + // Letters are listed under their uppercase key. + auto iter = positions.find((char)std::toupper((unsigned char)ch)); + if (iter == positions.end()){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::string("System keyboard: no key for '") + ch + "'.", + stream + ); + } + move_cursor(stream, context, iter->second, ch); + if (std::isupper((unsigned char)ch)){ + pbf_press_button(context, BUTTON_LCLICK, 100ms, 250ms); + } + pbf_press_button(context, BUTTON_A, KEY_HOLD, KEY_RELEASE); + } + context.wait_for_all_requests(); +} + + + +} +} +} diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h new file mode 100644 index 0000000000..b0ec331111 --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h @@ -0,0 +1,39 @@ +/* System Keyboard Typing + * + * From: https://github.com/PokemonAutomation/ + * + * Types text on the Switch's on-screen keyboard with a Pro Controller, reading + * the cursor from the video before each key. Unlike keyboard_enter_code(), + * which moves blind from an assumed starting key, it works wherever the + * keyboard left its cursor and catches presses the keyboard ignored. + * + */ + +#ifndef PokemonAutomation_NintendoSwitch_SystemKeyboardTyping_H +#define PokemonAutomation_NintendoSwitch_SystemKeyboardTyping_H + +#include +#include "CommonFramework/Tools/VideoStream.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "NintendoSwitch_KeyboardEntryMappings.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace FastCodeEntry{ + + +// Types `text` (letters, digits and the symbols on the keyboard's first page) +// on the open keyboard. An uppercase letter is typed with Shift (left stick +// click). Throws, with a screenshot, for a character that has no key, or if +// the cursor can't be read or doesn't reach a key. +void type_on_system_keyboard( + VideoStream& stream, ProControllerContext& context, + const std::string& text, + KeyboardLayout layout = KeyboardLayout::QWERTY +); + + +} +} +} +#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 39c48d531e..f82e61647b 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1323,6 +1323,8 @@ file(GLOB LIBRARY_SOURCES Source/NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.h Source/NintendoSwitch/Inference/NintendoSwitch_StartGameUserSelectDetector.cpp Source/NintendoSwitch/Inference/NintendoSwitch_StartGameUserSelectDetector.h + Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp + Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h Source/NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.cpp Source/NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.h Source/NintendoSwitch/NintendoSwitch_ConsoleHandle.cpp @@ -1374,6 +1376,8 @@ file(GLOB LIBRARY_SOURCES Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.h Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_NumberCodeEntry.cpp Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_NumberCodeEntry.h + Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.cpp + Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h Source/NintendoSwitch/Programs/NintendoSwitch_FriendCodeAdder.cpp Source/NintendoSwitch/Programs/NintendoSwitch_FriendCodeAdder.h Source/NintendoSwitch/Programs/NintendoSwitch_FriendDelete.cpp From 5d80115640d9a86dd16aa7b9cb522cbf1081a8c8 Mon Sep 17 00:00:00 2001 From: dayfine Date: Fri, 13 Nov 2026 08:24:43 -0800 Subject: [PATCH 2/5] System keyboard: read every key, robust detector and field checks, checked typing Review of #62: - The cursor reader covers the right column (backspace, Return, OK) and the bottom row; the typer steps back onto the character grid from there. - The detector's key-panel check uses two strips (rows 0/1 and 2/3), so a cursor anywhere can only spoil one of them. - system_keyboard_field_has_text() reads the backspace key (dark with text), not the field over the game's background. - The typer checks every character before the first press (UserSetupError), logs the cursor read and the moves of each try, names the last cursor and the target in its error, and checks that each key press adds white pixels to the field. - Presses use the code entry hold for Switch 1; the read-back waits stay fixed. - read_system_keyboard_cursor() returns std::optional. - Tests derive ImageUnitTest, add six Legends: Z-A frames (en/fr/de/it/ja/es) and list the screens still to capture, which are added once their file exists. - Headers state the supported keyboards (Latin QWERTY/AZERTY), Korean and Chinese out of scope, Switch 2 untested. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../NintendoSwitch_SystemKeyboardDetector.cpp | 244 +++++++++++++----- .../NintendoSwitch_SystemKeyboardDetector.h | 57 +++- .../NintendoSwitch_KeyboardCodeEntry.h | 2 +- .../NintendoSwitch_SystemKeyboardTyping.cpp | 225 +++++++++++++--- .../NintendoSwitch_SystemKeyboardTyping.h | 36 ++- 5 files changed, 432 insertions(+), 132 deletions(-) diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp index 428abbafa3..4f0e37dedf 100644 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp @@ -5,137 +5,249 @@ */ #include +#include +#include +#include +#include +#include "Common/Cpp/Filesystem/Filesystem.h" #include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "CommonFramework/ImageTools/ImageStats.h" #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" #include "CommonTools/Images/SolidColorTest.h" +#include "Tests/ImageUnitTest.h" #include "NintendoSwitch_SystemKeyboardDetector.h" namespace PokemonAutomation{ namespace NintendoSwitch{ -// Centres of the four rows of character keys, and of the first column and the -// spacing between columns, as fractions of the screen. -static constexpr double KEY_ROW_Y[] = {0.500, 0.593, 0.681, 0.770}; -static constexpr double KEY_COL_X0 = 0.077; -static constexpr double KEY_COL_STEP = 0.075; +namespace{ + +// Centres of the five rows of keys, and of the first column and the spacing +// between columns, as fractions of the screen. +constexpr double KEY_ROW_Y[] = {0.500, 0.593, 0.681, 0.770, 0.858}; +constexpr double KEY_COL_X0 = 0.077; +constexpr double KEY_COL_STEP = 0.075; + +// In the right column, Return spans rows 1-2 and OK rows 3-4. These are the +// gaps between those keys. +constexpr double RIGHT_COLUMN_RETURN_TOP = 0.548; +constexpr double RIGHT_COLUMN_OK_TOP = 0.725; + +// The cursor border and everything else the reader looks at, from the top of the +// digit row to the bottom of the bottom row. +const ImageFloatBox KEYS_AREA(0.0, 0.43, 1.0, 0.49); + +// The cursor border is cyan (about 80, 230, 210). +bool is_cursor_pixel(uint32_t pixel){ + const int red = (pixel >> 16) & 0xff; + const int green = (pixel >> 8) & 0xff; + const int blue = pixel & 0xff; + return red < 150 && green > 200 && blue > 190; +} + +uint8_t nearest_row(double y){ + uint8_t row = 0; + for (uint8_t r = 1; r < 5; r++){ + if (std::abs(KEY_ROW_Y[r] - y) < std::abs(KEY_ROW_Y[row] - y)){ + row = r; + } + } + return row; +} + +// The key whose cursor border has its centre at (x, y), or nothing if that is +// off the keyboard. +std::optional key_at(double x, double y){ + const long col = std::lround((x - KEY_COL_X0) / KEY_COL_STEP); + if (col < 0 || col > 11){ + return std::nullopt; + } + if (col < 11){ + return SystemKeyboardKey{nearest_row(y), (uint8_t)col}; + } + if (y < RIGHT_COLUMN_RETURN_TOP){ + return SystemKeyboardKey{0, 11}; + } + if (y < RIGHT_COLUMN_OK_TOP){ + return SystemKeyboardKey{1, 11}; + } + return SystemKeyboardKey{3, 11}; +} + +} // namespace + SystemKeyboardDetector::SystemKeyboardDetector(Color color) : m_color(color) - , m_key_panel(0.15, 0.555, 0.65, 0.020) + , m_key_panel_upper(0.15, 0.532, 0.65, 0.036) + , m_key_panel_lower(0.15, 0.715, 0.65, 0.032) , m_bottom_bar(0.10, 0.935, 0.35, 0.030) , m_ok_button(0.875, 0.760, 0.045, 0.020) {} void SystemKeyboardDetector::make_overlays(VideoOverlaySet& items) const{ - items.add(m_color, m_key_panel); + items.add(m_color, m_key_panel_upper); + items.add(m_color, m_key_panel_lower); items.add(m_color, m_bottom_bar); items.add(m_color, m_ok_button); } bool SystemKeyboardDetector::detect(const ImageViewRGB32& screen){ - if (!is_white(extract_box_reference(screen, m_key_panel)) || - !is_white(extract_box_reference(screen, m_bottom_bar)) - ){ + // Each strip crosses white keys and the slightly brighter gaps between them, + // which gives a stddev sum of about 11 (measured on BDSP and Legends: Z-A). + const double max_panel_stddev = 20; + const bool panel = + is_white(extract_box_reference(screen, m_key_panel_upper), 500, max_panel_stddev) || + is_white(extract_box_reference(screen, m_key_panel_lower), 500, max_panel_stddev); + if (!panel || !is_white(extract_box_reference(screen, m_bottom_bar))){ return false; } - // OK is blue, or flat grey while the field is empty (disabled). + // OK is blue (40, 93, 255), or flat grey (221, 221, 221) while the field is + // empty (disabled). Measured on BDSP and Legends: Z-A. const ImageStats ok = image_stats(extract_box_reference(screen, m_ok_button)); - return is_solid(ok, {0.172, 0.276, 0.552}) || is_grey(ok, 600, 690); + return is_solid(ok, {0.103, 0.240, 0.657}) || is_grey(ok, 600, 690); } -bool find_system_keyboard_cursor(const ImageViewRGB32& screen, int& row, int& col){ +std::optional read_system_keyboard_cursor(const ImageViewRGB32& screen){ // The cursor is a cyan border around one key. Neighbouring keys overlap that border, // so take the centroid of all cyan pixels and map it to the nearest key. - const ImageFloatBox area(0.02, 0.44, 0.84, 0.38); - ImageViewRGB32 keys = extract_box_reference(screen, area); + const ImageViewRGB32 keys = extract_box_reference(screen, KEYS_AREA); double count = 0, sum_x = 0, sum_y = 0; for (size_t y = 0; y < keys.height(); y++){ for (size_t x = 0; x < keys.width(); x++){ - const uint32_t pixel = keys.pixel(x, y); - const int red = (pixel >> 16) & 0xff; - const int green = (pixel >> 8) & 0xff; - const int blue = pixel & 0xff; - if (red < 150 && green > 200 && blue > 190){ + if (is_cursor_pixel(keys.pixel(x, y))){ count++; sum_x += (double)x; sum_y += (double)y; } } } - // The border is about 0.15% of the screen; a few stray pixels are not a cursor. + // The border is about 0.18% of the screen; a few stray pixels are not a cursor. if (count < 0.0005 * screen.width() * screen.height()){ - return false; + return std::nullopt; } - const double cx = area.x + sum_x / count / keys.width() * area.width; - const double cy = area.y + sum_y / count / keys.height() * area.height; - col = (int)std::lround((cx - KEY_COL_X0) / KEY_COL_STEP); - row = 0; - for (int r = 1; r < 4; r++){ - if (std::abs(KEY_ROW_Y[r] - cy) < std::abs(KEY_ROW_Y[row] - cy)){ - row = r; - } - } - return 0 <= col && col < 11; + return key_at( + KEYS_AREA.x + sum_x / count / keys.width() * KEYS_AREA.width, + KEYS_AREA.y + sum_y / count / keys.height() * KEYS_AREA.height + ); } bool system_keyboard_field_has_text(const ImageViewRGB32& screen){ - // Left part of the field, after the caret's position on an empty field. - const ImageStats stats = image_stats(extract_box_reference(screen, ImageFloatBox(0.11, 0.235, 0.19, 0.030))); - return stats.stddev.sum() > 30; + // Left part of the backspace key, clear of its icon and of the cursor border. + // Light grey (221) with an empty field, dark grey (52) with text. + const ImageStats stats = image_stats(extract_box_reference(screen, ImageFloatBox(0.878, 0.470, 0.008, 0.060))); + return stats.average.sum() < 400; +} + +size_t system_keyboard_text_pixels(const ImageViewRGB32& screen){ + // The field, above its underline, which is white in some games. + const ImageViewRGB32 field = extract_box_reference(screen, ImageFloatBox(0.03, 0.17, 0.87, 0.105)); + size_t count = 0; + for (size_t y = 0; y < field.height(); y++){ + for (size_t x = 0; x < field.width(); x++){ + const uint32_t pixel = field.pixel(x, y); + if (((pixel >> 16) & 0xff) > 190 && ((pixel >> 8) & 0xff) > 190 && (pixel & 0xff) > 190){ + count++; + } + } + } + return count; } + +namespace{ + // One screenshot and what each function should read from it. -class Test_SystemKeyboardDetector : public UnitTest{ +class Test_SystemKeyboardDetector : public ImageUnitTest{ public: - // cursor_row/col: -1 for a screen without the keyboard. - Test_SystemKeyboardDetector(const std::string& image, bool keyboard, int cursor_row, int cursor_col, bool has_text) - : UnitTest("NintendoSwitch::SystemKeyboardDetector - " + image) - , m_image(UNIT_TEST_RESOURCE_PATH() + image) - , m_keyboard(keyboard) - , m_cursor_row(cursor_row) - , m_cursor_col(cursor_col) + // `cursor`: the key under the cursor, or nothing for a screen without the keyboard. + Test_SystemKeyboardDetector(const std::string& image, std::optional cursor, bool has_text) + : ImageUnitTest(std::format("NintendoSwitch::SystemKeyboardDetector - {}", image), {image}) + , m_cursor(cursor) , m_has_text(has_text) {} - virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ - ImageRGB32 image(m_image); - if (SystemKeyboardDetector().detect(image) != m_keyboard){ - return false; + virtual UnitTestResult run_on_images( + Logger& logger, CancellableScope& scope, + const std::vector& images + ) const override{ + const ImageRGB32& image = images[0]; + const bool keyboard = SystemKeyboardDetector().detect(image); + if (keyboard != m_cursor.has_value()){ + return std::format("detect() returned {}.", keyboard); } - if (!m_keyboard){ + if (!keyboard){ return true; } - int row = -1; - int col = -1; - return find_system_keyboard_cursor(image, row, col) - && row == m_cursor_row && col == m_cursor_col - && system_keyboard_field_has_text(image) == m_has_text; + const std::optional cursor = read_system_keyboard_cursor(image); + if (cursor != m_cursor){ + return cursor + ? std::format("Read the cursor at ({}, {}).", cursor->row, cursor->col) + : std::format("Read no cursor."); + } + const bool has_text = system_keyboard_field_has_text(image); + if (has_text != m_has_text){ + return std::format("system_keyboard_field_has_text() returned {}.", has_text); + } + // An empty field has no white text pixels; one typed character has many. + const size_t text_pixels = system_keyboard_text_pixels(image); + if ((text_pixels >= 20) != m_has_text){ + return std::format("system_keyboard_text_pixels() returned {}.", text_pixels); + } + return true; } private: - std::string m_image; - bool m_keyboard; - int m_cursor_row; - int m_cursor_col; + std::optional m_cursor; bool m_has_text; }; +} // namespace + void add_tests_SystemKeyboardDetector(UnitTestDatabase& database){ - const std::string folder = "NintendoSwitch/SystemKeyboardDetector/"; + const auto path = [](std::string_view image){ + return std::format("NintendoSwitch/SystemKeyboardDetector/{}", image); + }; + const SystemKeyboardKey key_1{0, 0}; // The OK button is blue with text in the field and grey without. - database.add(folder + "BDSP-PlayerName-Empty.png", true, 0, 0, false); - database.add(folder + "BDSP-FriendName-Typed.png", true, 0, 0, true); - database.add(folder + "BDSP-Dialog.png", false, -1, -1, false); - database.add(folder + "HomeMenu.png", false, -1, -1, false); - database.add(folder + "HomeMenu-OnlineLicensePopup.png", false, -1, -1, false); - database.add(folder + "UserSelect.png", false, -1, -1, false); + database.add(path("BDSP-PlayerName-Empty.png"), key_1, false); + database.add(path("BDSP-FriendName-Typed.png"), key_1, true); + for (std::string_view language : {"English", "French", "German", "Italian", "Japanese", "Spanish"}){ + database.add(path(std::format("PLZA-{}-Empty.png", language)), key_1, false); + } + for (std::string_view image : {"BDSP-Dialog.png", "HomeMenu.png", "HomeMenu-OnlineLicensePopup.png", "UserSelect.png"}){ + database.add(path(image), std::nullopt, false); + } + + // Screens still to be captured. Each test is added once its image is in the + // test repo. + struct Planned{ + std::string_view image; + std::optional cursor; + bool has_text; + }; + const Planned planned[] = { + {"PLZA-Cursor-3.png", SystemKeyboardKey{0, 2}, false}, + {"PLZA-Cursor-T.png", SystemKeyboardKey{1, 4}, false}, + {"PLZA-Cursor-Minus.png", SystemKeyboardKey{0, 10}, false}, + {"PLZA-Cursor-Backspace.png", SystemKeyboardKey{0, 11}, true}, + {"PLZA-Cursor-OK.png", SystemKeyboardKey{3, 11}, true}, + {"PLZA-Cursor-Space.png", SystemKeyboardKey{4, 8}, false}, + {"AZERTY-Empty.png", key_1, false}, + {"PLZA-Korean.png", std::nullopt, false}, + }; + for (const Planned& test : planned){ + const std::string image = path(test.image); + if (Filesystem::exists(std::format("{}{}", UNIT_TEST_RESOURCE_PATH(), image))){ + database.add(image, test.cursor, test.has_text); + } + } } -} -} +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h index 7a2952a2ce..5c7e837eaf 100644 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h @@ -3,18 +3,24 @@ * From: https://github.com/PokemonAutomation/ * * The Switch's own on-screen keyboard, which games open for names (e.g. the - * player's name in BDSP): detect it, find the key under its cursor, and tell - * whether its text field is empty. Typing with it is in + * player's name in BDSP and in Legends: Z-A): detect it, read the key under its + * cursor, and tell whether its text field is empty. Typing with it is in * Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h. * - * Measured on 1920x1080 captures of the QWERTY layout. + * Supported: the Latin keyboard (QWERTY; AZERTY shares the key grid but has no + * test screenshot yet). Measured on 1920x1080 Switch 1 captures of BDSP's and + * Legends: Z-A's name entry, the latter in English, Spanish, German, Italian, + * French and Japanese. Not supported: the Korean keyboard, whose candidate strip + * moves the keys down (the detector returns false on it), and probably the + * Chinese ones (not tested). Switch 2 is not tested. * */ #ifndef PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H #define PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H -#include +#include +#include #include "Common/Cpp/TestRunners/UnitTestDatabase.h" #include "CommonFramework/ImageTools/ImageBoxes.h" #include "CommonTools/VisualDetector.h" @@ -23,8 +29,8 @@ namespace PokemonAutomation{ namespace NintendoSwitch{ -// White key panel and bottom bar, and the OK button (blue, or grey while the -// field is empty). +// The white key panel, the white bar below it, and the OK button (blue, or grey +// while the field is empty). Works wherever the cursor is. class SystemKeyboardDetector : public StaticScreenDetector{ public: SystemKeyboardDetector(Color color = COLOR_CYAN); @@ -34,10 +40,16 @@ class SystemKeyboardDetector : public StaticScreenDetector{ private: Color m_color; - ImageFloatBox m_key_panel; + // Two strips across the key panel, one over the gap between rows 0 and 1 and + // one over the gap between rows 2 and 3. The cursor's border reaches into at + // most one of them, so the panel is white if either is. + ImageFloatBox m_key_panel_upper; + ImageFloatBox m_key_panel_lower; ImageFloatBox m_bottom_bar; ImageFloatBox m_ok_button; }; + +// Finds SystemKeyboardDetector on the video, checking every 250 ms. class SystemKeyboardWatcher : public DetectorToFinder{ public: SystemKeyboardWatcher(Color color = COLOR_CYAN) @@ -46,21 +58,40 @@ class SystemKeyboardWatcher : public DetectorToFinder{ }; -// The key under the cursor, as row (0: digits .. 3: "zxcv") and column (0..10). +// A key of the keyboard. +// Rows 0-3 are the character rows (0: digits .. 3: "zxcv"), row 4 is the bottom +// row (globe, Shift, ABC, #+=, the accent pages and Space). +// Columns 0-10 are the character columns. Column 11 is the right column, whose +// keys are taller: backspace is (0, 11), Return (1, 11) and OK (3, 11). +// A row-4 key reports the character column nearest its centre (Space: 8). +struct SystemKeyboardKey{ + uint8_t row; + uint8_t col; + + bool operator==(const SystemKeyboardKey&) const = default; +}; + +// The key under the cursor, or nothing if no cursor border is visible. // The keyboard remembers the cursor between uses, so read it, don't assume it. -// Returns false if no cursor border is visible. -bool find_system_keyboard_cursor(const ImageViewRGB32& screen, int& row, int& col); +std::optional read_system_keyboard_cursor(const ImageViewRGB32& screen); -// Whether the text field holds any text. Pressing B on an empty field closes the +// Whether the text field holds any text: the backspace key is dark then and light +// grey while the field is empty. Pressing B on an empty field closes the // keyboard, so delete only while this is true. bool system_keyboard_field_has_text(const ImageViewRGB32& screen); +// The number of white pixels in the text field. Each typed character adds some +// (a "1" adds about 110 at 1920x1080), so comparing two counts tells whether a +// key press reached the field. +size_t system_keyboard_text_pixels(const ImageViewRGB32& screen); + +// Register the tests of this file. void add_tests_SystemKeyboardDetector(UnitTestDatabase& database); -} -} +} // namespace NintendoSwitch +} // namespace PokemonAutomation #endif diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.h b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.h index 38f4f4d841..1aa290cdc1 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.h +++ b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.h @@ -13,7 +13,7 @@ #include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" #include "NintendoSwitch/NintendoSwitch_ConsoleHandle.h" #include "NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardEntryMappings.h" -#include "NintendoSwitch_CodeEntryTools.h" +#include "NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_CodeEntryTools.h" namespace PokemonAutomation{ namespace NintendoSwitch{ diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.cpp index 43a3329df5..511e36ecad 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.cpp @@ -5,11 +5,19 @@ */ #include +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" #include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" #include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "NintendoSwitch/NintendoSwitch_Settings.h" #include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" #include "NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h" -#include "NintendoSwitch_KeyboardCodeEntry.h" +#include "NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_CodeEntryTools.h" +#include "NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.h" #include "NintendoSwitch_SystemKeyboardTyping.h" namespace PokemonAutomation{ @@ -21,58 +29,202 @@ using namespace std::chrono_literals; namespace{ -constexpr Milliseconds MOVE_HOLD = 100ms; // One key per press, without auto-repeat. +// Waits after each press. They are longer than the code entry delays because +// the next step reads the screen: the cursor animation and the capture latency +// must be over first. constexpr Milliseconds MOVE_RELEASE = 200ms; constexpr Milliseconds WRAP_RELEASE = 400ms; // Wrapping around a side animates for longer. constexpr Milliseconds SETTLE = 300ms; // Before reading the cursor after moving it. -constexpr Milliseconds KEY_HOLD = 80ms; +constexpr Milliseconds SHIFT_RELEASE = 250ms; constexpr Milliseconds KEY_RELEASE = 300ms; // A press made while the keyboard is still sliding in is ignored, so the // cursor is read again after each try until it is on the target. constexpr size_t MAX_MOVE_TRIES = 10; +// Reads of the text field after a key press, SETTLE apart, before giving up. +constexpr size_t MAX_FIELD_READS = 3; -bool read_cursor(VideoStream& stream, ProControllerContext& context, KeyboardEntryPosition& position){ +// The fewest white pixels a typed character adds to the field ("." or "'"). +constexpr size_t MIN_CHARACTER_PIXELS = 10; + + +// The key of each character of `text`. Throws UserSetupError, listing every +// character that has no key, before anything is typed. +std::vector text_keys(Logger& logger, const std::string& text, KeyboardLayout layout){ + const std::map& positions = KEYBOARD_POSITIONS(layout); + std::vector keys; + std::string missing; + for (char ch : text){ + // Letters are listed under their uppercase key. + auto iter = positions.find((char)std::toupper((unsigned char)ch)); + if (iter == positions.end()){ + missing += ch; + }else{ + keys.emplace_back(iter->second); + } + } + if (!missing.empty()){ + throw UserSetupError( + logger, + std::format("System keyboard: \"{}\" has characters with no key on the keyboard: \"{}\".", text, missing) + ); + } + return keys; +} + +// How long to hold each button: the hold of the code entry timings for Switch 1. +Milliseconds button_hold(ProControllerContext& context){ + const ConsoleSettings& settings = ConsoleSettings::instance(); + const CodeEntryDelays delays = context->performance_class() == ControllerPerformanceClass::SerialPABotBase_Wired + ? CodeEntryDelays(settings.CODEBOARD_ENTRY_SWITCH1_WIRED) + : CodeEntryDelays(settings.CODEBOARD_ENTRY_SWITCH1_WIRELESS); + return delays.hold; +} + +// The D-pad direction of a move. +DpadPosition move_direction(CodeEntryAction action){ + switch (action){ + case CodeEntryAction::NORM_MOVE_UP: + case CodeEntryAction::WRAP_MOVE_UP: + return DpadPosition::DPAD_UP; + case CodeEntryAction::NORM_MOVE_RIGHT: + case CodeEntryAction::WRAP_MOVE_RIGHT: + return DpadPosition::DPAD_RIGHT; + case CodeEntryAction::NORM_MOVE_DOWN: + case CodeEntryAction::WRAP_MOVE_DOWN: + return DpadPosition::DPAD_DOWN; + case CodeEntryAction::NORM_MOVE_LEFT: + case CodeEntryAction::WRAP_MOVE_LEFT: + return DpadPosition::DPAD_LEFT; + default: + return DpadPosition::DPAD_NONE; + } +} + +// The moves toward `target`. Off the character grid (the right column or the +// bottom row), one step back onto it, after which the cursor is read again. +std::vector path_toward(SystemKeyboardKey cursor, KeyboardEntryPosition target){ + if (cursor.col == 11){ + return {CodeEntryAction::NORM_MOVE_LEFT}; + } + if (cursor.row == 4){ + return {CodeEntryAction::NORM_MOVE_UP}; + } + std::vector path = keyboard_get_path({cursor.row, cursor.col}, target); + std::erase_if(path, [](CodeEntryAction action){ return !is_move(action); }); + return path; +} + +// The moves as letters for the log, e.g. "DLL" or "R*" for a wrap to the right. +std::string path_to_string(const std::vector& path){ + std::string ret; + for (CodeEntryAction action : path){ + switch (move_direction(action)){ + case DpadPosition::DPAD_UP: + ret += 'U'; + break; + case DpadPosition::DPAD_RIGHT: + ret += 'R'; + break; + case DpadPosition::DPAD_DOWN: + ret += 'D'; + break; + case DpadPosition::DPAD_LEFT: + ret += 'L'; + break; + default: + ret += '?'; + break; + } + if (is_wrap(action)){ + ret += '*'; + } + } + return ret; +} + +std::optional read_cursor(VideoStream& stream, ProControllerContext& context){ context.wait_for_all_requests(); VideoSnapshot screen = stream.video().snapshot(); - int row = -1; - int col = -1; - if (!screen || !find_system_keyboard_cursor(screen, row, col)){ - return false; + if (!screen){ + return std::nullopt; } - position = {(uint8_t)row, (uint8_t)col}; - return true; + return read_system_keyboard_cursor(screen); +} + +[[noreturn]] void throw_cursor_failure( + VideoStream& stream, + std::optional last, KeyboardEntryPosition target, char ch +){ + std::string message = last + ? std::format("the cursor is at ({}, {}), not on '{}' at ({}, {})", last->row, last->col, ch, target.row, target.col) + : std::format("the cursor could not be read while moving to '{}' at ({}, {})", ch, target.row, target.col); + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::format("System keyboard: {}, after {} tries.", message, MAX_MOVE_TRIES), + stream + ); } -void move_cursor(VideoStream& stream, ProControllerContext& context, KeyboardEntryPosition target, char ch){ +void move_cursor( + VideoStream& stream, ProControllerContext& context, Milliseconds hold, + KeyboardEntryPosition target, char ch +){ + std::optional last; for (size_t attempt = 0; attempt < MAX_MOVE_TRIES; attempt++){ - KeyboardEntryPosition cursor; - if (!read_cursor(stream, context, cursor)){ + const std::optional cursor = read_cursor(stream, context); + if (!cursor){ + stream.logger().log(std::format("System keyboard: no cursor seen, try {}.", attempt + 1)); pbf_wait(context, SETTLE); continue; } - if (cursor.row == target.row && cursor.col == target.col){ + last = cursor; + if (cursor->row == target.row && cursor->col == target.col){ return; } - for (CodeEntryAction action : keyboard_get_path(cursor, target)){ - if (!is_move(action)){ - continue; - } - const bool wrap = ((uint8_t)action & 0x80) != 0; - const DpadPosition direction = (DpadPosition)((uint8_t)action & 0x7f); - pbf_press_dpad(context, direction, MOVE_HOLD, wrap ? WRAP_RELEASE : MOVE_RELEASE); + const std::vector path = path_toward(*cursor, target); + stream.logger().log(std::format( + "System keyboard: cursor ({}, {}) -> '{}' ({}, {}): {}, try {}.", + cursor->row, cursor->col, ch, target.row, target.col, path_to_string(path), attempt + 1 + )); + for (CodeEntryAction action : path){ + pbf_press_dpad(context, move_direction(action), hold, is_wrap(action) ? WRAP_RELEASE : MOVE_RELEASE); + } + pbf_wait(context, SETTLE); + } + throw_cursor_failure(stream, last, target, ch); +} + +size_t read_text_pixels(VideoStream& stream, ProControllerContext& context){ + context.wait_for_all_requests(); + VideoSnapshot screen = stream.video().snapshot(); + return screen ? system_keyboard_text_pixels(screen) : 0; +} + +// Types `ch` with the cursor on its key, and checks that the text field grew. +void press_key(VideoStream& stream, ProControllerContext& context, Milliseconds hold, char ch){ + const size_t before = read_text_pixels(stream, context); + if (std::isupper((unsigned char)ch)){ + pbf_press_button(context, BUTTON_LCLICK, hold, SHIFT_RELEASE); + } + pbf_press_button(context, BUTTON_A, hold, KEY_RELEASE); + size_t after = 0; + for (size_t read = 0; read < MAX_FIELD_READS; read++){ + after = read_text_pixels(stream, context); + if (after >= before + MIN_CHARACTER_PIXELS){ + return; } pbf_wait(context, SETTLE); } OperationFailedExceptionWithScreenshot::fire( ErrorReportMode::SEND_ERROR_REPORT, - std::string("System keyboard: the cursor did not reach '") + ch + "'.", + std::format("System keyboard: pressed '{}', but the text field did not change ({} -> {} white pixels).", ch, before, after), stream ); } -} +} // namespace @@ -81,28 +233,17 @@ void type_on_system_keyboard( const std::string& text, KeyboardLayout layout ){ - const std::map& positions = KEYBOARD_POSITIONS(layout); - for (char ch : text){ - // Letters are listed under their uppercase key. - auto iter = positions.find((char)std::toupper((unsigned char)ch)); - if (iter == positions.end()){ - OperationFailedExceptionWithScreenshot::fire( - ErrorReportMode::SEND_ERROR_REPORT, - std::string("System keyboard: no key for '") + ch + "'.", - stream - ); - } - move_cursor(stream, context, iter->second, ch); - if (std::isupper((unsigned char)ch)){ - pbf_press_button(context, BUTTON_LCLICK, 100ms, 250ms); - } - pbf_press_button(context, BUTTON_A, KEY_HOLD, KEY_RELEASE); + const std::vector keys = text_keys(stream.logger(), text, layout); + const Milliseconds hold = button_hold(context); + for (size_t c = 0; c < text.size(); c++){ + move_cursor(stream, context, hold, keys[c], text[c]); + press_key(stream, context, hold, text[c]); } context.wait_for_all_requests(); } -} -} -} +} // namespace FastCodeEntry +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h index b0ec331111..2fe1497e4c 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h +++ b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h @@ -4,8 +4,14 @@ * * Types text on the Switch's on-screen keyboard with a Pro Controller, reading * the cursor from the video before each key. Unlike keyboard_enter_code(), - * which moves blind from an assumed starting key, it works wherever the - * keyboard left its cursor and catches presses the keyboard ignored. + * which moves blind from an assumed starting key, it starts wherever the + * keyboard left its cursor and corrects moves the keyboard ignored. + * + * Supported keyboards are those of NintendoSwitch_SystemKeyboardDetector.h: the + * Latin keyboard, QWERTY or AZERTY. Console-tested on Switch 1 (2026-11-13): + * the player's name in Legends: Z-A in English, Spanish, German, Italian, French + * and Japanese, and earlier the player's and rival's names in BDSP. Not + * supported: the Korean and Chinese keyboards. Switch 2 is not tested. * */ @@ -15,17 +21,27 @@ #include #include "CommonFramework/Tools/VideoStream.h" #include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" -#include "NintendoSwitch_KeyboardEntryMappings.h" +#include "NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardEntryMappings.h" namespace PokemonAutomation{ namespace NintendoSwitch{ namespace FastCodeEntry{ -// Types `text` (letters, digits and the symbols on the keyboard's first page) -// on the open keyboard. An uppercase letter is typed with Shift (left stick -// click). Throws, with a screenshot, for a character that has no key, or if -// the cursor can't be read or doesn't reach a key. +// Types `text` on the open keyboard: letters, digits and the symbols on the +// keyboard's first page. An uppercase letter is typed with Shift (left stick +// click), which assumes the keyboard opened with Shift off, as BDSP's and +// Legends: Z-A's do. +// +// Before the first press, throws UserSetupError if `text` has a character with +// no key. While typing, throws OperationFailedException, with a screenshot, if +// the cursor can't be read or doesn't reach a key, or if a key press doesn't +// add to the text field. The check counts the field's white pixels, so it +// catches a dropped press but not a wrong case. +// +// Presses are held for the "Fast Code Entry Timings" hold of Switch 1 with the +// controller's connection type, as keyboard_enter_code() does. The waits after +// them are fixed, since each move is read back from the video. void type_on_system_keyboard( VideoStream& stream, ProControllerContext& context, const std::string& text, @@ -33,7 +49,7 @@ void type_on_system_keyboard( ); -} -} -} +} // namespace FastCodeEntry +} // namespace NintendoSwitch +} // namespace PokemonAutomation #endif From c44b908bef4c3aa143f8b3dd31dc2c6fb8faed4d Mon Sep 17 00:00:00 2001 From: dayfine Date: Fri, 13 Nov 2026 08:29:56 -0800 Subject: [PATCH 3/5] System keyboard: read the cursor in its dark phase; tests on real cursor frames The cursor border pulses between bright cyan and a darker teal (0, 185, 210); the reader only matched the bright phase, so a cursor on '/' read as nothing. Widen the colour test (red < 130, green > 160, blue > 170, green - red > 60), which still leaves out the OK button and ABC underline blue. Add tests on Legends: Z-A console frames with the cursor on T, /, !, 0, -, backspace, Return, OK, globe, the N-accent key and Space. AZERTY and Korean stay planned: the layout follows the system language, not the game's. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../NintendoSwitch_SystemKeyboardDetector.cpp | 41 +++++++++++++------ .../NintendoSwitch_SystemKeyboardDetector.h | 8 ++-- .../NintendoSwitch_SystemKeyboardTyping.h | 5 ++- 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp index 4f0e37dedf..00600fdf64 100644 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "Common/Cpp/Filesystem/Filesystem.h" #include "CommonFramework/GlobalAutoPaths.h" @@ -38,12 +39,14 @@ constexpr double RIGHT_COLUMN_OK_TOP = 0.725; // digit row to the bottom of the bottom row. const ImageFloatBox KEYS_AREA(0.0, 0.43, 1.0, 0.49); -// The cursor border is cyan (about 80, 230, 210). +// The cursor border pulses between bright cyan (about 50, 240, 220) and a darker +// teal (about 0, 185, 210), measured on Legends: Z-A. The blue of the OK button +// and of the ABC underline (40, 90, 250) is not green enough to match. bool is_cursor_pixel(uint32_t pixel){ const int red = (pixel >> 16) & 0xff; const int green = (pixel >> 8) & 0xff; const int blue = pixel & 0xff; - return red < 150 && green > 200 && blue > 190; + return red < 130 && green > 160 && blue > 170 && green - red > 60; } uint8_t nearest_row(double y){ @@ -222,22 +225,36 @@ void add_tests_SystemKeyboardDetector(UnitTestDatabase& database){ database.add(path(image), std::nullopt, false); } - // Screens still to be captured. Each test is added once its image is in the - // test repo. + // The cursor on other keys (Legends: Z-A, English and Spanish prompts). The + // border pulses between bright cyan and a darker teal; T, / and ! are dark. + const std::pair cursors[] = { + {"PLZA-Cursor-T.png", {1, 4}}, + {"PLZA-Cursor-Slash.png", {1, 10}}, + {"PLZA-Cursor-Exclamation.png", {3, 10}}, + {"PLZA-Cursor-0.png", {0, 9}}, + {"PLZA-Cursor-Minus.png", {0, 10}}, + {"PLZA-Cursor-Backspace.png", {0, 11}}, + {"PLZA-Cursor-Return.png", {1, 11}}, + {"PLZA-Cursor-OK.png", {3, 11}}, + {"PLZA-Cursor-Globe.png", {4, 0}}, + {"PLZA-Cursor-NTilde.png", {4, 5}}, + {"PLZA-Cursor-Space.png", {4, 8}}, + }; + for (const auto& [image, cursor] : cursors){ + database.add(path(image), cursor, false); + } + + // Screens still to be captured: AZERTY needs a console whose system language + // uses it (the layout follows the system, not the game). Each test is added + // once its image is in the test repo. struct Planned{ std::string_view image; std::optional cursor; bool has_text; }; const Planned planned[] = { - {"PLZA-Cursor-3.png", SystemKeyboardKey{0, 2}, false}, - {"PLZA-Cursor-T.png", SystemKeyboardKey{1, 4}, false}, - {"PLZA-Cursor-Minus.png", SystemKeyboardKey{0, 10}, false}, - {"PLZA-Cursor-Backspace.png", SystemKeyboardKey{0, 11}, true}, - {"PLZA-Cursor-OK.png", SystemKeyboardKey{3, 11}, true}, - {"PLZA-Cursor-Space.png", SystemKeyboardKey{4, 8}, false}, - {"AZERTY-Empty.png", key_1, false}, - {"PLZA-Korean.png", std::nullopt, false}, + {"AZERTY-Empty.png", key_1, false}, + {"PLZA-Korean.png", std::nullopt, false}, }; for (const Planned& test : planned){ const std::string image = path(test.image); diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h index 5c7e837eaf..3a5bbf7f03 100644 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h @@ -8,9 +8,11 @@ * Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h. * * Supported: the Latin keyboard (QWERTY; AZERTY shares the key grid but has no - * test screenshot yet). Measured on 1920x1080 Switch 1 captures of BDSP's and - * Legends: Z-A's name entry, the latter in English, Spanish, German, Italian, - * French and Japanese. Not supported: the Korean keyboard, whose candidate strip + * test screenshot yet). Its layout follows the console's system language, not + * the game's. Measured on 1920x1080 Switch 1 captures of BDSP's and Legends: + * Z-A's name entry, the latter with the game in English, Spanish, German, + * Italian, French and Japanese (all QWERTY on an English console). Not + * supported: the Korean keyboard, whose candidate strip * moves the keys down (the detector returns false on it), and probably the * Chinese ones (not tested). Switch 2 is not tested. * diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h index 2fe1497e4c..3d2e2dcc2f 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h +++ b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h @@ -9,8 +9,9 @@ * * Supported keyboards are those of NintendoSwitch_SystemKeyboardDetector.h: the * Latin keyboard, QWERTY or AZERTY. Console-tested on Switch 1 (2026-11-13): - * the player's name in Legends: Z-A in English, Spanish, German, Italian, French - * and Japanese, and earlier the player's and rival's names in BDSP. Not + * the player's name in Legends: Z-A with the game in English, Spanish, German, + * Italian, French and Japanese (QWERTY, English system language), and earlier + * the player's and rival's names in BDSP. AZERTY is not tested. Not * supported: the Korean and Chinese keyboards. Switch 2 is not tested. * */ From c9058545c2722eea40af36dcedbd643d2ebdb66d Mon Sep 17 00:00:00 2001 From: dayfine Date: Fri, 13 Nov 2026 08:30:39 -0800 Subject: [PATCH 4/5] System keyboard tests: split add_tests into cursor and planned helpers (function length) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../NintendoSwitch_SystemKeyboardDetector.cpp | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp index 00600fdf64..b08bee33a4 100644 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp @@ -207,25 +207,13 @@ class Test_SystemKeyboardDetector : public ImageUnitTest{ bool m_has_text; }; -} // namespace - - -void add_tests_SystemKeyboardDetector(UnitTestDatabase& database){ - const auto path = [](std::string_view image){ - return std::format("NintendoSwitch/SystemKeyboardDetector/{}", image); - }; - const SystemKeyboardKey key_1{0, 0}; - // The OK button is blue with text in the field and grey without. - database.add(path("BDSP-PlayerName-Empty.png"), key_1, false); - database.add(path("BDSP-FriendName-Typed.png"), key_1, true); - for (std::string_view language : {"English", "French", "German", "Italian", "Japanese", "Spanish"}){ - database.add(path(std::format("PLZA-{}-Empty.png", language)), key_1, false); - } - for (std::string_view image : {"BDSP-Dialog.png", "HomeMenu.png", "HomeMenu-OnlineLicensePopup.png", "UserSelect.png"}){ - database.add(path(image), std::nullopt, false); - } +std::string test_image_path(std::string_view image){ + return std::format("NintendoSwitch/SystemKeyboardDetector/{}", image); +} - // The cursor on other keys (Legends: Z-A, English and Spanish prompts). The +// The cursor on keys other than "1". +void add_cursor_tests(UnitTestDatabase& database){ + // Legends: Z-A, English and Spanish prompts. The // border pulses between bright cyan and a darker teal; T, / and ! are dark. const std::pair cursors[] = { {"PLZA-Cursor-T.png", {1, 4}}, @@ -241,9 +229,13 @@ void add_tests_SystemKeyboardDetector(UnitTestDatabase& database){ {"PLZA-Cursor-Space.png", {4, 8}}, }; for (const auto& [image, cursor] : cursors){ - database.add(path(image), cursor, false); + database.add(test_image_path(image), cursor, false); } +} +// Tests whose screen is not captured yet. +void add_planned_tests(UnitTestDatabase& database){ + const SystemKeyboardKey key_1{0, 0}; // Screens still to be captured: AZERTY needs a console whose system language // uses it (the layout follows the system, not the game). Each test is added // once its image is in the test repo. @@ -257,13 +249,31 @@ void add_tests_SystemKeyboardDetector(UnitTestDatabase& database){ {"PLZA-Korean.png", std::nullopt, false}, }; for (const Planned& test : planned){ - const std::string image = path(test.image); + const std::string image = test_image_path(test.image); if (Filesystem::exists(std::format("{}{}", UNIT_TEST_RESOURCE_PATH(), image))){ database.add(image, test.cursor, test.has_text); } } } +} // namespace + + +void add_tests_SystemKeyboardDetector(UnitTestDatabase& database){ + const SystemKeyboardKey key_1{0, 0}; + // The OK button is blue with text in the field and grey without. + database.add(test_image_path("BDSP-PlayerName-Empty.png"), key_1, false); + database.add(test_image_path("BDSP-FriendName-Typed.png"), key_1, true); + for (std::string_view language : {"English", "French", "German", "Italian", "Japanese", "Spanish"}){ + database.add(test_image_path(std::format("PLZA-{}-Empty.png", language)), key_1, false); + } + for (std::string_view image : {"BDSP-Dialog.png", "HomeMenu.png", "HomeMenu-OnlineLicensePopup.png", "UserSelect.png"}){ + database.add(test_image_path(image), std::nullopt, false); + } + add_cursor_tests(database); + add_planned_tests(database); +} + } // namespace NintendoSwitch From a730195c378d10f89d976757de6504ff6b7a949c Mon Sep 17 00:00:00 2001 From: dayfine Date: Fri, 13 Nov 2026 23:16:53 -0800 Subject: [PATCH 5/5] CI (fork only): CommandLineTests from dayfine/CommandLineTests The SystemKeyboardDetector screenshots are on the fork's CommandLineTests main, not upstream's, so the image tests could not load them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .github/workflows/cpp-ci-serial-programs-base.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index efb3c80a04..f8a299851d 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -245,7 +245,9 @@ jobs: uses: actions/checkout@v7 if: inputs.run-tests with: - repository: 'PokemonAutomation/CommandLineTests' + # The fork's fixtures: upstream's plus the SystemKeyboardDetector screenshots. + repository: 'dayfine/CommandLineTests' + ref: '5de4ff774f9141d904c4b5c557fa7b9dc85eaef1' path: 'CommandLineTests' # The unit tests assume the downloadable resources they need are already downloaded