From dcdf8a9ec8c3eae96e8f0853b8a4b03775b13462 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sat, 14 Nov 2026 01:15:53 -0800 Subject: [PATCH 1/6] AutoStory: shared launch from HOME to the overworld on a save One driver for every game: a per-game StoryLaunchPolicy (HOME tile check, title, backup-save presses, prompts such as the license check, the overworld judge) and launch_story_game(), which looks, does one thing, counts each screen's presses and ends only on a settled overworld. It never starts a new game. No game uses it yet. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../AutoStory/Engine/Pokemon_StoryLaunch.cpp | 246 ++++++++++++++++ .../AutoStory/Engine/Pokemon_StoryLaunch.h | 140 ++++++++++ .../Engine/Pokemon_StoryLaunch_Tests.cpp | 262 ++++++++++++++++++ .../Engine/Pokemon_StoryLaunch_Tests.h | 23 ++ .../AutoStory/Pokemon_AutoStory_Tests.h | 2 + SerialPrograms/cmake/SourceFiles.cmake | 4 + 6 files changed, 677 insertions(+) create mode 100644 SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.cpp create mode 100644 SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h create mode 100644 SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp create mode 100644 SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.h diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.cpp new file mode 100644 index 0000000000..0d94676521 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.cpp @@ -0,0 +1,246 @@ +/* Story Launch + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include "Pokemon_StoryLaunch.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + +using namespace NintendoSwitch; + + +namespace{ + + +// What a look shows, as far as a launch cares. +enum class LaunchScreen{ + HOME, + NEW_GAME, + PROMPT, + TITLE, + OVERWORLD, + OTHER, +}; + +struct LaunchLook{ + LaunchScreen screen = LaunchScreen::OTHER; + // The prompt on screen, for PROMPT. + const StoryLaunchPrompt* prompt = nullptr; +}; + +// `look` as `policy` reads it. HOME first: a game's detectors may read +// HOME's plain screen as a quiet overworld. +LaunchLook read_launch_look(const StoryLaunchPolicy& policy, const ScreenObservation& look){ + if (look.events.has(ScreenEvent::SWITCH_HOME_MENU)){ + return {.screen = LaunchScreen::HOME}; + } + if (policy.new_game && policy.new_game(look)){ + return {.screen = LaunchScreen::NEW_GAME}; + } + for (const StoryLaunchPrompt& prompt : policy.prompts){ + if (prompt.matches && prompt.matches(look)){ + return {.screen = LaunchScreen::PROMPT, .prompt = &prompt}; + } + } + if (policy.title(look)){ + return {.screen = LaunchScreen::TITLE}; + } + const bool overworld = policy.overworld ? policy.overworld(look) : look.events.has(ScreenEvent::PLAYER_CAN_MOVE); + return {.screen = overworld ? LaunchScreen::OVERWORLD : LaunchScreen::OTHER}; +} + +Outcome launch_failed(std::string detail){ + return outcome_failed(std::string(STORY_LAUNCH_FAILED), std::move(detail)); +} + +// Why `policy` cannot launch `request`, before any press; nothing when it +// can. +std::optional launch_problem(const StoryLaunchPolicy& policy, StoryLaunchRequest request){ + if (!policy.tile_selected || !policy.title){ + return launch_failed(std::format("{}: the launch policy has no tile or title check", policy.game)); + } + if (request.save == StoryLaunchRequest::Save::BACKUP && policy.backup_save.empty()){ + return launch_failed(std::format("{}: the game has no backup save", policy.game)); + } + return std::nullopt; +} + + +// One launch: a look, one thing done about it, until the overworld has +// settled or a screen took too many presses. +template +class LaunchDriver{ +public: + LaunchDriver(StoryBackend& backend, const StoryLaunchPolicy& policy, StoryLaunchRequest request) + : m_backend(backend) + , m_policy(policy) + , m_request(request) + {} + + // One look and its answer; the launch's end when this look ended it. + std::optional step(){ + const Observation observation = m_backend.observe(); + const LaunchLook look = read_launch_look(m_policy, observation); + if (look.screen != LaunchScreen::OVERWORLD){ + m_settled = 0; + } + switch (look.screen){ + case LaunchScreen::HOME: + return on_home(observation); + case LaunchScreen::NEW_GAME: + return launch_failed(std::format("{}: the game shows a new game: no save to load, and a launch never starts one", m_policy.game)); + case LaunchScreen::PROMPT: + return press(look.prompt->name, look.prompt->press, look.prompt->max_presses); + case LaunchScreen::TITLE: + return on_title(); + case LaunchScreen::OVERWORLD: + return on_overworld(); + case LaunchScreen::OTHER: + break; + } + return on_other(); + } + +private: + // Whether an overworld up now is not yet the save asked for. + bool needs_load() const{ + return !m_loading && (m_request.running == StoryLaunchRequest::Running::RESTART || m_request.save == StoryLaunchRequest::Save::BACKUP); + } + + // The cursor to the game's tile, then the game closed if it must + // load again, then A on the tile. + std::optional on_home(const ScreenObservation& observation){ + if (!m_policy.tile_selected(observation)){ + return move_to_tile(); + } + if (needs_load()){ + m_loading = true; + return press("home", {.button = BUTTON_X}, m_policy.presses_per_screen, "close the game"); + } + m_launched = true; + return press("home", {.button = BUTTON_A}, m_policy.presses_per_screen, "launch the game"); + } + + std::optional move_to_tile(){ + const size_t left = m_policy.tile_search; + if (m_tile_moves >= 3 * left){ + return launch_failed(std::format("{}: the game's tile is not on HOME: {} tiles searched", m_policy.game, m_tile_moves)); + } + const Button button = m_tile_moves < left ? BUTTON_LEFT : BUTTON_RIGHT; + m_tile_moves++; + log("home", std::format("not on the game's tile: {}", button_to_string(button))); + m_backend.press(button, Milliseconds(80), Milliseconds(400)); + return std::nullopt; + } + + std::optional on_title(){ + m_loading = true; + m_launched = true; + if (m_request.save == StoryLaunchRequest::Save::BACKUP && !m_backup_sent){ + m_backup_sent = true; + log("title", "the backup save"); + for (const StoryLaunchPress& each : m_policy.backup_save){ + m_backend.press(each.button, each.hold, each.release); + } + } + return press("title", m_policy.title_press, m_policy.presses_per_screen); + } + + std::optional on_overworld(){ + if (needs_load()){ + return press("overworld", {.button = BUTTON_HOME}, m_policy.presses_per_screen, "HOME, to load the save"); + } + if (++m_settled >= m_policy.settled_looks){ + log("overworld", std::format("settled for {} looks", m_settled)); + return outcome_ok(); + } + m_backend.wait(m_policy.poll); + return std::nullopt; + } + + // A screen the policy doesn't know: the game's boot or a load once + // launched; before that the game was in front already, left with HOME + // to load the save, or B toward its overworld. + std::optional on_other(){ + if (m_launched || m_loading){ + m_backend.wait(m_policy.poll); + return std::nullopt; + } + if (needs_load()){ + return press("unknown screen", {.button = BUTTON_HOME}, m_policy.presses_per_screen, "HOME, to load the save"); + } + return press("unknown screen", {.button = BUTTON_B}, m_policy.presses_per_screen, "B, toward the overworld"); + } + + // `what` on `screen`, unless `screen` already took `max` presses. + std::optional press(std::string_view screen, const StoryLaunchPress& what, size_t max, std::string_view why = {}){ + size_t& count = m_presses[std::string(screen)]; + if (count >= max){ + return launch_failed(std::format("{}: stuck on {}: {} presses changed nothing", m_policy.game, screen, count)); + } + count++; + log(screen, why.empty() ? button_to_string(what.button) : std::string(why)); + m_backend.press(what.button, what.hold, what.release); + return std::nullopt; + } + + void log(std::string_view screen, std::string_view what){ + m_backend.log(std::format("[Launch] {}: {}: {}", m_policy.game, screen, what)); + } + + StoryBackend& m_backend; + const StoryLaunchPolicy& m_policy; + StoryLaunchRequest m_request; + std::map> m_presses; + size_t m_tile_moves = 0; + size_t m_settled = 0; + // The game was closed from HOME, or showed its title: what comes up + // next is a save loading. + bool m_loading = false; + // A on the game's tile, or the title: until the overworld, a screen + // the policy doesn't know is the game booting or loading. + bool m_launched = false; + bool m_backup_sent = false; +}; + + +} // namespace + + +template +Outcome launch_story_game(StoryBackend& backend, const StoryLaunchPolicy& policy, StoryLaunchRequest request){ + if (std::optional problem = launch_problem(policy, request)){ + backend.log(std::format("[Launch] {}", problem->detail)); + return std::move(*problem); + } + LaunchDriver driver(backend, policy, request); + for (size_t looks = 0; looks < policy.max_looks; looks++){ + backend.scope().throw_if_cancelled(); + if (std::optional end = driver.step()){ + if (!end->succeeded()){ + backend.log(std::format("[Launch] {}", end->detail)); + } + return std::move(*end); + } + } + const Outcome failed = launch_failed(std::format("{}: no overworld after {} looks", policy.game, policy.max_looks)); + backend.log(std::format("[Launch] {}", failed.detail)); + return failed; +} + +template Outcome launch_story_game(StoryBackend&, const StoryLaunchPolicy&, StoryLaunchRequest); +template Outcome launch_story_game(StoryBackend&, const StoryLaunchPolicy&, StoryLaunchRequest); + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h new file mode 100644 index 0000000000..88c17db59e --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h @@ -0,0 +1,140 @@ +/* Story Launch + * + * From: https://github.com/PokemonAutomation/ + * + * A run's first event: from wherever the console is (the Switch's HOME + * menu with the game closed or suspended, the game's title, the + * overworld) to the overworld on a save. One driver for every game; the + * game's StoryLaunchPolicy says how its screens look and which press + * answers each. It runs before the first checkpoint, on its own, never + * inside a new game's language or intro steps, and it never starts a new + * game. + * + * The driver looks, does one thing, and looks again. It does not need to + * know every screen: one it doesn't know is waited out (a boot, a fade) + * or, before the game was launched, left with HOME or B. Each screen's + * presses are counted, so a screen that ignores its press fails the + * launch instead of looping, and the launch ends only once the overworld + * shows for several looks in a row. + * + */ + +#ifndef PokemonAutomation_Pokemon_StoryLaunch_H +#define PokemonAutomation_Pokemon_StoryLaunch_H + +#include +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "CommonTools/ScreenMachine/ScreenObservation.h" +#include "NintendoSwitch/Controllers/NintendoSwitch_ControllerButtons.h" +#include "Pokemon/AutoStory/Pokemon_StoryOutcome.h" +#include "Pokemon/AutoStory/Types/Pokemon_GridPose.h" +#include "Pokemon_StoryBackend.h" +#include "Pokemon_ZonePose.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// What a launch loads. +struct StoryLaunchRequest{ + enum class Save{ + LATEST, // the title's plain load + BACKUP, // the policy's backup_save presses on the title first + // (PLZA: Up, then B + X, the save before the latest) + }; + enum class Running{ + // A game already on the overworld is the start: it is resumed + // from HOME, not reloaded. + KEEP, + // A game already running is closed from HOME, then launched, so + // the save loads; a title on screen already counts as launched. + RESTART, + }; + Save save = Save::LATEST; + Running running = Running::RESTART; +}; + +// One press: `button` (several pressed together when or-ed), held for +// `hold`, then let go for `release`. +struct StoryLaunchPress{ + NintendoSwitch::Button button = NintendoSwitch::BUTTON_A; + Milliseconds hold = Milliseconds(80); + Milliseconds release = Milliseconds(400); +}; + +// A screen on the way that one press answers: the user select, an online +// license check, HOME's close-software dialog, the load menu's Continue, +// a recap of the story so far. +struct StoryLaunchPrompt{ + std::string name = {}; + std::function matches = {}; + StoryLaunchPress press = {}; + // Presses it may take before the launch is stuck on it. + size_t max_presses = 6; +}; + +// How one game's screens look on the way from HOME to its overworld. +struct StoryLaunchPolicy{ + // "frlg", "bdsp", "plza", for the log. + std::string game = {}; + // On HOME: whether the cursor is on this game's tile. The launch + // presses A on HOME only then, as A launches whatever tile is + // selected. Required. + std::function tile_selected = {}; + // How far the tile search moves each way: Left `tile_search` times + // (the most recently played games are leftmost), then Right twice as + // many. + size_t tile_search = 6; + // The title screen, which leads to the save. Required. + std::function title = {}; + StoryLaunchPress title_press = {}; + // Pressed once on the title before `title_press` for Save::BACKUP. + // Empty: the game has no backup save, and a BACKUP launch fails + // before it presses anything. + std::vector backup_save = {}; + // A screen that would start a new game: there is no save. The launch + // fails there without a press. Optional. + std::function new_game = {}; + // The game's other screens on the way, each answered by its press. + std::vector prompts = {}; + // The player in control. Unset: the look shows PLAYER_CAN_MOVE. + std::function overworld = {}; + // Looks in a row the overworld must show before the launch is done. + size_t settled_looks = 3; + // The wait between looks at a screen with nothing to press. + Milliseconds poll = Milliseconds(500); + // Presses HOME, the title, an overworld to leave or a screen the + // policy doesn't know may take before the launch is stuck there. + size_t presses_per_screen = 6; + // Looks before the launch gives up: about `poll` times this of + // loading at most. + size_t max_looks = 360; +}; + +// The failure of a launch that did not reach the overworld; its detail +// says where it stopped. Nothing in the story answers it: no reload, no +// recovery. +inline constexpr std::string_view STORY_LAUNCH_FAILED = "launch_failed"; + +// Drives `backend` to the overworld on the save `request` asks for, by +// `policy`; logs each press as "[Launch] : : ". +// Succeeds once the overworld shows `policy.settled_looks` looks in a +// row; fails as STORY_LAUNCH_FAILED on a new game, a missing tile, a +// screen that took too many presses, or after `policy.max_looks` looks. +template +Outcome launch_story_game(StoryBackend& backend, const StoryLaunchPolicy& policy, StoryLaunchRequest request); + +// Instantiated once, in Pokemon_StoryLaunch.cpp. +extern template Outcome launch_story_game(StoryBackend&, const StoryLaunchPolicy&, StoryLaunchRequest); +// And for ZonePose (Pokemon_ZonePose.h). +extern template Outcome launch_story_game(StoryBackend&, const StoryLaunchPolicy&, StoryLaunchRequest); + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp new file mode 100644 index 0000000000..dbc79818dc --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp @@ -0,0 +1,262 @@ +/* Story Launch Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The launch on a scripted console: each case is the screens the console + * shows, look by look (the last repeats), and the presses the launch must + * make, or where it must fail. The test game's screens are readings its + * policy reads; HOME is the shared SWITCH_HOME_MENU event and the + * overworld the shared PLAYER_CAN_MOVE. + * + */ + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "Pokemon_StoryFakeBackend.h" +#include "Pokemon_StoryLaunch.h" +#include "Pokemon_StoryLaunch_Tests.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + +using namespace NintendoSwitch; + + +namespace{ + + +// What the scripted console shows. +enum class Screen{ + HOME_ON_GAME, // HOME, the cursor on the game's tile + HOME_OFF_GAME, // HOME, the cursor on another tile + CLOSE_DIALOG, // HOME's "Close software?" + LICENSE_CHECK, // the online license check + BLACK, // a boot or a load + TITLE, + NEW_GAME, // the title's menu without a save + BACKUP_CONFIRM, // "Load the backup save?" + LOAD_MENU, // Continue / New Game + OVERWORLD, +}; + +inline constexpr std::string_view SCREEN = "screen"; +inline constexpr std::string_view TILE = "home.tile"; + +Observation look(Screen screen){ + Observation observation; + switch (screen){ + case Screen::HOME_ON_GAME: + case Screen::HOME_OFF_GAME: + observation.events = ScreenEvents{ScreenEvent::SWITCH_HOME_MENU}; + observation.readings[std::string(TILE)] = screen == Screen::HOME_ON_GAME ? 1 : 0; + return observation; + case Screen::OVERWORLD: + observation.events = ScreenEvents{ScreenEvent::PLAYER_CAN_MOVE}; + return observation; + default: + observation.readings[std::string(SCREEN)] = (int)screen; + return observation; + } +} + +std::function shows(Screen screen){ + return [screen](const ScreenObservation& observation){ + const auto it = observation.readings.find(SCREEN); + return it != observation.readings.end() && it->second == (int)screen; + }; +} + +// Whether the test game has a backup save. +enum class Backup{ + PLZA, // Up, then B + X + NONE, +}; + +// The test game: A on the title, `backup` for the backup save, A on each +// prompt, and small bounds so a failure takes few looks. +StoryLaunchPolicy test_policy(Backup backup){ + StoryLaunchPolicy policy; + policy.game = "test"; + policy.tile_selected = [](const ScreenObservation& observation){ + const auto it = observation.readings.find(TILE); + return it != observation.readings.end() && it->second == 1; + }; + policy.tile_search = 2; + policy.title = shows(Screen::TITLE); + if (backup == Backup::PLZA){ + policy.backup_save = {{.button = BUTTON_UP}, {.button = BUTTON_B | BUTTON_X, .hold = Milliseconds(1000)}}; + } + policy.new_game = shows(Screen::NEW_GAME); + policy.prompts = { + {.name = "close software", .matches = shows(Screen::CLOSE_DIALOG)}, + {.name = "license check", .matches = shows(Screen::LICENSE_CHECK)}, + {.name = "backup confirm", .matches = shows(Screen::BACKUP_CONFIRM)}, + {.name = "load menu", .matches = shows(Screen::LOAD_MENU)}, + }; + policy.presses_per_screen = 3; + policy.max_looks = 12; + return policy; +} + +// `screens`, then the overworld for as many looks as it takes to settle. +std::vector then_settled(std::vector screens){ + screens.insert(screens.end(), {Screen::OVERWORLD, Screen::OVERWORLD, Screen::OVERWORLD}); + return screens; +} + +using Save = StoryLaunchRequest::Save; +using Running = StoryLaunchRequest::Running; + +// One launch. +struct LaunchCase{ + struct Input{ + std::vector screens; + StoryLaunchRequest request = {}; + Backup backup = Backup::PLZA; + }; + struct Expectation{ + // Empty: the launch succeeds; else a part of its failure's detail. + std::string_view failure = {}; + // The inputs in order, by space: a button ("A", "HOME", "B+X") + // or "wait". + std::string_view inputs = {}; + }; + std::string_view name; + Input input; + Expectation expectation; +}; + +const std::vector LAUNCH_CASES = { + {.name = "game closed on HOME", + .input = {.screens = then_settled({Screen::HOME_ON_GAME, Screen::HOME_ON_GAME, Screen::BLACK, Screen::TITLE, Screen::BLACK})}, + .expectation = {.inputs = "X A wait A wait wait wait"}}, + {.name = "game open on its title", + .input = {.screens = then_settled({Screen::TITLE, Screen::BLACK})}, + .expectation = {.inputs = "A wait wait wait"}}, + {.name = "game on the overworld kept", + .input = {.screens = then_settled({}), .request = {.running = Running::KEEP}}, + .expectation = {.inputs = "wait wait"}}, + {.name = "game on the overworld restarted", + .input = {.screens = then_settled({Screen::OVERWORLD, Screen::HOME_ON_GAME, Screen::CLOSE_DIALOG, Screen::HOME_ON_GAME, + Screen::BLACK, Screen::TITLE})}, + .expectation = {.inputs = "HOME X A A wait A wait wait"}}, + {.name = "backup save from the overworld", + .input = {.screens = then_settled({Screen::OVERWORLD, Screen::HOME_ON_GAME, Screen::HOME_ON_GAME, Screen::TITLE, + Screen::BACKUP_CONFIRM, Screen::BLACK}), + .request = {.save = Save::BACKUP, .running = Running::KEEP}}, + .expectation = {.inputs = "HOME X A Up B+X A A wait wait wait"}}, + {.name = "backup save the game does not have", + .input = {.screens = then_settled({}), .request = {.save = Save::BACKUP}, .backup = Backup::NONE}, + .expectation = {.failure = "no backup save", .inputs = ""}}, + {.name = "license check back to HOME", + .input = {.screens = then_settled({Screen::HOME_ON_GAME, Screen::HOME_ON_GAME, Screen::LICENSE_CHECK, Screen::HOME_ON_GAME, + Screen::BLACK, Screen::TITLE, Screen::LOAD_MENU})}, + .expectation = {.inputs = "X A A A wait A A wait wait"}}, + {.name = "cursor on another tile", + .input = {.screens = then_settled({Screen::HOME_OFF_GAME, Screen::HOME_OFF_GAME, Screen::HOME_ON_GAME}), + .request = {.running = Running::KEEP}}, + .expectation = {.inputs = "Left Left A wait wait"}}, + {.name = "game not on HOME", + .input = {.screens = {Screen::HOME_OFF_GAME}}, + .expectation = {.failure = "tile is not on HOME", .inputs = "Left Left Right Right Right Right"}}, + {.name = "title ignores A", + .input = {.screens = {Screen::TITLE}}, + .expectation = {.failure = "stuck on title", .inputs = "A A A"}}, + {.name = "no save never starts a new game", + .input = {.screens = {Screen::NEW_GAME}}, + .expectation = {.failure = "new game", .inputs = ""}}, + {.name = "black screen after launch times out", + .input = {.screens = {Screen::HOME_ON_GAME, Screen::HOME_ON_GAME, Screen::BLACK}}, + .expectation = {.failure = "no overworld after 12 looks", .inputs = "X A wait wait wait wait wait wait wait wait wait wait"}}, + {.name = "kept game off the overworld takes B", + .input = {.screens = then_settled({Screen::LOAD_MENU, Screen::BLACK}), .request = {.running = Running::KEEP}}, + .expectation = {.inputs = "A B wait wait"}}, +}; + +// The fake backend's record of `input` ("A" -> "press A"). +std::string recorded(std::string_view input){ + if (input == "wait"){ + return "wait 500ms"; + } + static const std::map BUTTONS = { + {"A", BUTTON_A}, {"B", BUTTON_B}, {"X", BUTTON_X}, {"HOME", BUTTON_HOME}, + {"Up", BUTTON_UP}, {"Left", BUTTON_LEFT}, {"Right", BUTTON_RIGHT}, {"B+X", BUTTON_B | BUTTON_X}, + }; + const auto it = BUTTONS.find(input); + return it == BUTTONS.end() ? std::format("unknown input {}", input) : std::format("press {}", button_to_string(it->second)); +} + +// `inputs` (by space) as the fake backend records them. +std::vector expected_inputs(std::string_view inputs){ + std::vector expected; + while (!inputs.empty()){ + const size_t space = inputs.find(' '); + expected.push_back(recorded(inputs.substr(0, space))); + inputs = space == std::string_view::npos ? std::string_view() : inputs.substr(space + 1); + } + return expected; +} + +std::string describe_inputs(const std::vector& inputs){ + std::string text; + for (const std::string& input : inputs){ + text += text.empty() ? input : std::format(", {}", input); + } + return std::format("[{}]", text); +} + +std::string check_launch(const LaunchCase& test, CancellableScope& scope){ + std::vector> looks; + for (Screen screen : test.input.screens){ + looks.push_back(look(screen)); + } + FakeStoryBackend backend({.looks = std::move(looks)}, scope); + const Outcome outcome = launch_story_game(backend, test_policy(test.input.backup), test.input.request); + if (test.expectation.failure.empty() && !outcome.succeeded()){ + return std::format("expected success, got {}", to_string(outcome)); + } + if (!test.expectation.failure.empty()){ + if (outcome.failure != STORY_LAUNCH_FAILED || outcome.detail.find(test.expectation.failure) == std::string::npos){ + return std::format("expected a launch failure with \"{}\", got {}", test.expectation.failure, to_string(outcome)); + } + } + const std::vector expected = expected_inputs(test.expectation.inputs); + if (backend.inputs() != expected){ + return std::format("expected inputs {}, got {}", describe_inputs(expected), describe_inputs(backend.inputs())); + } + return ""; +} + +class StoryLaunchTest : public UnitTest{ +public: + explicit StoryLaunchTest(LaunchCase test) + : UnitTest(std::format("Pokemon::StoryLaunch - {}", test.name)) + , m_case(std::move(test)) + {} + virtual UnitTestResult run(Logger&, CancellableScope& scope) const override{ + const std::string error = check_launch(m_case, scope); + return error.empty() ? UnitTestResult(true) : UnitTestResult(error); + } +private: + LaunchCase m_case; +}; + + +} // namespace + + +void add_tests_StoryLaunch(UnitTestDatabase& database){ + for (const LaunchCase& test : LAUNCH_CASES){ + database.add(test); + } +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.h b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.h new file mode 100644 index 0000000000..efba4cc847 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.h @@ -0,0 +1,23 @@ +/* Story Launch Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_Pokemon_StoryLaunch_Tests_H +#define PokemonAutomation_Pokemon_StoryLaunch_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// Registers unit tests for the launch from HOME to the overworld on a scripted console. +void add_tests_StoryLaunch(UnitTestDatabase& database); + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h index b19c74a333..9d9a1e62d3 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h @@ -15,6 +15,7 @@ #include "Data/Pokemon_AuthoredCheckpointJson_Tests.h" #include "Data/Pokemon_StoryStepJson_Tests.h" #include "Engine/Pokemon_StoryEnginePolicy_Tests.h" +#include "Engine/Pokemon_StoryLaunch_Tests.h" #include "Engine/Pokemon_StoryEngineRuns_Tests.h" #include "Engine/Pokemon_StoryEngine_Tests.h" #include "Pokemon/Options/Pokemon_StoryRangeOptions_Tests.h" @@ -58,6 +59,7 @@ inline void add_tests_AutoStory(UnitTestDatabase& database){ add_tests_StoryEngine(database); add_tests_StoryEnginePolicy(database); add_tests_StoryEngineRuns(database); + add_tests_StoryLaunch(database); add_tests_SceneJudges(database); add_tests_Interrupts(database); add_tests_Transitions(database); diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index d9273c6824..72619f0cab 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1582,6 +1582,10 @@ file(GLOB LIBRARY_SOURCES Source/Pokemon/AutoStory/Engine/Pokemon_StoryFakeBackend.cpp Source/Pokemon/AutoStory/Engine/Pokemon_StoryFakeBackend.h Source/Pokemon/AutoStory/Engine/Pokemon_StoryGame.h + Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.cpp + Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h + Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp + Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.h Source/Pokemon/AutoStory/Engine/Pokemon_StoryPoseConcept.cpp Source/Pokemon/AutoStory/Engine/Pokemon_StoryPoseConcept.h Source/Pokemon/AutoStory/Engine/Pokemon_StoryStepRunner.cpp From 16cd404c6428267bb6bdd575fd0249bfcd14c10e Mon Sep 17 00:00:00 2001 From: dayfine Date: Sat, 14 Nov 2026 01:46:10 -0800 Subject: [PATCH 2/6] AutoStory launch: a prompt's lead-in presses; game screens read only after the launch The update prompt needs the cursor moved before A, and PLZA's title and dialog checks are loose enough to match the game in play, where a press would answer the story. StoryLaunchPrompt::before presses before the prompt's press; StoryLaunchPrompt::after_launch and StoryLaunchPolicy::title_after_launch read those screens only once the launch closed or launched the game. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../AutoStory/Engine/Pokemon_StoryLaunch.cpp | 25 ++++++++++----- .../AutoStory/Engine/Pokemon_StoryLaunch.h | 11 +++++++ .../Engine/Pokemon_StoryLaunch_Tests.cpp | 31 +++++++++++++++++-- 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.cpp index 0d94676521..accab8a6d6 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.cpp +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.cpp @@ -37,9 +37,10 @@ struct LaunchLook{ const StoryLaunchPrompt* prompt = nullptr; }; -// `look` as `policy` reads it. HOME first: a game's detectors may read -// HOME's plain screen as a quiet overworld. -LaunchLook read_launch_look(const StoryLaunchPolicy& policy, const ScreenObservation& look){ +// `look` as `policy` reads it, `launched` once the launch launched or +// closed the game. HOME first: a game's detectors may read HOME's plain +// screen as a quiet overworld. +LaunchLook read_launch_look(const StoryLaunchPolicy& policy, const ScreenObservation& look, bool launched){ if (look.events.has(ScreenEvent::SWITCH_HOME_MENU)){ return {.screen = LaunchScreen::HOME}; } @@ -47,11 +48,11 @@ LaunchLook read_launch_look(const StoryLaunchPolicy& policy, const ScreenObserva return {.screen = LaunchScreen::NEW_GAME}; } for (const StoryLaunchPrompt& prompt : policy.prompts){ - if (prompt.matches && prompt.matches(look)){ + if ((launched || !prompt.after_launch) && prompt.matches && prompt.matches(look)){ return {.screen = LaunchScreen::PROMPT, .prompt = &prompt}; } } - if (policy.title(look)){ + if ((launched || !policy.title_after_launch) && policy.title(look)){ return {.screen = LaunchScreen::TITLE}; } const bool overworld = policy.overworld ? policy.overworld(look) : look.events.has(ScreenEvent::PLAYER_CAN_MOVE); @@ -89,7 +90,7 @@ class LaunchDriver{ // One look and its answer; the launch's end when this look ended it. std::optional step(){ const Observation observation = m_backend.observe(); - const LaunchLook look = read_launch_look(m_policy, observation); + const LaunchLook look = read_launch_look(m_policy, observation, m_launched || m_loading); if (look.screen != LaunchScreen::OVERWORLD){ m_settled = 0; } @@ -99,7 +100,7 @@ class LaunchDriver{ case LaunchScreen::NEW_GAME: return launch_failed(std::format("{}: the game shows a new game: no save to load, and a launch never starts one", m_policy.game)); case LaunchScreen::PROMPT: - return press(look.prompt->name, look.prompt->press, look.prompt->max_presses); + return on_prompt(*look.prompt); case LaunchScreen::TITLE: return on_title(); case LaunchScreen::OVERWORLD: @@ -142,6 +143,16 @@ class LaunchDriver{ return std::nullopt; } + // The prompt's lead-in presses, then its press. + std::optional on_prompt(const StoryLaunchPrompt& prompt){ + if (m_presses[prompt.name] < prompt.max_presses){ + for (const StoryLaunchPress& each : prompt.before){ + m_backend.press(each.button, each.hold, each.release); + } + } + return press(prompt.name, prompt.press, prompt.max_presses); + } + std::optional on_title(){ m_loading = true; m_launched = true; diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h index 88c17db59e..84b79ccc55 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h @@ -73,6 +73,13 @@ struct StoryLaunchPrompt{ std::string name = {}; std::function matches = {}; StoryLaunchPress press = {}; + // Pressed in order before `press`, each time: the cursor moves to the + // choice `press` picks (the update prompt's last choice). + std::vector before = {}; + // Read only once this launch launched or closed the game: a screen + // the game also shows in play (a dialog box, a choice), where a press + // before the launch would answer the story instead. + bool after_launch = false; // Presses it may take before the launch is stuck on it. size_t max_presses = 6; }; @@ -92,6 +99,10 @@ struct StoryLaunchPolicy{ // The title screen, which leads to the save. Required. std::function title = {}; StoryLaunchPress title_press = {}; + // `title` is read only once this launch launched or closed the game: + // a loose check (any quiet screen after the boot) that a game in play + // would pass too. + bool title_after_launch = false; // Pressed once on the title before `title_press` for Save::BACKUP. // Empty: the game has no backup save, and a BACKUP launch fails // before it presses anything. diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp index dbc79818dc..df435e47e5 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp @@ -41,6 +41,8 @@ enum class Screen{ NEW_GAME, // the title's menu without a save BACKUP_CONFIRM, // "Load the backup save?" LOAD_MENU, // Continue / New Game + UPDATE, // the update prompt: Up twice to its last choice, then A + GAME_DIALOG, // the game's dialog box, in play or on the way OVERWORLD, }; @@ -77,9 +79,16 @@ enum class Backup{ NONE, }; +// How loose the test game's title and dialog checks are. +enum class GameScreens{ + STRICT, // read whenever they show + AFTER_LAUNCH, // read only once the launch launched the game +}; + // The test game: A on the title, `backup` for the backup save, A on each -// prompt, and small bounds so a failure takes few looks. -StoryLaunchPolicy test_policy(Backup backup){ +// prompt (Up, Up, A on the update prompt; B on the game's dialog), and +// small bounds so a failure takes few looks. +StoryLaunchPolicy test_policy(Backup backup, GameScreens game_screens){ StoryLaunchPolicy policy; policy.game = "test"; policy.tile_selected = [](const ScreenObservation& observation){ @@ -97,7 +106,11 @@ StoryLaunchPolicy test_policy(Backup backup){ {.name = "license check", .matches = shows(Screen::LICENSE_CHECK)}, {.name = "backup confirm", .matches = shows(Screen::BACKUP_CONFIRM)}, {.name = "load menu", .matches = shows(Screen::LOAD_MENU)}, + {.name = "update", .matches = shows(Screen::UPDATE), .before = {{.button = BUTTON_UP}, {.button = BUTTON_UP}}}, + {.name = "game dialog", .matches = shows(Screen::GAME_DIALOG), .press = {.button = BUTTON_B}, + .after_launch = game_screens == GameScreens::AFTER_LAUNCH}, }; + policy.title_after_launch = game_screens == GameScreens::AFTER_LAUNCH; policy.presses_per_screen = 3; policy.max_looks = 12; return policy; @@ -118,6 +131,7 @@ struct LaunchCase{ std::vector screens; StoryLaunchRequest request = {}; Backup backup = Backup::PLZA; + GameScreens game_screens = GameScreens::STRICT; }; struct Expectation{ // Empty: the launch succeeds; else a part of its failure's detail. @@ -176,6 +190,17 @@ const std::vector LAUNCH_CASES = { {.name = "kept game off the overworld takes B", .input = {.screens = then_settled({Screen::LOAD_MENU, Screen::BLACK}), .request = {.running = Running::KEEP}}, .expectation = {.inputs = "A B wait wait"}}, + {.name = "update prompt takes its lead-in presses", + .input = {.screens = then_settled({Screen::HOME_ON_GAME, Screen::HOME_ON_GAME, Screen::UPDATE, Screen::BLACK, Screen::TITLE})}, + .expectation = {.inputs = "X A Up Up A wait A wait wait"}}, + {.name = "strict title in front is pressed", + .input = {.screens = then_settled({Screen::GAME_DIALOG, Screen::TITLE, Screen::BLACK})}, + .expectation = {.inputs = "B A wait wait wait"}}, + {.name = "loose title in front is left with HOME", + .input = {.screens = then_settled({Screen::GAME_DIALOG, Screen::TITLE, Screen::HOME_ON_GAME, Screen::HOME_ON_GAME, + Screen::BLACK, Screen::GAME_DIALOG, Screen::TITLE}), + .game_screens = GameScreens::AFTER_LAUNCH}, + .expectation = {.inputs = "HOME HOME X A wait B A wait wait"}}, }; // The fake backend's record of `input` ("A" -> "press A"). @@ -216,7 +241,7 @@ std::string check_launch(const LaunchCase& test, CancellableScope& scope){ looks.push_back(look(screen)); } FakeStoryBackend backend({.looks = std::move(looks)}, scope); - const Outcome outcome = launch_story_game(backend, test_policy(test.input.backup), test.input.request); + const Outcome outcome = launch_story_game(backend, test_policy(test.input.backup, test.input.game_screens), test.input.request); if (test.expectation.failure.empty() && !outcome.succeeded()){ return std::format("expected success, got {}", to_string(outcome)); } From f40746d40453836a68a7cfab79fe74642b34d065 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sat, 14 Nov 2026 02:09:11 -0800 Subject: [PATCH 3/6] AutoStory launch: read which HOME tile is selected NintendoSwitch_HomeTileReader finds HOME's cyan frame on the game row and reads the game name HOME writes above the row. switch_home_tile_named() turns it into a launch policy's tile check, so a game names its tile instead of trusting whatever tile is selected (console B, 2026-11-14: a blind launch opened FireRed after the HOME order changed). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../workflows/cpp-ci-serial-programs-base.yml | 4 +- .../ComputerPrograms_UnitTestRunner.cpp | 3 +- .../NintendoSwitch_HomeTileReader.cpp | 248 ++++++++++++++++++ .../Inference/NintendoSwitch_HomeTileReader.h | 55 ++++ .../AutoStory/Engine/Pokemon_StoryLaunch.cpp | 9 + .../AutoStory/Engine/Pokemon_StoryLaunch.h | 10 +- .../Engine/Pokemon_StoryLaunch_Tests.cpp | 17 +- SerialPrograms/cmake/SourceFiles.cmake | 2 + 8 files changed, 342 insertions(+), 6 deletions(-) create mode 100644 SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp create mode 100644 SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index f8a299851d..412df7d233 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -245,9 +245,9 @@ jobs: uses: actions/checkout@v7 if: inputs.run-tests with: - # The fork's fixtures: upstream's plus the SystemKeyboardDetector screenshots. + # The fork's fixtures: upstream's plus the SystemKeyboardDetector and HomeTileReader screenshots. repository: 'dayfine/CommandLineTests' - ref: '5de4ff774f9141d904c4b5c557fa7b9dc85eaef1' + ref: 'eae4697738d5fe18975e77c05a4e01216bed68e7' path: 'CommandLineTests' # The unit tests assume the downloadable resources they need are already downloaded diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index f9c4f6f51b..91326d92a2 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -31,6 +31,7 @@ #include "NintendoSwitch/Inference/NintendoSwitch_CheckOnlineDetector.h" #include "NintendoSwitch/Inference/NintendoSwitch_FailedToConnectDetector.h" +#include "NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h" #include "NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h" #include "NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.h" #include "NintendoSwitch/Programs/Typewriter/NintendoSwitch_Typewriter_Tests.h" @@ -68,7 +69,6 @@ namespace ComputerPrograms{ UnitTestDatabase make_UNIT_TESTS_ALL(){ UnitTestDatabase ret; - add_tests_QtEventThreadPool(ret, GlobalThreadPools::computation_normal()); add_tests_Sockets(ret, GlobalThreadPools::unlimited_normal()); add_tests_HttpServer(ret, GlobalThreadPools::unlimited_normal()); @@ -101,6 +101,7 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){ Pokemon::add_tests_AutoStory(ret); NintendoSwitch::add_tests_CheckOnlineDetector(ret); NintendoSwitch::add_tests_FailedToConnectDetector(ret); + NintendoSwitch::add_tests_HomeTileReader(ret); NintendoSwitch::add_tests_SystemKeyboardDetector(ret); NintendoSwitch::add_tests_UpdatePopupDetector(ret); NintendoSwitch::Typewriter::add_tests_TypewriterTextMapping(ret); diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp new file mode 100644 index 0000000000..789f94ccf8 --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp @@ -0,0 +1,248 @@ +/* Home Tile Reader + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonTools/Images/ImageFilter.h" +#include "CommonTools/OCR/OCR_Routines.h" +#include "CommonTools/OCR/OCR_StringNormalization.h" +#include "Tests/ImageUnitTest.h" +#include "NintendoSwitch_HomeTileReader.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + + +namespace{ + +// The selected tile's frame lines, about 0.008 of the screen tall, just +// outside the tile art: strips across the screen that the art of the +// other tiles stays out of. +const ImageFloatBox FRAME_TOP_STRIP{0.0, 0.252, 1.0, 0.014}; +const ImageFloatBox FRAME_BOTTOM_STRIP{0.0, 0.627, 1.0, 0.014}; +constexpr double FRAME_TOP = 0.256; +constexpr double FRAME_BOTTOM = 0.637; +// A column crosses a frame line when this many of its pixels in the strip +// are the frame's cyan. +constexpr size_t MIN_LINE_PIXELS = 3; +// A tile frame is 0.214 of the screen wide. +constexpr double MIN_FRAME_WIDTH = 0.18; +constexpr double MAX_FRAME_WIDTH = 0.25; + +// The band above the row where HOME writes the selected game's name. The +// name starts 0.044 left of the frame, at the game card icon. +constexpr double NAME_TOP = 0.19; +constexpr double NAME_HEIGHT = 0.06; +constexpr double NAME_LEAD = 0.06; +constexpr double NAME_RIGHT = 0.98; +// The name's cyan (ARGB), dark text on HOME's light background once +// filtered: red low, green and blue high. +constexpr uint32_t NAME_MINS = 0xff000096; +constexpr uint32_t NAME_MAXS = 0xff96ffff; + +// The frame's cyan, which pulses between about (0, 187, 229) and +// (52, 249, 229), with lighter anti-aliased edges. +bool is_frame_cyan(uint32_t pixel){ + const int red = (pixel >> 16) & 0xff; + const int green = (pixel >> 8) & 0xff; + const int blue = pixel & 0xff; + return green >= 160 && blue >= 160 && green - red >= 60 && blue - red >= 60; +} + +// Per column of `strip`, whether it crosses a frame line. +std::vector frame_line_columns(const ImageViewRGB32& strip){ + std::vector columns(strip.width(), false); + for (size_t x = 0; x < strip.width(); x++){ + size_t cyan = 0; + for (size_t y = 0; y < strip.height(); y++){ + cyan += is_frame_cyan(strip.pixel(x, y)) ? 1 : 0; + } + columns[x] = cyan >= MIN_LINE_PIXELS; + } + return columns; +} + +// The longest run of columns that cross both frame lines, as [begin, end). +std::pair longest_framed_run(const std::vector& top, const std::vector& bottom){ + std::pair best{0, 0}; + size_t begin = 0; + for (size_t x = 0; x <= top.size(); x++){ + const bool framed = x < top.size() && x < bottom.size() && top[x] && bottom[x]; + if (framed){ + continue; + } + if (x - begin > best.second - best.first){ + best = {begin, x}; + } + begin = x + 1; + } + return best; +} + +} // namespace + + +std::optional find_selected_home_tile(const ImageViewRGB32& screen){ + if (screen.width() == 0 || screen.height() == 0){ + return std::nullopt; + } + const std::pair run = longest_framed_run( + frame_line_columns(extract_box_reference(screen, FRAME_TOP_STRIP)), + frame_line_columns(extract_box_reference(screen, FRAME_BOTTOM_STRIP)) + ); + const double width = (double)screen.width(); + const double frame_width = (double)(run.second - run.first) / width; + if (frame_width < MIN_FRAME_WIDTH || frame_width > MAX_FRAME_WIDTH){ + return std::nullopt; + } + return ImageFloatBox((double)run.first / width, FRAME_TOP, frame_width, FRAME_BOTTOM - FRAME_TOP); +} + +std::optional read_selected_home_tile_name(Language language, const ImageViewRGB32& screen){ + const std::optional frame = find_selected_home_tile(screen); + if (!frame){ + return std::nullopt; + } + const double left = std::max(0.0, frame->x - NAME_LEAD); + const ImageViewRGB32 band = extract_box_reference(screen, ImageFloatBox(left, NAME_TOP, NAME_RIGHT - left, NAME_HEIGHT)); + const ImageRGB32 text = to_blackwhite_rgb32_range(band, true, NAME_MINS, NAME_MAXS); + return OCR::ocr_read(language, text); +} + +bool selected_home_tile_is(Language language, const ImageViewRGB32& screen, std::string_view name){ + const std::optional read = read_selected_home_tile_name(language, screen); + if (!read){ + return false; + } + const std::u32string wanted = OCR::normalize_utf32(std::string(name)); + return !wanted.empty() && OCR::normalize_utf32(*read).find(wanted) != std::u32string::npos; +} + + + +namespace{ + +// The names HOME shows for the two games on console B's frames. +constexpr std::string_view FIRERED = "Pokémon FireRed Version"; +constexpr std::string_view LEGENDS_ZA = "Pokémon Legends: Z-A"; + +// Where the first tile's frame starts, and the step to the next tile, as +// parts of the screen's width, before the row scrolls. +constexpr double FIRST_TILE_LEFT = 0.076; +constexpr double TILE_STEP = 0.211; +constexpr double TILE_TOLERANCE = 0.02; + +struct HomeTileCase{ + struct Input{ + std::string_view image; + }; + struct Expectation{ + // The selected tile's place in the row, from 0; nothing: no tile + // is selected. + std::optional tile = std::nullopt; + // The name the selected tile shows: FIRERED or LEGENDS_ZA. + std::string_view game = {}; + }; + std::string_view name; + Input input; + Expectation expectation; +}; + +// Console B, 2026-11-14: HOME after the order changed (the game played +// last moves to the first tile), and the user select over HOME. +const std::vector HOME_TILE_CASES = { + {.name = "FireRed on the first tile 1", + .input = {.image = "NintendoSwitch/HomeTileReader/FireRedFirstTile_1.png"}, + .expectation = {.tile = 0, .game = FIRERED}}, + {.name = "FireRed on the first tile 2", + .input = {.image = "NintendoSwitch/HomeTileReader/FireRedFirstTile_2.png"}, + .expectation = {.tile = 0, .game = FIRERED}}, + {.name = "FireRed on the first tile 3", + .input = {.image = "NintendoSwitch/HomeTileReader/FireRedFirstTile_3.png"}, + .expectation = {.tile = 0, .game = FIRERED}}, + {.name = "Legends Z-A on the first tile", + .input = {.image = "NintendoSwitch/HomeTileReader/LegendsZAFirstTile.png"}, + .expectation = {.tile = 0, .game = LEGENDS_ZA}}, + {.name = "Legends Z-A on the second tile", + .input = {.image = "NintendoSwitch/HomeTileReader/LegendsZASecondTile.png"}, + .expectation = {.tile = 1, .game = LEGENDS_ZA}}, + {.name = "user select over HOME", + .input = {.image = "NintendoSwitch/HomeTileReader/UserSelectOverHome.png"}, + .expectation = {}}, +}; + +// What went wrong with the frame on `screen`; empty when it is where `test` expects. +std::string check_frame(const HomeTileCase& test, const ImageViewRGB32& screen){ + const std::optional frame = find_selected_home_tile(screen); + if (!test.expectation.tile){ + return frame ? std::format("Expected no selected tile, found a frame at x={:.3f}.", frame->x) : ""; + } + if (!frame){ + return std::format("Expected a frame on tile {}, found none.", *test.expectation.tile); + } + const double expected_left = FIRST_TILE_LEFT + TILE_STEP * (double)*test.expectation.tile; + if (std::abs(frame->x - expected_left) > TILE_TOLERANCE){ + return std::format("Expected the frame on tile {} (x={:.3f}), found it at x={:.3f}.", *test.expectation.tile, expected_left, frame->x); + } + return ""; +} + +// What went wrong with the names on `screen`: it must be the game `test` +// expects and not the other one. +std::string check_names(const HomeTileCase& test, const ImageViewRGB32& screen){ + const std::string read = read_selected_home_tile_name(Language::English, screen).value_or(""); + for (std::string_view game : {FIRERED, LEGENDS_ZA}){ + const bool expected = game == test.expectation.game; + if (selected_home_tile_is(Language::English, screen, game) != expected){ + return std::format("Expected the tile {} \"{}\", read \"{}\".", expected ? "to be" : "not to be", game, read); + } + } + return ""; +} + +class Test_HomeTileReader : public ImageUnitTest{ +public: + explicit Test_HomeTileReader(const HomeTileCase& test) + : ImageUnitTest(std::format("NintendoSwitch::HomeTileReader - {}", test.name), {std::string(test.input.image)}) + , m_case(test) + { + // Plus the PaddleOCR (ONNX) recognition session and its working set. + m_memory += PADDLE_OCR_SINGLE_LINE_MEMORY_ESTIMATE; + } + + virtual UnitTestResult run_on_images( + Logger&, CancellableScope&, + const std::vector& images + ) const override{ + std::string error = check_frame(m_case, images[0]); + if (error.empty()){ + error = check_names(m_case, images[0]); + } + return error.empty() ? UnitTestResult(true) : UnitTestResult(error); + } + +private: + HomeTileCase m_case; +}; + +} // namespace + + +void add_tests_HomeTileReader(UnitTestDatabase& database){ + for (const HomeTileCase& test : HOME_TILE_CASES){ + database.add(test); + } +} + + + +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h new file mode 100644 index 0000000000..c0c57f1052 --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h @@ -0,0 +1,55 @@ +/* Home Tile Reader + * + * From: https://github.com/PokemonAutomation/ + * + * Which game tile the HOME menu's cursor is on. HOME draws a cyan frame + * around the selected tile of the game row and writes that game's name + * above the row, also in cyan. The frame says that a tile is selected and + * where; the name says which game it is. Neither depends on the order of + * the row, which changes as games are played (the last one played moves + * to the first tile). + * + * Measured on the Switch 1 HOME menu at 1920x1080; positions are parts of + * the screen, so other capture sizes read the same. + * + */ + +#ifndef PokemonAutomation_NintendoSwitch_HomeTileReader_H +#define PokemonAutomation_NintendoSwitch_HomeTileReader_H + +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "CommonFramework/Language.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + + +// The selected tile's cyan frame on `screen`; nothing when no tile of the +// game row is selected (the cursor on the icons below it, a dialog or the +// user select over HOME). +std::optional find_selected_home_tile(const ImageViewRGB32& screen); + +// The name HOME writes above the selected tile, as OCR reads it in +// `language` (the console's, not the game's); nothing when no tile is +// selected. +std::optional read_selected_home_tile_name(Language language, const ImageViewRGB32& screen); + +// Whether a tile is selected and its name contains `name` ("Pokémon +// Legends: Z-A", or a distinctive part of it), compared as OCR text: +// case, accents, spaces and punctuation ignored. +bool selected_home_tile_is(Language language, const ImageViewRGB32& screen, std::string_view name); + + +// The screenshot tests in CommandLineTests/NintendoSwitch/HomeTileReader/. +void add_tests_HomeTileReader(UnitTestDatabase& database); + + + +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.cpp index accab8a6d6..d394730cde 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.cpp +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.cpp @@ -10,6 +10,8 @@ #include #include #include +#include +#include "NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h" #include "Pokemon_StoryLaunch.h" namespace PokemonAutomation{ @@ -227,6 +229,13 @@ class LaunchDriver{ } // namespace +std::function switch_home_tile_named(Language language, std::string name){ + return [language, name = std::move(name)](const ScreenObservation& look){ + return look.frame && look.frame->frame && selected_home_tile_is(language, *look.frame->frame, name); + }; +} + + template Outcome launch_story_game(StoryBackend& backend, const StoryLaunchPolicy& policy, StoryLaunchRequest request){ if (std::optional problem = launch_problem(policy, request)){ diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h index 84b79ccc55..9ff0aff49a 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h @@ -28,6 +28,7 @@ #include #include #include "Common/Cpp/Time.h" +#include "CommonFramework/Language.h" #include "CommonTools/ScreenMachine/ScreenObservation.h" #include "NintendoSwitch/Controllers/NintendoSwitch_ControllerButtons.h" #include "Pokemon/AutoStory/Pokemon_StoryOutcome.h" @@ -90,7 +91,8 @@ struct StoryLaunchPolicy{ std::string game = {}; // On HOME: whether the cursor is on this game's tile. The launch // presses A on HOME only then, as A launches whatever tile is - // selected. Required. + // selected. Required; on a Switch, switch_home_tile_named() with the + // name HOME shows for the game. std::function tile_selected = {}; // How far the tile search moves each way: Left `tile_search` times // (the most recently played games are leftmost), then Right twice as @@ -126,6 +128,12 @@ struct StoryLaunchPolicy{ size_t max_looks = 360; }; +// A `tile_selected` for a Switch game: HOME's cursor is on the tile whose +// name, above the game row, contains `name` ("Pokémon Legends: Z-A"), +// read in `language`, the console's (NintendoSwitch_HomeTileReader.h). +// False on a look without a frame, as in a simulation. +std::function switch_home_tile_named(Language language, std::string name); + // The failure of a launch that did not reach the overworld; its detail // says where it stopped. Nothing in the story answers it: no reload, no // recovery. diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp index df435e47e5..db2cec0722 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp @@ -79,6 +79,12 @@ enum class Backup{ NONE, }; +// How the test game finds its tile on HOME. +enum class TileCheck{ + READINGS, // the scripted "home.tile" reading + SWITCH_NAME, // switch_home_tile_named(), which reads the look's frame +}; + // How loose the test game's title and dialog checks are. enum class GameScreens{ STRICT, // read whenever they show @@ -88,13 +94,16 @@ enum class GameScreens{ // The test game: A on the title, `backup` for the backup save, A on each // prompt (Up, Up, A on the update prompt; B on the game's dialog), and // small bounds so a failure takes few looks. -StoryLaunchPolicy test_policy(Backup backup, GameScreens game_screens){ +StoryLaunchPolicy test_policy(Backup backup, GameScreens game_screens, TileCheck tile_check){ StoryLaunchPolicy policy; policy.game = "test"; policy.tile_selected = [](const ScreenObservation& observation){ const auto it = observation.readings.find(TILE); return it != observation.readings.end() && it->second == 1; }; + if (tile_check == TileCheck::SWITCH_NAME){ + policy.tile_selected = switch_home_tile_named(Language::English, "Pokémon Legends: Z-A"); + } policy.tile_search = 2; policy.title = shows(Screen::TITLE); if (backup == Backup::PLZA){ @@ -132,6 +141,7 @@ struct LaunchCase{ StoryLaunchRequest request = {}; Backup backup = Backup::PLZA; GameScreens game_screens = GameScreens::STRICT; + TileCheck tile_check = TileCheck::READINGS; }; struct Expectation{ // Empty: the launch succeeds; else a part of its failure's detail. @@ -178,6 +188,9 @@ const std::vector LAUNCH_CASES = { {.name = "game not on HOME", .input = {.screens = {Screen::HOME_OFF_GAME}}, .expectation = {.failure = "tile is not on HOME", .inputs = "Left Left Right Right Right Right"}}, + {.name = "Switch tile check never launches without a frame", + .input = {.screens = {Screen::HOME_ON_GAME}, .tile_check = TileCheck::SWITCH_NAME}, + .expectation = {.failure = "tile is not on HOME", .inputs = "Left Left Right Right Right Right"}}, {.name = "title ignores A", .input = {.screens = {Screen::TITLE}}, .expectation = {.failure = "stuck on title", .inputs = "A A A"}}, @@ -241,7 +254,7 @@ std::string check_launch(const LaunchCase& test, CancellableScope& scope){ looks.push_back(look(screen)); } FakeStoryBackend backend({.looks = std::move(looks)}, scope); - const Outcome outcome = launch_story_game(backend, test_policy(test.input.backup, test.input.game_screens), test.input.request); + const Outcome outcome = launch_story_game(backend, test_policy(test.input.backup, test.input.game_screens, test.input.tile_check), test.input.request); if (test.expectation.failure.empty() && !outcome.succeeded()){ return std::format("expected success, got {}", to_string(outcome)); } diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 72619f0cab..cb9f9aa05c 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1376,6 +1376,8 @@ file(GLOB LIBRARY_SOURCES Source/NintendoSwitch/Inference/NintendoSwitch_HomeMenuDetector.h Source/NintendoSwitch/Inference/NintendoSwitch_HomeScreenObserver.cpp Source/NintendoSwitch/Inference/NintendoSwitch_HomeScreenObserver.h + Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp + Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.h Source/NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.cpp Source/NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.h Source/NintendoSwitch/Inference/NintendoSwitch_StartGameUserSelectDetector.cpp From 2e5f442ad78a6886b5c297b5e94b6bd4c600a201 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sat, 14 Nov 2026 02:11:21 -0800 Subject: [PATCH 4/6] HomeTileReader: start the name crop right of the game card icon Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../Inference/NintendoSwitch_HomeTileReader.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp index 789f94ccf8..b8913f5001 100644 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_HomeTileReader.cpp @@ -38,10 +38,11 @@ constexpr double MIN_FRAME_WIDTH = 0.18; constexpr double MAX_FRAME_WIDTH = 0.25; // The band above the row where HOME writes the selected game's name. The -// name starts 0.044 left of the frame, at the game card icon. +// name starts 0.005 to 0.01 left of the frame, right of a game card icon +// that ends 0.017 or more left of it. constexpr double NAME_TOP = 0.19; constexpr double NAME_HEIGHT = 0.06; -constexpr double NAME_LEAD = 0.06; +constexpr double NAME_LEAD = 0.015; constexpr double NAME_RIGHT = 0.98; // The name's cyan (ARGB), dark text on HOME's light background once // filtered: red low, green and blue high. From f90c44ab77da39f91ef5aeba9610c8b0263191fc Mon Sep 17 00:00:00 2001 From: dayfine Date: Sat, 14 Nov 2026 02:19:58 -0800 Subject: [PATCH 5/6] AutoStory launch: move the Switch launch prompts here from PLZA Pokemon_StorySwitchPrompts.{h,cpp} come unchanged from plza/story-engine (723921ba2). The Close software dialog, the user select and the update prompt are the same in every game, so the shared launch owns them and FRLG, BDSP and PLZA use this one copy. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../AutoStory/Pokemon_StorySwitchPrompts.cpp | 53 +++++++++++++++++++ .../AutoStory/Pokemon_StorySwitchPrompts.h | 33 ++++++++++++ SerialPrograms/cmake/SourceFiles.cmake | 2 + 3 files changed, 88 insertions(+) create mode 100644 SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySwitchPrompts.cpp create mode 100644 SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySwitchPrompts.h diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySwitchPrompts.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySwitchPrompts.cpp new file mode 100644 index 0000000000..3dcb5a4d94 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySwitchPrompts.cpp @@ -0,0 +1,53 @@ +/* AutoStory Switch Launch Prompts + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "NintendoSwitch/NintendoSwitch_Settings.h" +#include "NintendoSwitch/NintendoSwitch_ConsoleHandle.h" +#include "NintendoSwitch/Inference/NintendoSwitch_CloseGameDetector.h" +#include "NintendoSwitch/Inference/NintendoSwitch_StartGameUserSelectDetector.h" +#include "NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.h" +#include "Pokemon_StorySwitchPrompts.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + +using namespace std::chrono_literals; +using namespace NintendoSwitch; + + +namespace{ + +// Whether `detector` fires on the look's frame; false without one. +template +std::function frame_shows(std::shared_ptr detector){ + return [detector = std::move(detector)](const ScreenObservation& look){ + return look.frame && look.frame->frame && detector->detect(*look.frame->frame); + }; +} + +} // namespace + + +std::vector switch_launch_prompts(ConsoleHandle& console){ + const StoryLaunchPress scroll_up{.button = BUTTON_UP, .hold = 40ms, .release = 40ms}; + return { + {.name = "close software", .matches = frame_shows(std::make_shared(console)), + .press = {.button = BUTTON_A, .hold = 160ms, .release = ConsoleSettings::instance().CLOSE_GAME_DELAY}}, + {.name = "user select", .matches = frame_shows(std::make_shared(console)), + .press = {.button = BUTTON_A, .hold = 160ms}}, + {.name = "update prompt", .matches = frame_shows(std::make_shared(console)), + .press = {.button = BUTTON_A, .hold = 160ms}, .before = {scroll_up, scroll_up}}, + }; +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySwitchPrompts.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySwitchPrompts.h new file mode 100644 index 0000000000..5bfa8dba26 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySwitchPrompts.h @@ -0,0 +1,33 @@ +/* AutoStory Switch Launch Prompts + * + * From: https://github.com/PokemonAutomation/ + * + * The Switch's own screens between HOME and a game, each answered by one + * press: HOME's "Close software?", the user select and the update prompt. + * They are the same in every game, so each game's StoryLaunchPolicy + * (Engine/Pokemon_StoryLaunch.h) starts its prompts with these. + * + */ + +#ifndef PokemonAutomation_Pokemon_StorySwitchPrompts_H +#define PokemonAutomation_Pokemon_StorySwitchPrompts_H + +#include +#include "NintendoSwitch/NintendoSwitch_ConsoleHandle.h" +#include "Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// On `console`'s frames: "Close software?" takes A, then the console's +// close delay (Switch settings); the user select takes A on the user +// highlighted; the update prompt takes Up twice, to "Start Software", then +// A. A look without a frame (a simulation) shows none of them. +std::vector switch_launch_prompts(NintendoSwitch::ConsoleHandle& console); + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index cb9f9aa05c..c64cc92e1d 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1662,6 +1662,8 @@ file(GLOB LIBRARY_SOURCES Source/Pokemon/AutoStory/Pokemon_StoryStepExecutor.h Source/Pokemon/AutoStory/Pokemon_StoryStepExecutor_Tests.cpp Source/Pokemon/AutoStory/Pokemon_StoryStepExecutor_Tests.h + Source/Pokemon/AutoStory/Pokemon_StorySwitchPrompts.cpp + Source/Pokemon/AutoStory/Pokemon_StorySwitchPrompts.h Source/Pokemon/AutoStory/Pokemon_TilePose.cpp Source/Pokemon/AutoStory/Pokemon_TilePose.h Source/Pokemon/AutoStory/Pokemon_TestExpect.h From 9452b7c94a8bc3e5c41536e8fae5293703aca24f Mon Sep 17 00:00:00 2001 From: dayfine Date: Sat, 14 Nov 2026 02:44:39 -0800 Subject: [PATCH 6/6] AutoStory launch: HOME after the A on the tile waits, not A again On console B (FRLG, 2026-11-14 02:41) HOME showed for about 1.5 s after the launch's A on the tile, and each look there pressed A again until the launch failed as stuck on HOME, while the game was booting. After that A, HOME now waits for up to the policy's launch_wait (30 s, as the FRLG RNG reset's wait for the boot) before another A, and moves no cursor in that time. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../AutoStory/Engine/Pokemon_StoryLaunch.cpp | 14 ++++++++++++++ .../Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h | 5 +++++ .../AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp | 13 ++++++++++++- 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.cpp index d394730cde..26d7c2b2a8 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.cpp +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.cpp @@ -122,6 +122,14 @@ class LaunchDriver{ // The cursor to the game's tile, then the game closed if it must // load again, then A on the tile. std::optional on_home(const ScreenObservation& observation){ + if (m_launch_pressed && m_home_waited < m_policy.launch_wait){ + if (m_home_waited == Milliseconds(0)){ + log("home", "waiting for the game to start"); + } + m_home_waited += m_policy.poll; + m_backend.wait(m_policy.poll); + return std::nullopt; + } if (!m_policy.tile_selected(observation)){ return move_to_tile(); } @@ -130,6 +138,8 @@ class LaunchDriver{ return press("home", {.button = BUTTON_X}, m_policy.presses_per_screen, "close the game"); } m_launched = true; + m_launch_pressed = true; + m_home_waited = Milliseconds(0); return press("home", {.button = BUTTON_A}, m_policy.presses_per_screen, "launch the game"); } @@ -222,6 +232,10 @@ class LaunchDriver{ // A on the game's tile, or the title: until the overworld, a screen // the policy doesn't know is the game booting or loading. bool m_launched = false; + // A was pressed on the game's tile, and HOME has shown for + // `m_home_waited` since. + bool m_launch_pressed = false; + Milliseconds m_home_waited = Milliseconds(0); bool m_backup_sent = false; }; diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h index 9ff0aff49a..eb90245801 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch.h @@ -120,6 +120,11 @@ struct StoryLaunchPolicy{ size_t settled_looks = 3; // The wait between looks at a screen with nothing to press. Milliseconds poll = Milliseconds(500); + // After A on the game's tile, how long HOME may still show (its zoom + // into the game, a check or the user select in front of it) before + // the launch takes A again: until then each look at HOME waits, as + // another A there would launch the game again. + Milliseconds launch_wait = Milliseconds(30000); // Presses HOME, the title, an overworld to leave or a screen the // policy doesn't know may take before the launch is stuck there. size_t presses_per_screen = 6; diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp index db2cec0722..1853b06ac0 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryLaunch_Tests.cpp @@ -121,6 +121,7 @@ StoryLaunchPolicy test_policy(Backup backup, GameScreens game_screens, TileCheck }; policy.title_after_launch = game_screens == GameScreens::AFTER_LAUNCH; policy.presses_per_screen = 3; + policy.launch_wait = Milliseconds(2000); policy.max_looks = 12; return policy; } @@ -180,7 +181,17 @@ const std::vector LAUNCH_CASES = { {.name = "license check back to HOME", .input = {.screens = then_settled({Screen::HOME_ON_GAME, Screen::HOME_ON_GAME, Screen::LICENSE_CHECK, Screen::HOME_ON_GAME, Screen::BLACK, Screen::TITLE, Screen::LOAD_MENU})}, - .expectation = {.inputs = "X A A A wait A A wait wait"}}, + .expectation = {.inputs = "X A A wait wait A A wait wait"}}, + // Console B, 2026-11-14 02:41 (FRLG): HOME showed for about 1.5 s + // after the A on the tile, and each look there pressed A again. + {.name = "HOME after the launch press waits", + .input = {.screens = then_settled({Screen::HOME_ON_GAME, Screen::HOME_ON_GAME, Screen::HOME_ON_GAME, Screen::HOME_ON_GAME, + Screen::HOME_ON_GAME, Screen::BLACK, Screen::TITLE})}, + .expectation = {.inputs = "X A wait wait wait wait A wait wait"}}, + {.name = "HOME past the launch wait takes A again", + .input = {.screens = then_settled({Screen::HOME_ON_GAME, Screen::HOME_ON_GAME, Screen::HOME_ON_GAME, Screen::HOME_ON_GAME, + Screen::HOME_ON_GAME, Screen::HOME_ON_GAME, Screen::HOME_ON_GAME, Screen::TITLE})}, + .expectation = {.inputs = "X A wait wait wait wait A A wait wait"}}, {.name = "cursor on another tile", .input = {.screens = then_settled({Screen::HOME_OFF_GAME, Screen::HOME_OFF_GAME, Screen::HOME_ON_GAME}), .request = {.running = Running::KEEP}},