diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp index 339b23c3cf..41e22ca2a7 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp @@ -22,6 +22,7 @@ #include "Inference/PokemonLZA_PartyBarDetector.h" #include "Inference/PokemonLZA_SelectionArrowDetector.h" #include "Programs/AutoStory/PokemonLZA_AutoStoryFailures.h" +#include "Programs/AutoStory/Actions/PokemonLZA_AutoStoryDialogs_Tests.h" #include "Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader_Tests.h" #include "Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox_Tests.h" #include "Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h" @@ -57,6 +58,7 @@ void add_tests(UnitTestDatabase& database){ add_tests_AutoStoryDialogChoiceBox(database); add_tests_AutoStoryObjectiveMarkerReader(database); add_tests_AutoStoryFailures(database); + add_tests_AutoStoryDialogs(database); } diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryActions.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryActions.cpp new file mode 100644 index 0000000000..a819b26063 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryActions.cpp @@ -0,0 +1,213 @@ +/* PokemonLZA AutoStory - Actions + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "Common/Cpp/Color.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "VideoGames/PokemonLZA/Programs/PokemonLZA_MenuNavigation.h" +#include "VideoGames/PokemonLZA/Inference/Battles/PokemonLZA_RunFromBattleDetector.h" +#include "VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.h" +#include "VideoGames/PokemonLZA/Inference/PokemonLZA_OverworldPartySelectionDetector.h" +#include "VideoGames/PokemonLZA/Inference/PokemonLZA_SelectionArrowDetector.h" +#include "VideoGames/PokemonLZA/Programs/PokemonLZA_BasicNavigation.h" +#include "VideoGames/PokemonLZA/Programs/PokemonLZA_TrainerBattle.h" +#include "PokemonLZA_AutoStoryActions.h" +#include "PokemonLZA_AutoStoryDialogs.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + +using namespace std::chrono_literals; + + +void fight_battle( + ProgramEnvironment& env, ConsoleHandle& console, ProControllerContext& context, + BattleAIOption& battle_ai +){ + // The battle's text first: the overworld only ends it once it has + // begun, and it must begin within a minute. + RunFromBattleWatcher battle_menu(COLOR_GREEN, &console.overlay(), 10ms); + context.wait_for_all_requests(); + const int started = run_until( + console, context, + [](ProControllerContext& context){ + pbf_mash_button(context, BUTTON_B, 60s); + }, + {battle_menu} + ); + if (started < 0){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + "fight_battle(): no battle began within a minute.", + console + ); + } + TrainerBattleState battle_state(battle_ai); + OverworldPartySelectionWatcher overworld(COLOR_WHITE, &console.overlay(), 100ms); + const int ended = run_battle_until( + env, console, context, battle_state, overworld, 120s, + [](ProControllerContext& context){ + pbf_press_button(context, BUTTON_UP, 500ms, 1000ms); + } + ); + if (ended < 0){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + "fight_battle(): the battle's menu was gone for two minutes.", + console + ); + } +} + + + +void fight_battle_in_progress( + ProgramEnvironment& env, ConsoleHandle& console, ProControllerContext& context, + BattleAIOption& battle_ai, std::chrono::seconds timeout +){ + TrainerBattleState battle_state(battle_ai); + StoryDialogDetector any_dialog; + const WallClock deadline = current_time() + timeout; + bool attacked = false; + while (true){ + context.wait_for_all_requests(); + const VideoSnapshot screen = console.video().snapshot(); + if (!screen){ + context.wait_for(100ms); + continue; + } + const bool dialog = any_dialog.detect(screen); + if (dialog && attacked){ + return; + } + if (current_time() > deadline){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + "fight_battle_in_progress(): no dialog after the battle within the timeout.", + console + ); + } + if (dialog || !battle_state.attempt_one_attack(env, console, context)){ + if (!attacked){ + pbf_press_button(context, BUTTON_A, 160ms, 400ms); + } + continue; + } + attacked = true; + } +} + + + +namespace{ + +// Whether the report page's Save button (its cursor arrow, where +// save_game_to_menu() looks for it) shows within `timeout`. +bool save_button_shows(ConsoleHandle& console, ProControllerContext& context, Milliseconds timeout){ + // TODO: unify with save_game_to_menu() (PokemonLZA_BasicNavigation.cpp), + // which keeps this box inline - naming it there adds a symbol to a + // header at the code-health symbol limit. + SelectionArrowWatcher save_button( + COLOR_GREEN, &console.overlay(), SelectionArrowType::RIGHT, {0.287118, 0.638835, 0.064410, 0.108738} + ); + return wait_until(console, context, timeout, {save_button}) == 0; +} + +// From the overworld: X opens the menu, R its report page. The first menu +// of a playthrough opens under a save tutorial card that A closes, leaving +// the menu with the cursor on the Bag; R does +// nothing over the card. So when R does not bring up the Save button, A +// once, then R again. A is never pressed on the plain menu, where it +// would open the Bag. +void open_save_report(ConsoleHandle& console, ProControllerContext& context){ + pbf_press_button(context, BUTTON_X, 160ms, 2340ms); + pbf_press_button(context, BUTTON_R, 160ms, 240ms); + if (save_button_shows(console, context, 5s)){ + return; + } + console.log("save_story_game(): no Save button after R: the menu's tutorial card; pressing A, then R."); + pbf_press_button(context, BUTTON_A, 160ms, 1340ms); + pbf_press_button(context, BUTTON_R, 160ms, 240ms); + if (!save_button_shows(console, context, 5s)){ + console.log("save_story_game(): still no Save button; save_game_to_menu() takes over.", COLOR_RED); + } +} + +// save_game_to_menu(), its failures as false. +bool try_save(ConsoleHandle& console, ProControllerContext& context){ + try{ + return save_game_to_menu(console, context); + }catch (OperationFailedException& e){ + console.log(std::format("save_story_game(): {}", e.message()), COLOR_RED); + return false; + } +} + +} // namespace + +// Opens the report page (open_save_report(), for the first menu's +// tutorial), then save_game_to_menu() presses Save once its button shows +// and waits for the "Saved!" text, so the save is confirmed. If that +// fails, B back to the overworld (B is harmless while writing) and one +// more try. +void save_story_game(ConsoleHandle& console, ProControllerContext& context){ + open_save_report(console, context); + if (!try_save(console, context)){ + mash_to_overworld(console, context, BUTTON_B, 10s); + if (!save_game_to_menu(console, context)){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + "save_story_game(): the game did not save.", + console + ); + } + } + mash_to_overworld(console, context, BUTTON_B, 10s); +} + +void claim_mable_research_reward(ConsoleHandle& console, ProControllerContext& context){ + overworld_to_main_menu(console, context); + for (int i = 0; i < 3; i++){ + pbf_press_dpad(context, DPAD_DOWN, 160ms, 400ms); + } + pbf_press_button(context, BUTTON_A, 160ms, 2840ms); + + // The page opens on Rewards under Mable's lines on the level 2 reward, + // TM004 Rock Smash: A while a dialog shows, + // until none has for 3 s, then B back to the overworld, through her + // talk after the menu closes (over 20 s). + // No line at all: the page did not open on her reward (03.10, + // not in her office, the claim passed on X alone). + const ScreensPassed passed = press_a_until_quiet(console, context, {.item_receive = true}, 3s, 60s); + if (passed.timed_out){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + "claim_mable_research_reward(): Mable's lines did not end within 60 s.", + console + ); + } + if (!passed.text){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + "claim_mable_research_reward(): no dialog after opening Mable's Research: no reward was claimed.", + console + ); + } + console.log("claim_mable_research_reward(): Mable's lines on the reward passed."); + mash_to_overworld(console, context, BUTTON_B, 60s); +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryActions.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryActions.h new file mode 100644 index 0000000000..9aa7141ed8 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryActions.h @@ -0,0 +1,67 @@ +/* PokemonLZA AutoStory - Actions + * + * From: https://github.com/PokemonAutomation/ + * + * The battle and save actions of the story game (PokemonLZA_AutoStoryGame.h), + * with the dialogue ones in PokemonLZA_AutoStoryDialogs.h, the walks in + * PokemonLZA_AutoStoryMoves.h and the scenes in PokemonLZA_AutoStoryScenes.h. + * Each is a short function over the LZA battle and menu code that already + * exists (TrainerBattle, BasicNavigation, the dialog and selection-arrow + * detectors). + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryActions_H +#define PokemonAutomation_PokemonLZA_AutoStoryActions_H + +#include +#include "CommonFramework/Tools/ProgramEnvironment.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "NintendoSwitch/NintendoSwitch_ConsoleHandle.h" +#include "VideoGames/PokemonLZA/Options/PokemonLZA_BattleAIOption.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// From a trainer's or a wild Pokemon's battle starting, fights it with +// TrainerBattleState (`battle_ai`'s moves), pressing B through the text +// around it, until the overworld is back. Throws OperationFailedException +// if neither shows for two minutes. +void fight_battle( + ProgramEnvironment& env, ConsoleHandle& console, ProControllerContext& context, + BattleAIOption& battle_ai +); + +// A story battle, from the talk before it: attacks with TrainerBattleState +// until a dialog box shows (the speech after a win or a loss). Until the +// first attack lands, a dialog box or a failed lock-on gets A instead: the +// trainer's lines, the challenge cutscene's subtitle (no text box) and +// tutorial cards (Taunie's battle in 03.02). For +// story battles, which show no Run prompt for fight_battle() to start on. +// Throws OperationFailedException after `timeout`. +void fight_battle_in_progress( + ProgramEnvironment& env, ConsoleHandle& console, ProControllerContext& context, + BattleAIOption& battle_ai, std::chrono::seconds timeout +); + +// From the overworld, saves the game (save_game_to_menu()) and returns to +// the overworld. The first menu of a playthrough shows a save tutorial +// card, which it closes with A. If the save fails, B back to the +// overworld and one more try; throws OperationFailedException if that +// fails too. +void save_story_game(ConsoleHandle& console, ProControllerContext& context); + +// From the overworld: the main menu, DOWN x3 to "Mable's Research" (as +// recorded), A; through Mable's lines on the reward, back +// to the overworld. Throws OperationFailedException when no line shows +// (no reward claimed) or the lines do not end within 60 s. +void claim_mable_research_reward(ConsoleHandle& console, ProControllerContext& context); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryDialogs.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryDialogs.cpp new file mode 100644 index 0000000000..3f3047530d --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryDialogs.cpp @@ -0,0 +1,217 @@ +/* PokemonLZA AutoStory - Dialogs + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "Common/Cpp/Color.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.h" +#include "VideoGames/PokemonLZA/Inference/PokemonLZA_OverworldPartySelectionDetector.h" +#include "VideoGames/PokemonLZA/Inference/PokemonLZA_SelectionArrowDetector.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.h" +#include "PokemonLZA_AutoStoryDialogs.h" +#include "PokemonLZA_AutoStoryScenes.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + +using namespace std::chrono_literals; + + +void press_until_overworld( + ConsoleHandle& console, ProControllerContext& context, + const std::function& presses, + Milliseconds timeout, std::string_view failure +){ + OverworldPartySelectionWatcher overworld(COLOR_WHITE, &console.overlay(), OVERWORLD_HOLD); + context.wait_for_all_requests(); + const int ret = run_until( + console, context, + [&presses, timeout](ProControllerContext& context){ + const WallClock deadline = current_time() + timeout; + while (current_time() < deadline){ + presses(context); + context.wait_for_all_requests(); + } + }, + {overworld} + ); + if (ret < 0){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::string(failure), + console + ); + } +} + +void mash_to_overworld(ConsoleHandle& console, ProControllerContext& context, Button button, Milliseconds timeout){ + press_until_overworld( + console, context, + [button, timeout](ProControllerContext& context){ + pbf_mash_button(context, button, timeout); + }, + timeout, std::format("mash_to_overworld(): no overworld after {} ms.", timeout.count()) + ); +} + +QuietPresses::Step QuietPresses::look(bool busy, WallClock now){ + if (!m_first){ + m_first = now; + m_last_busy = now; + } + if (busy){ + m_last_busy = now; + }else if (now - *m_last_busy >= m_quiet){ + return QuietPresses::Step::QUIET; + } + if (now - *m_first >= m_timeout){ + return QuietPresses::Step::TIMED_OUT; + } + return busy ? QuietPresses::Step::PRESS_A : QuietPresses::Step::WAIT; +} + +ScreensPassed press_a_until_quiet( + ConsoleHandle& console, ProControllerContext& context, + StoryDialogKinds kinds, Milliseconds quiet, Milliseconds timeout +){ + StoryDialogDetector any_dialog(kinds); + ScreensPassed passed; + QuietPresses clock(quiet, timeout); + while (true){ + context.wait_for_all_requests(); + const VideoSnapshot screen = console.video().snapshot(); + const bool card_now = screen && tutorial_card_shows(screen); + const bool busy = card_now || (screen && any_dialog.detect(screen)); + if (busy){ + passed.card = passed.card || card_now; + passed.text = passed.text || !card_now; + } + switch (clock.look(busy, current_time())){ + case QuietPresses::Step::QUIET: + return passed; + case QuietPresses::Step::TIMED_OUT: + passed.timed_out = true; + return passed; + case QuietPresses::Step::PRESS_A: + pbf_press_button(context, BUTTON_A, 160ms, 840ms); + break; + case QuietPresses::Step::WAIT: + context.wait_for(250ms); + break; + } + } +} + +void talk_to_facing(ConsoleHandle& console, ProControllerContext& context){ + StoryDialogWatcher dialog; + context.wait_for_all_requests(); + const int ret = run_until( + console, context, + [](ProControllerContext& context){ + pbf_press_button(context, BUTTON_A, 160ms, 5000ms); + }, + {dialog} + ); + if (ret < 0){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + "talk_to_facing(): A opened no dialogue.", + console + ); + } +} + +namespace{ + +// Whether the selection arrow shows among the choices, held still, within +// `timeout`. +bool choice_arrow_shows(ConsoleHandle& console, ProControllerContext& context, Milliseconds timeout){ + SelectionArrowWatcher arrow(COLOR_YELLOW, &console.overlay(), SelectionArrowType::RIGHT, dialog_choice_box()); + context.wait_for_all_requests(); + return wait_until(console, context, timeout, {arrow}) >= 0; +} + +// The highlighted choice row, 0 the top, read with read_dialog_choice_rows(); +// none if no choice menu reads. +std::optional read_choice_row(ConsoleHandle& console){ + const VideoSnapshot screen = console.video().snapshot(); + if (!screen){ + return std::nullopt; + } + const std::optional rows = read_dialog_choice_rows(screen); + if (!rows){ + return std::nullopt; + } + console.log(std::format("Choice row {} of {} highlighted.", rows->selected, rows->count)); + return rows->selected; +} + +} // namespace + +int dialog_choice_moves(size_t current, size_t target){ + return static_cast(target) - static_cast(current); +} + +void choose_dialog_option(ConsoleHandle& console, ProControllerContext& context, size_t index){ + // Some questions show their choices only after A closes the question's + // text: the Research Lab elevator's "which floor?". + // A press comes only after 3 s without choices, so it + // cannot pick the first choice the moment the choices appear. + bool shows = choice_arrow_shows(console, context, 3s); + for (int press = 0; !shows && press < 3; press++){ + pbf_press_button(context, BUTTON_A, 160ms, 240ms); + shows = choice_arrow_shows(console, context, 3s); + } + if (!shows){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + "choose_dialog_option(): no choices on screen.", + console + ); + } + const std::optional row = read_choice_row(console); + if (!row){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + "choose_dialog_option(): no highlighted choice row reads (read_dialog_choice_rows()).", + console + ); + } + const int moves = dialog_choice_moves(*row, index); + const DpadPosition direction = moves > 0 ? DPAD_DOWN : DPAD_UP; + for (int moved = 0; moved < std::abs(moves); moved++){ + pbf_press_dpad(context, direction, 160ms, 240ms); + } + if (moves != 0){ + choice_arrow_shows(console, context, 2s); + const std::optional now = read_choice_row(console); + if (now != index){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::format( + "choose_dialog_option(): the highlight is on row {}, not {}, after {} presses from row {}.", + now ? std::to_string(*now) : "none", index, moves, *row + ), + console + ); + } + } + pbf_press_button(context, BUTTON_A, 160ms, 240ms); + context.wait_for_all_requests(); +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryDialogs.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryDialogs.h new file mode 100644 index 0000000000..d05039dee7 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryDialogs.h @@ -0,0 +1,109 @@ +/* PokemonLZA AutoStory - Dialogs + * + * From: https://github.com/PokemonAutomation/ + * + * The story's dialogue actions (PokemonLZA_AutoStoryActions.h has the + * rest): through a scene to the overworld, a talk, a choice. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryDialogs_H +#define PokemonAutomation_PokemonLZA_AutoStoryDialogs_H + +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "NintendoSwitch/Controllers/NintendoSwitch_ControllerButtons.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "NintendoSwitch/NintendoSwitch_ConsoleHandle.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryScreenEvents.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// From a dialogue or a scene, runs `presses` (repeating it) until the +// overworld is back: OverworldPartySelectionWatcher held OVERWORLD_HOLD. +// Throws OperationFailedException with `failure` if it is not back within +// `timeout`. +void press_until_overworld( + ConsoleHandle& console, ProControllerContext& context, + const std::function& presses, + Milliseconds timeout, std::string_view failure +); + +// press_until_overworld() with `button` mashed. +void mash_to_overworld(ConsoleHandle& console, ProControllerContext& context, Button button, Milliseconds timeout); + +// What press_a_until_quiet() pressed A through. +struct ScreensPassed{ + bool card = false; + bool text = false; + bool timed_out = false; // still busy at the timeout +}; + +// press_a_until_quiet()'s clock. Both times count from the first look, +// not from the call: the presses sent before it still run then (03.10, +// The menu's A and its 3 s ran out the quiet time, +// and the claim ended on one look with no dialog seen). +class QuietPresses{ +public: + // What press_a_until_quiet() does on a look. + enum class Step{ + PRESS_A, // a dialog or card shows + WAIT, // nothing shows, not for `quiet` yet + QUIET, // nothing has shown for `quiet`: done + TIMED_OUT, // `timeout` since the first look, still busy + }; + + QuietPresses(Milliseconds quiet, Milliseconds timeout) + : m_quiet(quiet) + , m_timeout(timeout) + {} + + // A look at `now`, `busy` when a dialog or card shows. + Step look(bool busy, WallClock now); + +private: + Milliseconds m_quiet; + Milliseconds m_timeout; + std::optional m_first; + std::optional m_last_busy; +}; + +// A through tutorial cards and the dialogue boxes of `kinds` +// (StoryDialogDetector) until none has shown for `quiet`, or `timeout` +// (QuietPresses: both from the first look, once the presses sent before +// the call have run). +ScreensPassed press_a_until_quiet( + ConsoleHandle& console, ProControllerContext& context, + StoryDialogKinds kinds, Milliseconds quiet, Milliseconds timeout +); + +// From the overworld, presses A at what the player faces. Returns once a +// dialogue box shows; throws OperationFailedException if none does. +void talk_to_facing(ConsoleHandle& console, ProControllerContext& context); + +// The D-pad presses from choice row `current` to row `target`: down when +// positive, up when negative. +int dialog_choice_moves(size_t current, size_t target); + +// With a dialogue's choices on screen, picks choice `index` (0 is the top): +// waits for the selection arrow in dialog_choice_box(), reads the +// highlighted row with read_dialog_choice_rows() (both in +// Detectors/PokemonLZA_AutoStoryDialogChoiceBox.h), presses the difference, +// reads the row again, then A. Throws OperationFailedException if the arrow +// never shows, no highlighted row reads, or the highlight is not on row +// `index` after the presses. +void choose_dialog_option(ConsoleHandle& console, ProControllerContext& context, size_t index); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryDialogs_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryDialogs_Tests.cpp new file mode 100644 index 0000000000..1918ce032d --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryDialogs_Tests.cpp @@ -0,0 +1,164 @@ +/* PokemonLZA AutoStory - Dialogs Tests + * + * From: https://github.com/PokemonAutomation/ + * + * Tables of cases: each a name, the row the cursor is on and the row + * wanted, and the presses that get there; or a name, the looks of A + * through dialogs until quiet, and what each look does. + * + */ + +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "PokemonLZA_AutoStoryDialogs.h" +#include "PokemonLZA_AutoStoryDialogs_Tests.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + +using namespace std::chrono_literals; + + +namespace{ + +// dialog_choice_moves(): the D-pad presses from one choice row to another. +struct ChoiceMovesCase{ + std::string name; + struct Input{ + size_t current; + size_t target; + }; + Input input; + struct Expectation{ + int moves; // down when positive, up when negative + }; + Expectation expectation; +}; + +const ChoiceMovesCase CHOICE_MOVES_CASES[] = { + { + .name = "opens on the top row, the top choice: no press", + .input = {.current = 0, .target = 0}, + .expectation = {.moves = 0}, + }, + { + .name = "opens on the top row, the second choice: one down", + .input = {.current = 0, .target = 1}, + .expectation = {.moves = 1}, + }, + { + .name = "opens on the top row, the third choice: two down", + .input = {.current = 0, .target = 2}, + .expectation = {.moves = 2}, + }, + { + .name = "opens on the second row, the second choice: no press", + .input = {.current = 1, .target = 1}, + .expectation = {.moves = 0}, + }, + { + .name = "opens on the second row, the top choice: one up", + .input = {.current = 1, .target = 0}, + .expectation = {.moves = -1}, + }, +}; + +class Test_ChoiceMoves : public UnitTest{ +public: + explicit Test_ChoiceMoves(const ChoiceMovesCase& test_case) + : UnitTest(std::format("PokemonLZA::AutoStory - Dialogs: choice moves: {}", test_case.name)) + , m_case(test_case) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const int moves = dialog_choice_moves(m_case.input.current, m_case.input.target); + if (moves != m_case.expectation.moves){ + return std::format("Expected {} moves, got {}.", m_case.expectation.moves, moves); + } + return true; + } + +private: + const ChoiceMovesCase& m_case; +}; + +// QuietPresses: what each look does, quiet 3 s and timeout 10 s. +struct QuietPressesCase{ + std::string name; + struct Look{ + Milliseconds at; // after the presses were sent + bool busy; // a dialog or a tutorial card on screen + }; + struct Input{ + std::vector looks; + }; + Input input; + struct Expectation{ + std::vector steps; + }; + Expectation expectation; +}; + +const QuietPressesCase QUIET_PRESSES_CASES[] = { + { + // 03.10's claim: the menu's A and its 3 s + // ran before the first look, which ended the claim on one look. + .name = "first look 5 s after the presses, nothing busy: the quiet time starts there", + .input = {.looks = {{.at = 5000ms, .busy = false}, {.at = 7000ms, .busy = false}, {.at = 8000ms, .busy = false}}}, + .expectation = {.steps = {QuietPresses::Step::WAIT, QuietPresses::Step::WAIT, QuietPresses::Step::QUIET}}, + }, + { + .name = "busy looks get A, the quiet time counts from the last", + .input = {.looks = {{.at = 0ms, .busy = true}, {.at = 1000ms, .busy = true}, {.at = 3500ms, .busy = false}, {.at = 4000ms, .busy = false}}}, + .expectation = {.steps = {QuietPresses::Step::PRESS_A, QuietPresses::Step::PRESS_A, QuietPresses::Step::WAIT, QuietPresses::Step::QUIET}}, + }, + { + .name = "still busy at the timeout from the first look: timed out", + .input = {.looks = {{.at = 2000ms, .busy = true}, {.at = 7000ms, .busy = true}, {.at = 12000ms, .busy = true}}}, + .expectation = {.steps = {QuietPresses::Step::PRESS_A, QuietPresses::Step::PRESS_A, QuietPresses::Step::TIMED_OUT}}, + }, +}; + +class Test_QuietPresses : public UnitTest{ +public: + explicit Test_QuietPresses(const QuietPressesCase& test_case) + : UnitTest(std::format("PokemonLZA::AutoStory - Dialogs: quiet presses: {}", test_case.name)) + , m_case(test_case) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const WallClock sent = current_time(); + QuietPresses clock(3000ms, 10000ms); + const std::vector& looks = m_case.input.looks; + for (size_t i = 0; i < looks.size(); i++){ + const QuietPresses::Step step = clock.look(looks[i].busy, sent + looks[i].at); + if (step != m_case.expectation.steps[i]){ + return std::format("Look {}: expected step {}, got {}.", i, (int)m_case.expectation.steps[i], (int)step); + } + } + return true; + } + +private: + using Look = QuietPressesCase::Look; + const QuietPressesCase& m_case; +}; + +} // namespace + + +void add_tests_AutoStoryDialogs(UnitTestDatabase& database){ + for (const ChoiceMovesCase& item : CHOICE_MOVES_CASES){ + database.add(item); + } + for (const QuietPressesCase& item : QUIET_PRESSES_CASES){ + database.add(item); + } +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryDialogs_Tests.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryDialogs_Tests.h new file mode 100644 index 0000000000..5b17377ad7 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryDialogs_Tests.h @@ -0,0 +1,26 @@ +/* PokemonLZA AutoStory - Dialogs Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryDialogs_Tests_H +#define PokemonAutomation_PokemonLZA_AutoStoryDialogs_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Tests that need no console: the D-pad presses from the choice row the +// cursor is on to the one wanted. +void add_tests_AutoStoryDialogs(UnitTestDatabase& database); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryScenes.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryScenes.cpp new file mode 100644 index 0000000000..3a32adbbfc --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryScenes.cpp @@ -0,0 +1,141 @@ +/* PokemonLZA AutoStory - Scenes + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryReadings.h" +#include "PokemonLZA_AutoStoryDialogs.h" +#include "PokemonLZA_AutoStoryScenes.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryFailures.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + +using namespace std::chrono_literals; + + +namespace{ + +// How often a mash looks when it waits. +constexpr Milliseconds MASH_POLL = 250ms; + +// Logs the look's events whenever they change (the still-screen flag left +// out), so a run's log shows each scene's fades, boxes and party bar. +void log_event_change(PlzaBackend& backend, const ScreenEvents& events, ScreenEvents& last){ + ScreenEvents shown = events; + shown.set(ScreenEvent::SCREEN_STILL, false); + if (shown == last){ + return; + } + last = shown; + backend.log(std::format("mash_scene(): screen now: {}", describe(shown))); +} + +void press_for(PlzaBackend& backend, SceneMashAction action, Button text_button){ + switch (action){ + case SceneMashAction::PRESS_TEXT: + backend.press(text_button, 160ms, 240ms); + return; + case SceneMashAction::PRESS_A: + backend.press(BUTTON_A, 160ms, 240ms); + return; + case SceneMashAction::PRESS_B: + backend.press(BUTTON_B, 160ms, 240ms); + return; + case SceneMashAction::WAIT: + case SceneMashAction::DONE: + backend.wait(MASH_POLL); + return; + } +} + +} // namespace + +Pokemon::Outcome mash_scene( + PlzaBackend& backend, SceneMashGoal goal, Button text_button, Milliseconds timeout, + std::optional start_window, SceneOpening opening +){ + SceneMash mash(goal, text_button); + ScreenEvents last_events; + // Set by the first look: when the mash started, and its scene's start + // window from there. + struct Started{ + WallClock at; + SceneStart scene; + }; + std::optional started; + // Every look presses or waits at least MASH_POLL on a console; a + // scripted backend's clock stands still between inputs. + const size_t max_looks = static_cast(timeout / MASH_POLL) + 1; + for (size_t looks = 0; looks < max_looks; looks++){ + backend.scope().throw_if_cancelled(); + const Pokemon::Observation look = backend.observe(); + if (!started){ + started = Started{look.when, SceneStart(look.when, start_window.value_or(timeout))}; + } + if (look.when - started->at >= timeout){ + break; + } + if (look.readings.contains(TUTORIAL_CARD_READING)){ + backend.log("mash_scene(): a tutorial card: A."); + started->scene.see(); + backend.press(BUTTON_A, 160ms, 840ms); + continue; + } + log_event_change(backend, look.events, last_events); + const bool never_started = started->scene.never_started(look.events, look.when); + if (start_window && never_started){ + return plza_failed("scene_never_started", std::format( + "mash_scene(): the scene never started (player not at its trigger?): no box, choice or battle in {} ms.", + start_window->count() + )); + } + const SceneMashAction action = mash.look(look.events, look.when); + if (action == SceneMashAction::DONE && (opening == SceneOpening::CONTROL_ENDS || started->scene.seen())){ + return Pokemon::outcome_ok(); + } + press_for(backend, action, text_button); + } + return plza_failed("scene_timeout", std::format("mash_scene(): the goal did not show within {} ms.", timeout.count())); +} + +Pokemon::Outcome mash_story_scene(PlzaBackend& backend, Button text_button, SceneOpening opening){ + return mash_scene(backend, SceneMashGoal::OVERWORLD, text_button, Milliseconds(120000), SCENE_START_WINDOW, opening); +} + + +// Taunie's talk, the Map & Missions card, the first map and the rest of her +// talk: A and B in turn, until the party bar has held (OVERWORLD_HOLD). A pages +// the talk and the card; B closes the map, which takes no A. The map is not +// waited for: MapDetector needs the Travel Spots icon, which this first +// map does not show. +void talk_through_map_screen(ConsoleHandle& console, ProControllerContext& context){ + press_until_overworld( + console, context, + [](ProControllerContext& context){ + pbf_press_button(context, BUTTON_A, 160ms, 840ms); + pbf_press_button(context, BUTTON_B, 160ms, 840ms); + }, + 120s, "talk_through_map_screen(): no overworld within 120 s of A and B in turn." + ); + console.log("talk_through_map_screen(): the party bar held: back in the overworld."); +} + +void climb_ladder(ConsoleHandle&, ProControllerContext& context){ + pbf_move_left_joystick(context, {1, 0}, 600ms, 300ms); + pbf_move_left_joystick(context, {0, 1}, 150ms, 300ms); + pbf_press_button(context, BUTTON_A, 160ms, 1000ms); + pbf_move_left_joystick(context, {0, 1}, 5000ms, 500ms); + context.wait_for_all_requests(); +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryScenes.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryScenes.h new file mode 100644 index 0000000000..03b31bdc58 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryScenes.h @@ -0,0 +1,75 @@ +/* PokemonLZA AutoStory - Scenes + * + * From: https://github.com/PokemonAutomation/ + * + * The story actions that press through scenes rather than walk: the scene + * mash over the story backend's looks (SceneMash, on the shared TalkOver), + * and the ladder, which is timed presses as recorded on the console. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryScenes_H +#define PokemonAutomation_PokemonLZA_AutoStoryScenes_H + +#include +#include "Common/Cpp/Time.h" +#include "NintendoSwitch/Controllers/NintendoSwitch_ControllerButtons.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "NintendoSwitch/NintendoSwitch_ConsoleHandle.h" +#include "Pokemon/AutoStory/Pokemon_StoryOutcome.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryBackend.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryScreenEvents.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Whether free control ends a scene mash before its scene has shown. +enum class SceneOpening{ + // Yes: the step before may have played the scene out already. + CONTROL_ENDS, + // No: control counts only after a box, choice or battle (SceneStart). + // A talk the step exists for: 03.10's mable_talk passed on control + // alone with no talk. + TALK_FIRST, +}; + +// From a scene, presses through it as SceneMash says on `backend`'s +// looks, until `goal` shows (OVERWORLD: the party bar held a second, by +// the shared TalkOver; with TALK_FIRST `opening`, only once the scene +// has shown). `text_button` advances the text (A or B). +// A tutorial card (TUTORIAL_CARD_READING) gets A whatever `text_button` +// is: only A turns its pages. With a `start_window`, the scene must start +// (SceneStart) within it. Fails as "scene_never_started" when it does not, +// and as "scene_timeout" when the goal does not show within `timeout` +// (by the looks' time). +Pokemon::Outcome mash_scene( + PlzaBackend& backend, SceneMashGoal goal, Button text_button, Milliseconds timeout, + std::optional start_window = std::nullopt, + SceneOpening opening = SceneOpening::CONTROL_ENDS +); + +// A "scene" step: mash_scene() to the overworld with `text_button`, up to +// 120 s (a scene can hold a ~25 s black screen: 02.07's first evening), +// and the scene must start within SCENE_START_WINDOW: a run not at the +// scene's trigger fails there, before an A mash talks to whoever stands +// in front. `opening` says whether control before the scene ends it. +Pokemon::Outcome mash_story_scene(PlzaBackend& backend, Button text_button, SceneOpening opening); + + +// Taunie's talk after her battle (03.02), the "Map & Missions" card, the +// first map and the rest of her talk: A and B in turn until the overworld +// has held. Throws OperationFailedException if it has not within 120 s. +void talk_through_map_screen(ConsoleHandle& console, ProControllerContext& context); + +// Facing the side street's wall (03.03): sidestep right to the ladder +// (0.6 s), face it (150 ms), A at "A Climb Up", hold up 5 s to the top. +void climb_ladder(ConsoleHandle& console, ProControllerContext& context); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryBackend.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryBackend.cpp new file mode 100644 index 0000000000..eff48c43d4 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryBackend.cpp @@ -0,0 +1,239 @@ +/* PokemonLZA AutoStory - Story backend + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include "CommonFramework/GlobalAutoPaths.h" +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/PrettyPrint.h" +#include "Common/Cpp/Color.h" +#include "CommonFramework/VideoPipeline/VideoOverlay.h" +#include "CommonTools/ScreenMachine/ScreenMachine.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "Pokemon/AutoStory/Engine/Pokemon_ScreenObservation.h" +#include "Pokemon/AutoStory/Engine/Pokemon_StoryActionNotImplemented.h" +#include "VideoGames/PokemonLZA/Programs/PokemonLZA_BasicNavigation.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryActions.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.h" +#include "PokemonLZA_AutoStoryBackend.h" +#include "PokemonLZA_AutoStoryFailures.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + +using namespace std::chrono_literals; +using Pokemon::Direction; + + +PlzaConsoleBackend::PlzaConsoleBackend(StoryEnv& env) + : m_env(env) + , m_home(env.env.console, COLOR_RED) +{} + +void PlzaConsoleBackend::press(Button button, Milliseconds hold, Milliseconds release){ + pbf_press_button(m_env.context, button, hold, release); +} + +void PlzaConsoleBackend::tap(Direction direction){ + story_turn(*this, Pokemon::direction_degrees(direction)); +} + +void PlzaConsoleBackend::hold(Direction direction, Milliseconds duration, Pokemon::StoryPace pace){ + stick(Pokemon::direction_degrees(direction), duration, 0ms, pace); +} + +void PlzaConsoleBackend::stick(double degrees, Milliseconds duration, Milliseconds release, Pokemon::StoryPace){ + const double radians = degrees * std::numbers::pi / 180.0; + pbf_move_left_joystick(m_env.context, {std::sin(radians), std::cos(radians)}, duration, release); +} + +void PlzaConsoleBackend::wait(Milliseconds duration){ + m_env.context.wait_for(duration); +} + +Pokemon::Observation PlzaConsoleBackend::observe(){ + m_env.context.wait_for_all_requests(); + VideoSnapshot frame = m_env.env.console.video().snapshot(); + Pokemon::Observation look; + if (frame && m_home.detect(*frame.frame)){ + look.events.set(PokemonAutomation::ScreenEvent::SWITCH_HOME_MENU, true); + }else if (frame){ + static_cast(look) = Pokemon::screen_observation(m_observer.read(*frame.frame)); + if (m_platform.detect(*frame.frame)){ + look.readings[std::string(PLATFORM_READING)] = 1; + } + if (m_skip_prompt.detect(*frame.frame)){ + look.readings[std::string(SKIP_PROMPT_READING)] = 1; + } + if (m_autosave.detect(*frame.frame)){ + look.readings[std::string(AUTOSAVE_READING)] = 1; + } + if (m_controls_hint.detect(*frame.frame)){ + look.readings[std::string(CONTROLS_HINT_READING)] = 1; + } + if (m_advance_arrow.detect(*frame.frame)){ + look.readings[std::string(ADVANCE_ARROW_READING)] = 1; + } + if (m_mission_complete.detect(*frame.frame)){ + look.readings[std::string(MISSION_COMPLETE_READING)] = 1; + }else if (m_mission_banner.detect(*frame.frame)){ + look.readings[std::string(MISSION_BANNER_READING)] = 1; + } + if (tutorial_card_shows(*frame.frame)){ + look.readings[std::string(TUTORIAL_CARD_READING)] = 1; + } + if (same_screen(m_previous, frame)){ + look.events.set(PokemonAutomation::ScreenEvent::SCREEN_STILL, true); + } + } + look.when = current_time(); + const std::optional autosave_line = + m_autosave_spots.on_look(look.readings.contains(AUTOSAVE_READING), look.when); + if (autosave_line){ + log(*autosave_line); + } + m_previous = frame; + look.frame = std::move(frame); + capture(look); + return look; +} + +namespace{ + +// The look's events and readings, for a capture's file name: +// "black_fade+screen_still", "nothing+skip_prompt", +// "player_can_move+platform+autosave+controls_hint". +std::string look_text(const ScreenObservation& look){ + std::string text = describe(look.events, "+"); + std::replace(text.begin(), text.end(), ' ', '_'); + if (look.readings.contains(PLATFORM_READING)){ + text += "+platform"; + } + if (look.readings.contains(SKIP_PROMPT_READING)){ + text += "+skip_prompt"; + } + if (look.readings.contains(AUTOSAVE_READING)){ + text += "+autosave"; + } + if (look.readings.contains(CONTROLS_HINT_READING)){ + text += "+controls_hint"; + } + if (look.readings.contains(ADVANCE_ARROW_READING)){ + text += "+advance_arrow"; + } + if (look.readings.contains(MISSION_BANNER_READING)){ + text += "+mission_banner"; + } + if (look.readings.contains(MISSION_COMPLETE_READING)){ + text += "+mission_complete"; + } + if (look.readings.contains(TUTORIAL_CARD_READING)){ + text += "+tutorial_card"; + } + return text; +} + +constexpr Milliseconds CAPTURE_EVERY(2000); + +} // namespace + +void PlzaConsoleBackend::capture(const Pokemon::Observation& look){ + if (!m_env.capture_looks || !look.frame || !look.frame->frame){ + return; + } + const std::string text = look_text(look); + const bool changed = m_captured_text != text; + if (!changed && look.when - m_captured_at < CAPTURE_EVERY){ + return; + } + if (m_capture_folder.empty()){ + m_capture_folder = std::format("{}PLZA-capture/{}/", SCREENSHOTS_PATH(), now_to_filestring()); + Filesystem::create_directories(Filesystem::Path(m_capture_folder)); + } + m_captures++; + const std::string path = std::format("{}{:04}-{}.png", m_capture_folder, m_captures, text); + look.frame->frame->save(path); + m_env.env.console.log(std::format("[PLZA capture] {:04} {}{}, platform {}", m_captures, text, + changed ? " (changed)" : "", to_string(m_platform.last_distances()))); + m_captured_text = text; + m_captured_at = look.when; +} + +Pokemon::Outcome PlzaConsoleBackend::battle(const Pokemon::StoryActionArgs&){ + try{ + fight_battle(m_env.env, m_env.env.console, m_env.context, m_env.battle_ai); + }catch (Pokemon::StoryActionNotImplemented&){ + throw; + }catch (OperationFailedException& e){ + return plza_failed("battle_failed", e.message()); + } + return Pokemon::outcome_ok(); +} + +namespace{ + +// A look without the badge shorter than this is the same appearance. +constexpr Milliseconds AUTOSAVE_GAP(4000); + +} // namespace + +void AutosaveSpots::on_log(std::string_view line){ + constexpr std::string_view EVENT = "[Event] "; + constexpr std::string_view STARTED = ": start"; + if (!line.starts_with(EVENT) || !line.ends_with(STARTED)){ + return; + } + m_event = std::string(line.substr(EVENT.size(), line.size() - EVENT.size() - STARTED.size())); +} + +std::optional AutosaveSpots::on_look(bool badge, WallClock when){ + if (!badge){ + return std::nullopt; + } + const bool same = m_last_badge && when - *m_last_badge < AUTOSAVE_GAP; + m_last_badge = when; + if (same){ + return std::nullopt; + } + return std::format("[Autosave] {}", m_event); +} + +void PlzaConsoleBackend::log(std::string_view line){ + m_autosave_spots.on_log(line); + m_env.env.console.log(std::string(line), COLOR_BLUE); + // The checkpoint, the step under way and the autosaves, for whoever + // watches the stream: the engine's lines all come through here. + m_env.env.console.overlay().add_log(std::string(line)); +} + +CancellableScope& PlzaConsoleBackend::scope(){ + return m_env.context; +} + +void story_turn(PlzaBackend& backend, double degrees){ + // 150 ms is the tap the story was recorded with. + // Short releases: the legs run back to back, as when played by hand. + backend.press(BUTTON_L, 100ms, 150ms); + backend.stick(degrees, 150ms, 100ms, Pokemon::StoryPace::WALK); +} + +PlzaReader heading_reader(StoryEnv& env){ + return [&env](PlzaBackend&, Pokemon::Observation& observation){ + const double heading = get_facing_direction(env.env.console, env.context); + const int degrees = static_cast(std::lround(heading)); + observation.readings[std::string(Pokemon::STORY_HEADING)] = ((degrees % 360) + 360) % 360; + }; +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryBackend.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryBackend.h new file mode 100644 index 0000000000..290edbf27f --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryBackend.h @@ -0,0 +1,171 @@ +/* PokemonLZA AutoStory - Story backend + * + * From: https://github.com/PokemonAutomation/ + * + * PLZA on the console for the engine's backend + * (Pokemon/AutoStory/Engine/Pokemon_StoryBackend.h), and the inputs a game + * with no tiles builds from it. + * + * Directions are the stick relative to the camera: north is straight on, + * east to the right. tap() turns that way (story_turn()); hold() walks. + * Dash is a toggle in this game and the walks assume it is off, so + * StoryPace::RUN walks too. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryBackend_H +#define PokemonAutomation_PokemonLZA_AutoStoryBackend_H + +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "CommonFramework/Language.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "NintendoSwitch/NintendoSwitch_SingleSwitchProgram.h" +#include "NintendoSwitch/Inference/NintendoSwitch_HomeMenuDetector.h" +#include "Pokemon/AutoStory/Engine/Pokemon_StoryBackend.h" +#include "Pokemon/AutoStory/Engine/Pokemon_StoryGame.h" +#include "Pokemon/AutoStory/Types/Pokemon_StoryCheckLevel.h" +#include "VideoGames/PokemonLZA/Options/PokemonLZA_BattleAIOption.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.h" +#include "PokemonLZA_AutoStoryReadings.h" +#include "PokemonLZA_AutoStoryScreenEvents.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The starters Mission 1 offers. The player picks one by walking up to it +// and talking to it. +enum class StarterChoice{ + CHIKORITA, + TEPIG, + TOTODILE, +}; + +// The console a run plays on, for the actions only a console can do +// (menus, OCR, the battle AI). +struct StoryEnv{ + SingleSwitchProgramEnvironment& env; + ProControllerContext& context; + // The language the game is set to, for the steps that read text (the + // fast travel menu's place names). + Language language = Language::None; + BattleAIOption& battle_ai; + // The run's check level (StoryRunOptions::CHECKS): how much the steps + // read to confirm what they did. + Pokemon::StoryCheckLevel checks = Pokemon::StoryCheckLevel::DEFAULT; + // Whether observe() saves a frame each time the look's events change + // (and every 2 s) under SCREENSHOTS_PATH()/PLZA-capture/, to find the + // screens the detectors miss. Off by default: a long run writes + // thousands of images. + bool capture_looks = false; +}; + + +// The engine's backend for PLZA's pose. +using PlzaBackend = Pokemon::StoryBackend; +// A reader for PLZA's pose (StoryGame::readers). +using PlzaReader = Pokemon::StoryReader; + +// Turns the player `degrees` clockwise of straight on (negative: left), +// as the story was recorded: L recenters the camera behind the player, +// then the stick is tapped that way for 150 ms. The tap also takes a step. +void story_turn(PlzaBackend& backend, double degrees); + +// The STORY_HEADING reading for the console: the minimap's arrow +// (get_facing_direction()), clockwise from north in [0, 360). It waits up +// to 40 s for the minimap and throws without it, so its StoryGameReader +// in make_plza_story_game() is ON_REQUEST and console_only: no +// postcondition check runs it. +PlzaReader heading_reader(StoryEnv& env); + + +// Where the game autosaved, for the log: each appearance of the autosave +// badge once, as "[Autosave] 02.04/3 check_in", the event the engine last +// logged as started. The badge shows 1-3 s; one looked away from (a missed +// look) less than AUTOSAVE_GAP is the same appearance. +class AutosaveSpots{ +public: + // Takes in a log line; "[Event] 02.04/3 check_in: start" (the engine's + // event_log_line()) names the event. + void on_log(std::string_view line); + + // A look at `when`: the log line when it shows a new appearance. + std::optional on_look(bool badge, WallClock when); + +private: + std::string m_event = "(before the first event)"; + std::optional m_last_badge; +}; + + +// PLZA on the console: the controller and PlzaScreenObserver. +// code-health: allow public_methods -- the engine's backend calls; nothing PLZA adds +class PlzaConsoleBackend : public PlzaBackend{ +public: + explicit PlzaConsoleBackend(StoryEnv& env); + + virtual Pokemon::StoryBackendKind kind() const override{ + return Pokemon::StoryBackendKind::CONSOLE; + } + virtual void press(Button button, Milliseconds hold, Milliseconds release) override; + virtual void tap(Pokemon::Direction direction) override; + virtual void hold(Pokemon::Direction direction, Milliseconds duration, Pokemon::StoryPace pace) override; + virtual void stick(double degrees, Milliseconds duration, Milliseconds release, Pokemon::StoryPace pace) override; + virtual void wait(Milliseconds duration) override; + // Once the inputs are done: SWITCH_HOME_MENU alone on the Switch's + // HOME (PLZA's detectors may read its plain screen as the overworld); + // else PlzaScreenObserver's events on a frame + // (Pokemon::screen_observation()), STILL when it barely differs from the frame + // of the look before, and the readings PLATFORM_READING, + // SKIP_PROMPT_READING, AUTOSAVE_READING, CONTROLS_HINT_READING, + // ADVANCE_ARROW_READING, MISSION_BANNER_READING and + // MISSION_COMPLETE_READING (which wins over the banner: the card has + // the banner's bar), and TUTORIAL_CARD_READING. Logs each appearance + // of the autosave badge (AutosaveSpots). + virtual Pokemon::Observation observe() override; + // fight_battle() with the run's battle AI; a failure is an Outcome. + virtual Pokemon::Outcome battle(const Pokemon::StoryActionArgs& args) override; + virtual void log(std::string_view line) override; + virtual CancellableScope& scope() override; + +private: + StoryEnv& m_env; + HomeMenuDetector m_home; + PlzaScreenObserver m_observer; + PlatformArrivalDetector m_platform; + SkipPromptDetector m_skip_prompt; + AutosaveDetector m_autosave; + AutosaveSpots m_autosave_spots; + ControlsHintDetector m_controls_hint; + AdvanceArrowDetector m_advance_arrow; + MainMissionBannerDetector m_mission_banner; + MissionCompleteDetector m_mission_complete; + VideoSnapshot m_previous; + // With StoryEnv::capture_looks: every look's events, and a frame each + // time they change (and every CAPTURE_EVERY), go to the log and a + // capture folder: the record of what the detectors saw, to find the + // screens they miss. + void capture(const Pokemon::Observation& look); + std::string m_capture_folder; + std::optional m_captured_text; + WallClock m_captured_at; + size_t m_captures = 0; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/PokemonLZA_TrainerBattle.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/PokemonLZA_TrainerBattle.cpp index 3439e6ab17..a80ad34a39 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/PokemonLZA_TrainerBattle.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/PokemonLZA_TrainerBattle.cpp @@ -12,6 +12,7 @@ #include "NintendoSwitch/Commands/NintendoSwitch_Commands_Superscalar.h" #include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" #include "VideoGames/PokemonLZA/Inference/Battles/PokemonLZA_MoveEffectivenessSymbol.h" +#include "VideoGames/PokemonLZA/Inference/Battles/PokemonLZA_RunFromBattleDetector.h" #include "PokemonLZA_TrainerBattle.h" //#include @@ -140,6 +141,41 @@ void TrainerBattleState::run_lock_recovery(ConsoleHandle& console, ProController } +int run_battle_until( + ProgramEnvironment& env, + ConsoleHandle& console, + ProControllerContext& context, + TrainerBattleState& battle_state, + VisualInferenceCallback& end_watcher, + Milliseconds mash_time, + const std::function& after_failed_attack +){ + while (true){ + RunFromBattleWatcher battle_menu(COLOR_GREEN, &console.overlay(), 10ms); + context.wait_for_all_requests(); + int ret = run_until( + console, context, + [mash_time](ProControllerContext& context){ + pbf_mash_button(context, BUTTON_B, mash_time); + }, + {battle_menu, end_watcher} + ); + switch (ret){ + case 0: + console.log("Detected battle menu."); + if (!battle_state.attempt_one_attack(env, console, context) && after_failed_attack){ + after_failed_attack(context); + } + continue; + case 1: + return 0; + default: + return -1; + } + } +} + + diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/PokemonLZA_TrainerBattle.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/PokemonLZA_TrainerBattle.h index cfc1c3e2a6..18048129fd 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/PokemonLZA_TrainerBattle.h +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/PokemonLZA_TrainerBattle.h @@ -7,7 +7,9 @@ #ifndef PokemonAutomation_PokemonLZA_TrainerBattle_H #define PokemonAutomation_PokemonLZA_TrainerBattle_H -//#include "Common/Cpp/Time.h" +#include +#include "Common/Cpp/Time.h" +#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h" #include "NintendoSwitch/NintendoSwitch_ConsoleHandle.h" #include "VideoGames/PokemonLZA/Options/PokemonLZA_BattleAIOption.h" @@ -55,6 +57,24 @@ class TrainerBattleState{ }; +// Fight a battle until `end_watcher` fires: mash B (it clears battle dialogs) +// until the battle menu (RunFromBattleWatcher) or `end_watcher` shows; on the +// battle menu, attack once with `battle_state` and repeat. +// - after_failed_attack: if set, called when an attack finds no move to use. +// Return: +// - 0 if `end_watcher` fires +// - -1 if neither the battle menu nor `end_watcher` shows for `mash_time` +int run_battle_until( + ProgramEnvironment& env, + ConsoleHandle& console, + ProControllerContext& context, + TrainerBattleState& battle_state, + VisualInferenceCallback& end_watcher, + Milliseconds mash_time = Milliseconds(120000), + const std::function& after_failed_attack = nullptr +); + + } diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 8ee0d91478..c1c16db4b8 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -2367,6 +2367,14 @@ file(GLOB LIBRARY_SOURCES Source/VideoGames/PokemonLZA/PokemonLZA_Settings.h Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp Source/VideoGames/PokemonLZA/PokemonLZA_Tests.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryActions.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryActions.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryDialogs.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryDialogs.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryDialogs_Tests.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryDialogs_Tests.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryScenes.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryScenes.h Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.cpp @@ -2407,6 +2415,8 @@ file(GLOB LIBRARY_SOURCES Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.h Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.cpp Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryBackend.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryBackend.h Source/VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryFailures.cpp Source/VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryFailures.h Source/VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryFailures_Tests.cpp