diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.cpp index 3f350a27a0..b129c10e89 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.cpp @@ -205,8 +205,11 @@ void FlatWhiteDialogDetector::make_overlays(VideoOverlaySet& items) const{ items.add(m_color, m_bottom); items.add(m_color, DIALOG_ARROW_BOX); } +bool FlatWhiteDialogDetector::detect_box(const ImageViewRGB32& screen) const{ + return is_white(extract_box_reference(screen, m_bottom), 500.0, 20.0); +} bool FlatWhiteDialogDetector::detect(const ImageViewRGB32& screen){ - if (!is_white(extract_box_reference(screen, m_bottom), 500.0, 20.0)){ + if (!detect_box(screen)){ m_last_detected_box_scope.reset(); return false; } diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.h index b459a15596..56a37ef40f 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.h +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.h @@ -31,6 +31,10 @@ class FlatWhiteDialogDetector : public StaticScreenDetector{ // This is not const so that detectors can save/cache state. virtual bool detect(const ImageViewRGB32& screen) override; + // Only the box's white bottom edge, without the advance arrow: the box + // shows while its text is still being typed. + bool detect_box(const ImageViewRGB32& screen) const; + private: friend class FlatWhiteDialogWatcher; diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp index 757272eea3..70fb023ef1 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp @@ -21,7 +21,10 @@ #include "Inference/PokemonLZA_OverworldPartySelectionDetector.h" #include "Inference/PokemonLZA_PartyBarDetector.h" #include "Inference/PokemonLZA_SelectionArrowDetector.h" +#include "Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader_Tests.h" +#include "Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox_Tests.h" #include "Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h" +#include "Programs/AutoStory/Detectors/PokemonLZA_AutoStoryObjectiveMarkerReader_Tests.h" namespace PokemonAutomation{ namespace NintendoSwitch{ @@ -47,6 +50,9 @@ void add_tests(UnitTestDatabase& database){ add_tests_PartyBarDetector(database); add_tests_SelectionArrowDetector(database); add_tests_AutoStoryDetectors(database); + add_tests_AutoStoryConfirmChoicesReader(database); + add_tests_AutoStoryDialogChoiceBox(database); + add_tests_AutoStoryObjectiveMarkerReader(database); } diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader.cpp new file mode 100644 index 0000000000..27186e99bf --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader.cpp @@ -0,0 +1,133 @@ +/* PokemonLZA AutoStory - Confirm Choices reader + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "PokemonLZA_AutoStoryConfirmChoicesReader.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +enum class RowBox{ + HIGHLIGHTED, + PLAIN, + OTHER, +}; + +double brightness(const ImageStats& stats){ + return stats.average.sum() / 3; +} + +// The right ends of the "Confirm Choices" rows, right of their text: the +// boxes span x 144 to 742 at 1920 x 1080, the two choices from y 578 and +// 662 (68 high), "Ready!" from y 790 (112 high). Each strip is 100 wide +// from x 620. +ImageFloatBox confirm_row_strip(size_t row){ + switch (row){ + case 0: + return ImageFloatBox(620.0 / 1920, 592.0 / 1080, 100.0 / 1920, 40.0 / 1080); + case 1: + return ImageFloatBox(620.0 / 1920, 676.0 / 1080, 100.0 / 1920, 40.0 / 1080); + default: + return ImageFloatBox(620.0 / 1920, 820.0 / 1080, 100.0 / 1920, 60.0 / 1080); + } +} + +// A "Confirm Choices" row's strip: the highlighted row white, flat; the +// others slate. The highlight's glow pulses over the row next to it: on +// the console's frames in ten languages that row's spread went +// up to 20. +RowBox classify_confirm_row(const ImageStats& stats){ + if (stats.count == 0){ + return RowBox::OTHER; + } + const FloatPixel& average = stats.average; + const double spread = stats.stddev.sum(); + if (average.r >= 200 && average.g >= 200 && average.b >= 200 && spread <= 30){ + return RowBox::HIGHLIGHTED; + } + const bool slate = + average.r >= 20 && average.r <= 55 && + average.g >= 40 && average.g <= 80 && + average.b >= 65 && average.b <= 115 && + average.b >= average.r + 25; + if (slate && spread <= 30){ + return RowBox::PLAIN; + } + return RowBox::OTHER; +} + +// The dark blue, dotted background round the rows: between the choices and +// "Ready!", and right of all three. The highlight's blue glow reaches 40 +// pixels out and fills the gap under "Change looks" when that row is +// highlighted: (12-13, 56-59, 102-106), spread up to 51, against the plain +// background's (10-11, 29-37, 63-75), spread up to 31 (the console, ten +// languages). Both are dark and blue with little red; the +// intro's other screens are not. +bool confirm_background_dark(const ImageViewRGB32& frame){ + const ImageFloatBox boxes[] = { + ImageFloatBox(620.0 / 1920, 742.0 / 1080, 100.0 / 1920, 40.0 / 1080), + ImageFloatBox(770.0 / 1920, 580.0 / 1080, 30.0 / 1920, 320.0 / 1080), + }; + for (const ImageFloatBox& box : boxes){ + const ImageStats stats = image_stats(extract_box_reference(frame, box)); + if (stats.count == 0 || brightness(stats) >= 90 || stats.stddev.sum() > 80){ + return false; + } + if (stats.average.r > 30 || stats.average.b < stats.average.r + 40){ + return false; + } + } + return true; +} + +} // namespace + + +// Measured on the console, "Confirm Choices" as it opens: the +// highlighted row 255 white, flat; the others slate, (35-36, 52-61, +// 82-95), spread under 10; the background (10, 29-31, 64-66), spread under +// 25. The intro's language list and look picker fall on the background's +// strips, so neither reads as this screen. +std::optional read_confirm_choices_cursor(const ImageViewRGB32& frame){ + if (!frame || frame.width() < 160 || frame.height() < 90){ + return std::nullopt; + } + if (!confirm_background_dark(frame)){ + return std::nullopt; + } + std::optional highlighted; + for (size_t row = 0; row <= CONFIRM_CHOICES_READY_ROW; row++){ + const ImageStats stats = image_stats(extract_box_reference(frame, confirm_row_strip(row))); + switch (classify_confirm_row(stats)){ + case RowBox::HIGHLIGHTED: + if (highlighted){ + return std::nullopt; + } + highlighted = row; + break; + case RowBox::PLAIN: + break; + case RowBox::OTHER: + return std::nullopt; + } + } + return highlighted; +} + +bool is_confirm_choices(const ImageViewRGB32& frame){ + return read_confirm_choices_cursor(frame).has_value(); +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader.h new file mode 100644 index 0000000000..f675df1f58 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader.h @@ -0,0 +1,40 @@ +/* PokemonLZA AutoStory - Confirm Choices reader + * + * From: https://github.com/PokemonAutomation/ + * + * The new game's "Confirm Choices" screen, where checkpoint 01.00 starts: + * the PROFILE screen after the name keyboard, with its rows "Change hair + * color", "Change look" and "Ready!". + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryConfirmChoicesReader_H +#define PokemonAutomation_PokemonLZA_AutoStoryConfirmChoicesReader_H + +#include +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// "Confirm Choices" has three rows: "Change hair color" (0, highlighted as +// it opens), "Change look" and "Ready!", the last. +constexpr size_t CONFIRM_CHOICES_READY_ROW = 2; + +// On "Confirm Choices", the highlighted row, 0 to +// CONFIRM_CHOICES_READY_ROW: the one white box of the three, the others +// slate. Nullopt for any other frame, an empty one included. +std::optional read_confirm_choices_cursor(const ImageViewRGB32& frame); + +// "Confirm Choices", with one row highlighted. +bool is_confirm_choices(const ImageViewRGB32& frame); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader_Tests.cpp new file mode 100644 index 0000000000..8f2927370e --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader_Tests.cpp @@ -0,0 +1,260 @@ +/* PokemonLZA AutoStory - Confirm Choices reader Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "Tests/ImageUnitTest.h" +#include "VideoGames/PokemonLZA/PokemonLZA_PendingFixtures.h" +#include "PokemonLZA_AutoStoryConfirmChoicesReader.h" +#include "PokemonLZA_AutoStoryConfirmChoicesReader_Tests.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +constexpr uint32_t BACKGROUND = 0xff0a1d40; +constexpr uint32_t SLATE = 0xff23344f; +constexpr uint32_t WHITE = 0xffffffff; + +// "Confirm Choices" drawn as the console shows it (1920 x 1080): dark blue, +// three slate rows x 144 to 742, the two choices from y 578 and 662 (68 +// high) and "Ready!" from y 790 (112 high), the rows in `highlighted` +// white. +ImageRGB32 confirm_choices_frame(std::initializer_list highlighted){ + ImageRGB32 image(1920, 1080); + image.fill(BACKGROUND); + const size_t tops[] = {578, 662, 790}; + const size_t heights[] = {68, 68, 112}; + for (size_t row = 0; row <= CONFIRM_CHOICES_READY_ROW; row++){ + bool lit = false; + for (const size_t pick : highlighted){ + lit = lit || pick == row; + } + for (size_t y = tops[row]; y < tops[row] + heights[row]; y++){ + for (size_t x = 144; x < 742; x++){ + image.pixel(x, y) = lit ? WHITE : SLATE; + } + } + } + return image; +} + +ImageRGB32 flat_frame(uint32_t color){ + ImageRGB32 image(1920, 1080); + image.fill(color); + return image; +} + +std::string describe_cursor(std::optional cursor){ + return cursor ? std::format("row {}", *cursor) : std::string("no Confirm Choices"); +} + + +struct SyntheticCase{ + std::string name; + struct Input{ + std::function frame; + }; + Input input; + struct Expectation{ + std::optional cursor; // The row read; none: not Confirm Choices. + }; + Expectation expectation; +}; + +const SyntheticCase SYNTHETIC_CASES[] = { + {.name = "row 0 highlighted", .input = {.frame = []{ return confirm_choices_frame({0}); }}, .expectation = {.cursor = 0}}, + {.name = "row 1 highlighted", .input = {.frame = []{ return confirm_choices_frame({1}); }}, .expectation = {.cursor = 1}}, + { + .name = "Ready! highlighted", + .input = {.frame = []{ return confirm_choices_frame({CONFIRM_CHOICES_READY_ROW}); }}, + .expectation = {.cursor = CONFIRM_CHOICES_READY_ROW}, + }, + {.name = "no row highlighted", .input = {.frame = []{ return confirm_choices_frame({}); }}, .expectation = {.cursor = std::nullopt}}, + {.name = "two rows highlighted", .input = {.frame = []{ return confirm_choices_frame({0, 1}); }}, .expectation = {.cursor = std::nullopt}}, + {.name = "flat black", .input = {.frame = []{ return flat_frame(0xff000000); }}, .expectation = {.cursor = std::nullopt}}, + {.name = "flat white", .input = {.frame = []{ return flat_frame(WHITE); }}, .expectation = {.cursor = std::nullopt}}, + {.name = "flat background", .input = {.frame = []{ return flat_frame(BACKGROUND); }}, .expectation = {.cursor = std::nullopt}}, + {.name = "flat slate", .input = {.frame = []{ return flat_frame(SLATE); }}, .expectation = {.cursor = std::nullopt}}, + {.name = "an empty frame", .input = {.frame = []{ return ImageRGB32(); }}, .expectation = {.cursor = std::nullopt}}, + {.name = "a 1 x 1 frame", .input = {.frame = []{ return ImageRGB32(1, 1); }}, .expectation = {.cursor = std::nullopt}}, + {.name = "a 40 x 20 frame", .input = {.frame = []{ return ImageRGB32(40, 20); }}, .expectation = {.cursor = std::nullopt}}, +}; + +class Test_ConfirmChoicesSynthetic : public UnitTest{ +public: + explicit Test_ConfirmChoicesSynthetic(const SyntheticCase& test) + : UnitTest(std::format("PokemonLZA::AutoStory - Confirm Choices reader: {}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::optional cursor = read_confirm_choices_cursor(m_test.input.frame()); + if (cursor != m_test.expectation.cursor){ + return std::format("Expected {}, read {}.", describe_cursor(m_test.expectation.cursor), describe_cursor(cursor)); + } + return true; + } + +private: + const SyntheticCase& m_test; +}; + + +// The folder named by the environment variable `variable`; none if it is +// unset or not a folder. +std::optional env_folder(std::string_view variable){ + const auto* value = std::getenv(std::string(variable).c_str()); + if (value == nullptr){ + return std::nullopt; + } + std::string folder(value); + if (!Filesystem::is_directory(Filesystem::Path(folder))){ + return std::nullopt; + } + return folder; +} + +// 01.00's start check on the console's intro screens, from the folder in +// PLZA_INTRO_SCREENS: it accepts Confirm Choices as it opens, and refuses +// every other intro screen. A file the folder lacks is skipped. +struct CaptureCase{ + std::string name; + struct Input{ + std::string_view file; + }; + Input input; + struct Expectation{ + std::optional cursor; + }; + Expectation expectation; +}; + +const CaptureCase CAPTURE_CASES[] = { + {.name = "Confirm Choices as it opens", .input = {.file = "confirm-choices.png"}, .expectation = {.cursor = 0}}, + {.name = "HOME", .input = {.file = "home.png"}, .expectation = {.cursor = std::nullopt}}, + {.name = "a game in front", .input = {.file = "in-game.png"}, .expectation = {.cursor = std::nullopt}}, + {.name = "the language list", .input = {.file = "language-list.png"}, .expectation = {.cursor = std::nullopt}}, + {.name = "the look picker", .input = {.file = "look-picker.png"}, .expectation = {.cursor = std::nullopt}}, + {.name = "the name keyboard", .input = {.file = "keyboard.png"}, .expectation = {.cursor = std::nullopt}}, +}; + +class Test_ConfirmChoicesCapture : public UnitTest{ +public: + explicit Test_ConfirmChoicesCapture(const CaptureCase& test) + : UnitTest(std::format("PokemonLZA::AutoStory - Confirm Choices reader: {} (PLZA_INTRO_SCREENS)", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::optional folder = env_folder("PLZA_INTRO_SCREENS"); + if (!folder){ + return UnitTestResult(UnitTestResult::SKIPPED, "PLZA_INTRO_SCREENS is not a folder."); + } + const std::string path = std::format("{}/{}", *folder, m_test.input.file); + if (!Filesystem::exists(Filesystem::Path(path))){ + return UnitTestResult(UnitTestResult::SKIPPED, std::format("{} is not in the folder.", m_test.input.file)); + } + const std::optional cursor = read_confirm_choices_cursor(ImageRGB32(path)); + if (cursor != m_test.expectation.cursor){ + return std::format( + "{}: expected {}, read {}.", m_test.input.file, describe_cursor(m_test.expectation.cursor), describe_cursor(cursor) + ); + } + return true; + } + +private: + const CaptureCase& m_test; +}; + + +// "Confirm Choices" on the console in each game language, one frame per +// highlighted row: PokemonLZA/AutoStoryConfirmChoices/ +// _.png, row 0 to CONFIRM_CHOICES_READY_ROW. Each frame +// reads as its row. +struct LanguageFramesCase{ + std::string name; + struct Input{ + std::string_view language; + }; + Input input; + struct Expectation{ + }; + Expectation expectation; +}; + +const LanguageFramesCase LANGUAGE_FRAMES_CASES[] = { + {.name = "English", .input = {.language = "English"}, .expectation = {}}, + {.name = "Spanish", .input = {.language = "Spanish"}, .expectation = {}}, + {.name = "Latin American Spanish", .input = {.language = "SpanishLatAm"}, .expectation = {}}, + {.name = "French", .input = {.language = "French"}, .expectation = {}}, + {.name = "German", .input = {.language = "German"}, .expectation = {}}, + {.name = "Italian", .input = {.language = "Italian"}, .expectation = {}}, + {.name = "Japanese", .input = {.language = "Japanese"}, .expectation = {}}, + {.name = "Korean", .input = {.language = "Korean"}, .expectation = {}}, + {.name = "Simplified Chinese", .input = {.language = "ChineseSimplified"}, .expectation = {}}, + {.name = "Traditional Chinese", .input = {.language = "ChineseTraditional"}, .expectation = {}}, +}; + +std::vector language_frame_paths(std::string_view language){ + std::vector paths; + for (size_t row = 0; row <= CONFIRM_CHOICES_READY_ROW; row++){ + paths.push_back(std::format("PokemonLZA/AutoStoryConfirmChoices/{}_{}.png", language, row)); + } + return paths; +} + +class Test_ConfirmChoicesFrames : public ImageUnitTest{ +public: + explicit Test_ConfirmChoicesFrames(const LanguageFramesCase& test) + : ImageUnitTest( + std::format("PokemonLZA::AutoStory - Confirm Choices - {}", test.name), + language_frame_paths(test.input.language) + ) + {} + virtual UnitTestResult run_on_images( + Logger&, CancellableScope&, const std::vector& images + ) const override{ + for (size_t row = 0; row < images.size(); row++){ + const std::optional cursor = read_confirm_choices_cursor(images[row]); + if (cursor != row){ + return std::format("{}: expected row {}, read {}.", image_paths()[row], row, describe_cursor(cursor)); + } + } + return true; + } +}; + +} // namespace + + +void add_tests_AutoStoryConfirmChoicesReader(UnitTestDatabase& database){ + for (const SyntheticCase& test : SYNTHETIC_CASES){ + database.add(test); + } + for (const CaptureCase& test : CAPTURE_CASES){ + database.add(test); + } + for (const LanguageFramesCase& test : LANGUAGE_FRAMES_CASES){ + add_pending_fixture_test(database, test); + } +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader_Tests.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader_Tests.h new file mode 100644 index 0000000000..72f0b5e74f --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader_Tests.h @@ -0,0 +1,28 @@ +/* PokemonLZA AutoStory - Confirm Choices reader Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryConfirmChoicesReader_Tests_H +#define PokemonAutomation_PokemonLZA_AutoStoryConfirmChoicesReader_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Registers the Confirm Choices reader's tests: synthetic frames always; +// the console's frames in each language from the test images +// (PokemonLZA/AutoStoryConfirmChoices/), and the other intro screens from +// the folder in PLZA_INTRO_SCREENS, skipped when unset. +void add_tests_AutoStoryConfirmChoicesReader(UnitTestDatabase& database); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDexRegisteredDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDexRegisteredDetector.cpp new file mode 100644 index 0000000000..009d8f5809 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDexRegisteredDetector.cpp @@ -0,0 +1,46 @@ +/* PokemonLZA AutoStory - Dex registered detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStoryDexRegisteredDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +DexRegisteredDetector::DexRegisteredDetector(Color color) + : m_color(color) + , m_left_arc(852.0 / 1920, 140.0 / 1080, 20.0 / 1920, 80.0 / 1080) + , m_right_arc(1100.0 / 1920, 140.0 / 1080, 20.0 / 1920, 80.0 / 1080) + , m_tag(845.0 / 1920, 268.0 / 1080, 100.0 / 1920, 30.0 / 1080) +{} + +void DexRegisteredDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_left_arc); + items.add(m_color, m_right_arc); + items.add(m_color, m_tag); +} + +bool DexRegisteredDetector::detect(const ImageViewRGB32& screen){ + if (cyan_share(screen, m_left_arc) < 0.6 || cyan_share(screen, m_right_arc) < 0.4){ + return false; + } + // The tag: slate (about 65, 100, 135) with white text on it. + const double slate = share_of(screen, m_tag, [](Color pixel){ + const int r = pixel.red(); + return r >= 40 && r <= 100 && pixel.blue() >= 110 && pixel.blue() <= 175 + && pixel.blue() > r + 45 && pixel.green() > r + 20; + }); + return slate >= 0.4 && white_share(screen, m_tag) >= 0.08; +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDexRegisteredDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDexRegisteredDetector.h new file mode 100644 index 0000000000..466a498a40 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDexRegisteredDetector.h @@ -0,0 +1,46 @@ +/* PokemonLZA AutoStory - Dex registered detector + * + * From: https://github.com/PokemonAutomation/ + * + * The banner after catching a species the Pokedex did not have: a cyan + * ring at the top middle around the Pokemon, "Registered!" over it and + * its "No.###" tag under it. A repeat catch shows none. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryDexRegisteredDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryDexRegisteredDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The ring's left and right arcs cyan (x 852-872 and 1100-1120, y 140-220 +// on 1920x1080) and the slate "No.###" tag under it (x 845-945, y +// 268-298) with its white text. Only shapes and colors are read, so any +// language matches. +class DexRegisteredDetector : public StaticScreenDetector{ +public: + explicit DexRegisteredDetector(Color color = COLOR_CYAN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_left_arc; + ImageFloatBox m_right_arc; + ImageFloatBox m_tag; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox.cpp new file mode 100644 index 0000000000..fca1395daa --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox.cpp @@ -0,0 +1,110 @@ +/* PokemonLZA AutoStory - Dialog choice box + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTools/ImageStats.h" +#include "PokemonLZA_AutoStoryDialogChoiceBox.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +// Rows read from the bottom up before the reader stops looking. +constexpr size_t MAX_CHOICE_ROWS = 6; + +// The row pitch and the bottom row's strips at 1920 x 1080. Measured on +// the console on the tourist's two rows (Traditional Chinese) +// and the Research Lab elevator's three (Japanese): the panel x 1408 to +// 1728 ends at y 808 in both, its highlight x 1418 to 1715, the bottom +// row's from y 732 to 800, each row above 80 higher. The text starts at +// x 1443 on every row. +constexpr double ROW_PITCH = 80.0 / 1080; +constexpr double STRIP_TOP = 748.0 / 1080; +constexpr double STRIP_HEIGHT = 40.0 / 1080; + +// Right of the row's text, inside the highlight. +ImageFloatBox right_strip(size_t from_bottom){ + return ImageFloatBox(1650.0 / 1920, STRIP_TOP - ROW_PITCH * from_bottom, 55.0 / 1920, STRIP_HEIGHT); +} + +// Where every row's text starts. +ImageFloatBox text_strip(size_t from_bottom){ + return ImageFloatBox(1445.0 / 1920, STRIP_TOP - ROW_PITCH * from_bottom, 80.0 / 1920, STRIP_HEIGHT); +} + +enum class ChoiceSlot{ + HIGHLIGHTED, + PLAIN, + OTHER, +}; + +// A slot's right strip: the highlighted row 255 white, flat; a plain row +// the panel's dark blue, (35-45, 60-73, 108-120), spread 4-5. Its text +// strip: spread 220-240 on every row. Above the elevator's top row, the +// wall is as dark and blue, (23-24, 58-60, 102-104), but its text strip +// is flat (spread 5); above the tourist's, pavement, spread 34-36. +ChoiceSlot classify_slot(const ImageViewRGB32& frame, size_t from_bottom){ + const ImageStats right = image_stats(extract_box_reference(frame, right_strip(from_bottom))); + if (right.count == 0 || right.stddev.sum() > 30){ + return ChoiceSlot::OTHER; + } + const FloatPixel& average = right.average; + if (average.r >= 200 && average.g >= 200 && average.b >= 200){ + return ChoiceSlot::HIGHLIGHTED; + } + const bool dark_blue = + average.r <= 70 && average.g <= 100 && + average.b >= 80 && average.b <= 150 && + average.b >= average.r + 40; + if (!dark_blue){ + return ChoiceSlot::OTHER; + } + const ImageStats text = image_stats(extract_box_reference(frame, text_strip(from_bottom))); + if (text.count == 0 || text.stddev.sum() < 100){ + return ChoiceSlot::OTHER; + } + return ChoiceSlot::PLAIN; +} + +} // namespace + + +const ImageFloatBox& dialog_choice_box(){ + static const ImageFloatBox box{0.654308, 0.481553, 0.295529, 0.312621}; + return box; +} + +std::optional read_dialog_choice_rows(const ImageViewRGB32& frame){ + if (!frame || frame.width() < 192 || frame.height() < 108){ + return std::nullopt; + } + size_t count = 0; + std::optional highlighted_from_bottom; + for (; count < MAX_CHOICE_ROWS; count++){ + const ChoiceSlot slot = classify_slot(frame, count); + if (slot == ChoiceSlot::OTHER){ + break; + } + if (slot == ChoiceSlot::HIGHLIGHTED){ + if (highlighted_from_bottom){ + return std::nullopt; + } + highlighted_from_bottom = count; + } + } + if (!highlighted_from_bottom){ + return std::nullopt; + } + return DialogChoiceRows{.count = count, .selected = count - 1 - *highlighted_from_bottom}; +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox.h new file mode 100644 index 0000000000..ea46350646 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox.h @@ -0,0 +1,50 @@ +/* PokemonLZA AutoStory - Dialog choice box + * + * From: https://github.com/PokemonAutomation/ + * + * Where a dialogue's choices are drawn, for the selection arrow's + * detector (the screen events' choice box and choose_dialog_option() in + * Actions/PokemonLZA_AutoStoryDialogs.h), and a reader for the choice + * rows: how many there are and which one is highlighted. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryDialogChoiceBox_H +#define PokemonAutomation_PokemonLZA_AutoStoryDialogChoiceBox_H + +#include +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Where a dialogue's choices are drawn: the restaurant receptionist's +// (RestaurantFarmer). Other dialogues' choices are assumed to sit there. +const ImageFloatBox& dialog_choice_box(); + +// A dialogue's choice menu as read: how many rows it has, and the +// highlighted one, 0 the top. +struct DialogChoiceRows{ + size_t count = 0; + size_t selected = 0; +}; + +// Reads the choice menu on `frame`. The menu's panel is bottom-anchored: +// its last row sits at the same place whatever the row count, and each +// further row adds one row pitch on top. So the rows are counted from the +// bottom up, each either highlighted (white) or plain (dark blue, its +// text's start bright on it), until a slot is neither; the selected row is +// then counted from the top. None if no row, or not exactly one, is +// highlighted. +std::optional read_dialog_choice_rows(const ImageViewRGB32& frame); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox_Tests.cpp new file mode 100644 index 0000000000..3a611290b5 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox_Tests.cpp @@ -0,0 +1,232 @@ +/* PokemonLZA AutoStory - Dialog choice box Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "Tests/ImageUnitTest.h" +#include "VideoGames/PokemonLZA/PokemonLZA_PendingFixtures.h" +#include "PokemonLZA_AutoStoryDialogChoiceBox.h" +#include "PokemonLZA_AutoStoryDialogChoiceBox_Tests.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +constexpr uint32_t PAVEMENT = 0xffc8c3be; +constexpr uint32_t BLUE_WALL = 0xff18375c; +constexpr uint32_t PANEL = 0xff243c6c; +constexpr uint32_t WHITE = 0xffffffff; +constexpr uint32_t DARK_TEXT = 0xff1e1e28; + +void fill_box(size_t min_x, size_t min_y, size_t max_x, size_t max_y, uint32_t color, ImageRGB32& image){ + for (size_t y = min_y; y < max_y; y++){ + for (size_t x = min_x; x < max_x; x++){ + image.pixel(x, y) = color; + } + } +} + +// A choice menu of `rows` rows drawn as the console shows it (1920 x 1080) +// on `background`: the panel x 1408 to 1728, from y 808 up, one 80 pitch +// per row; row k from the bottom highlighted white at y 732 - 80k to +// 800 - 80k, x 1418 to 1715. Each row's text starts at x 1443, in strokes +// one column in three: dark on the highlighted rows (`highlighted`), +// white on the others. +ImageRGB32 choice_menu_frame(uint32_t background, size_t rows, std::initializer_list highlighted){ + ImageRGB32 image(1920, 1080); + image.fill(background); + fill_box(1408, 808 - 80 * rows - 4, 1728, 808, PANEL, image); + for (size_t row = 0; row < rows; row++){ + const size_t from_bottom = rows - 1 - row; + const size_t top = 732 - 80 * from_bottom; + bool lit = false; + for (const size_t pick : highlighted){ + lit = lit || pick == row; + } + if (lit){ + fill_box(1418, top, 1715, top + 68, WHITE, image); + } + for (size_t x = 1443; x < 1540; x += 3){ + fill_box(x, top + 16, x + 1, top + 52, lit ? DARK_TEXT : WHITE, image); + } + } + return image; +} + +ImageRGB32 flat_frame(uint32_t color){ + ImageRGB32 image(1920, 1080); + image.fill(color); + return image; +} + +std::string describe(const std::optional& rows){ + if (!rows){ + return "no choice menu"; + } + return std::format("row {} of {}", rows->selected, rows->count); +} + +bool same(const std::optional& a, const std::optional& b){ + if (!a || !b){ + return a.has_value() == b.has_value(); + } + return a->count == b->count && a->selected == b->selected; +} + + +struct SyntheticCase{ + std::string name; + struct Input{ + std::function frame; + }; + Input input; + struct Expectation{ + std::optional rows; // None: no choice menu read. + }; + Expectation expectation; +}; + +const SyntheticCase SYNTHETIC_CASES[] = { + { + .name = "two rows, the top highlighted", + .input = {.frame = []{ return choice_menu_frame(PAVEMENT, 2, {0}); }}, + .expectation = {.rows = DialogChoiceRows{.count = 2, .selected = 0}}, + }, + { + .name = "two rows, the bottom highlighted", + .input = {.frame = []{ return choice_menu_frame(PAVEMENT, 2, {1}); }}, + .expectation = {.rows = DialogChoiceRows{.count = 2, .selected = 1}}, + }, + { + .name = "three rows on a blue wall, the top highlighted", + .input = {.frame = []{ return choice_menu_frame(BLUE_WALL, 3, {0}); }}, + .expectation = {.rows = DialogChoiceRows{.count = 3, .selected = 0}}, + }, + { + .name = "three rows on a blue wall, the middle highlighted", + .input = {.frame = []{ return choice_menu_frame(BLUE_WALL, 3, {1}); }}, + .expectation = {.rows = DialogChoiceRows{.count = 3, .selected = 1}}, + }, + { + .name = "four rows, the bottom highlighted", + .input = {.frame = []{ return choice_menu_frame(PAVEMENT, 4, {3}); }}, + .expectation = {.rows = DialogChoiceRows{.count = 4, .selected = 3}}, + }, + { + .name = "three rows, none highlighted", + .input = {.frame = []{ return choice_menu_frame(PAVEMENT, 3, {}); }}, + .expectation = {.rows = std::nullopt}, + }, + { + .name = "three rows, two highlighted", + .input = {.frame = []{ return choice_menu_frame(PAVEMENT, 3, {0, 2}); }}, + .expectation = {.rows = std::nullopt}, + }, + {.name = "flat blue wall", .input = {.frame = []{ return flat_frame(BLUE_WALL); }}, .expectation = {.rows = std::nullopt}}, + {.name = "flat panel blue", .input = {.frame = []{ return flat_frame(PANEL); }}, .expectation = {.rows = std::nullopt}}, + {.name = "flat white", .input = {.frame = []{ return flat_frame(WHITE); }}, .expectation = {.rows = std::nullopt}}, + {.name = "flat black", .input = {.frame = []{ return flat_frame(0xff000000); }}, .expectation = {.rows = std::nullopt}}, + {.name = "an empty frame", .input = {.frame = []{ return ImageRGB32(); }}, .expectation = {.rows = std::nullopt}}, + {.name = "a 1 x 1 frame", .input = {.frame = []{ return ImageRGB32(1, 1); }}, .expectation = {.rows = std::nullopt}}, +}; + +class Test_DialogChoiceRowsSynthetic : public UnitTest{ +public: + explicit Test_DialogChoiceRowsSynthetic(const SyntheticCase& test) + : UnitTest(std::format("PokemonLZA::AutoStory - Dialog choice rows: {}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::optional rows = read_dialog_choice_rows(m_test.input.frame()); + if (!same(rows, m_test.expectation.rows)){ + return std::format("Expected {}, read {}.", describe(m_test.expectation.rows), describe(rows)); + } + return true; + } + +private: + const SyntheticCase& m_test; +}; + + +// the console's choice menus (1920 x 1080), each with the row highlighted as +// it opens. +struct FrameCase{ + std::string name; + struct Input{ + std::string_view file; // Relative to the test images. + }; + Input input; + struct Expectation{ + DialogChoiceRows rows; + }; + Expectation expectation; +}; + +const FrameCase FRAME_CASES[] = { + { + .name = "the tourist's yes or yes (2 rows, Traditional Chinese)", + .input = {.file = "PokemonLZA/AutoStoryDetectors/ArrivalToAd/0065-text_box+choice_box.jpg"}, + .expectation = {.rows = {.count = 2, .selected = 0}}, + }, + { + .name = "the Research Lab elevator's floors (3 rows, Japanese)", + .input = {.file = "PokemonLZA/AutoStoryDialogChoices/elevator_3rows_row0.png"}, + .expectation = {.rows = {.count = 3, .selected = 0}}, + }, +}; + +class Test_DialogChoiceRowsFrame : public ImageUnitTest{ +public: + explicit Test_DialogChoiceRowsFrame(const FrameCase& test) + : ImageUnitTest( + std::format("PokemonLZA::AutoStory - Dialog choice rows: {}", test.name), + {std::string(test.input.file)} + ) + , m_test(test) + {} + virtual UnitTestResult run_on_images( + Logger&, CancellableScope&, const std::vector& images + ) const override{ + const std::optional rows = read_dialog_choice_rows(images[0]); + if (!same(rows, m_test.expectation.rows)){ + return std::format("{}: expected {}, read {}.", image_paths()[0], describe(m_test.expectation.rows), describe(rows)); + } + return true; + } + +private: + const FrameCase& m_test; +}; + +} // namespace + + +void add_tests_AutoStoryDialogChoiceBox(UnitTestDatabase& database){ + for (const SyntheticCase& test : SYNTHETIC_CASES){ + database.add(test); + } + for (const FrameCase& test : FRAME_CASES){ + add_pending_fixture_test(database, test); + } +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox_Tests.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox_Tests.h new file mode 100644 index 0000000000..b1fe6b5f5f --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox_Tests.h @@ -0,0 +1,26 @@ +/* PokemonLZA AutoStory - Dialog choice box Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryDialogChoiceBox_Tests_H +#define PokemonAutomation_PokemonLZA_AutoStoryDialogChoiceBox_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Registers read_dialog_choice_rows()'s tests: synthetic menus of two and +// three rows always; the console's choice menus from the test images. +void add_tests_AutoStoryDialogChoiceBox(UnitTestDatabase& database); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector.cpp new file mode 100644 index 0000000000..f68448b7b6 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector.cpp @@ -0,0 +1,42 @@ +/* PokemonLZA AutoStory - Dialog Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "Common/Cpp/Color.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryDialogDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +StoryDialogDetector::StoryDialogDetector(StoryDialogKinds kinds) + : m_kinds(kinds) + , m_white(COLOR_RED) + , m_blue(COLOR_BLUE) + , m_teal(COLOR_GREEN) + , m_item_receive(COLOR_CYAN) +{} + +void StoryDialogDetector::make_overlays(VideoOverlaySet& items) const{ + m_white.make_overlays(items); + m_blue.make_overlays(items); + m_teal.make_overlays(items); + if (m_kinds.item_receive){ + m_item_receive.make_overlays(items); + } +} + +bool StoryDialogDetector::detect(const ImageViewRGB32& screen){ + const bool white = m_kinds.white == WhiteDialogPart::BOX_ONLY ? m_white.detect_box(screen) : m_white.detect(screen); + return white || m_blue.detect(screen) || m_teal.detect(screen) || + (m_kinds.item_receive && m_item_receive.detect(screen)); +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector.h new file mode 100644 index 0000000000..90c9a83ace --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector.h @@ -0,0 +1,61 @@ +/* PokemonLZA AutoStory - Dialog Detector + * + * From: https://github.com/PokemonAutomation/ + * + * Any of the story's dialogue boxes, from the upstream dialog detectors. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryDialogDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryDialogDetector_H + +#include "Common/Cpp/Time.h" +#include "CommonTools/VisualDetector.h" +#include "VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// How much of a white dialogue box StoryDialogDetector needs. +enum class WhiteDialogPart{ + BOX_AND_ARROW, // FlatWhiteDialogDetector: the box with its advance arrow + BOX_ONLY, // its bottom edge only: the box while the text still types +}; + +// Which dialogue boxes StoryDialogDetector counts, besides blue and teal. +struct StoryDialogKinds{ + WhiteDialogPart white = WhiteDialogPart::BOX_AND_ARROW; + bool item_receive = false; // ItemReceiveDetector's item card +}; + +// Any dialogue box the story shows: white, blue or teal (and the item card +// when `kinds` asks for it). +class StoryDialogDetector : public StaticScreenDetector{ +public: + explicit StoryDialogDetector(StoryDialogKinds kinds = {}); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + StoryDialogKinds m_kinds; + FlatWhiteDialogDetector m_white; + BlueDialogDetector m_blue; + TealDialogDetector m_teal; + ItemReceiveDetector m_item_receive; +}; +// StoryDialogDetector on the video, held `hold_duration`. +class StoryDialogWatcher : public DetectorToFinder{ +public: + explicit StoryDialogWatcher(StoryDialogKinds kinds = {}, Milliseconds hold_duration = Milliseconds(250)) + : DetectorToFinder("StoryDialogWatcher", hold_duration, kinds) + {} +}; + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryObjectiveMarkerReader.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryObjectiveMarkerReader.cpp new file mode 100644 index 0000000000..4f2cf5e28f --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryObjectiveMarkerReader.cpp @@ -0,0 +1,70 @@ +/* PokemonLZA AutoStory - Objective marker reader + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/Images/BinaryImage_FilterRgb32.h" +#include "CommonTools/OCR/OCR_NumberReader.h" +#include "Kernels/Waterfill/Kernels_Waterfill_Session.h" +#include "PokemonLZA_AutoStoryObjectiveMarkerReader.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + +using namespace Kernels::Waterfill; + + +std::optional read_objective_marker(Logger& logger, const ImageViewRGB32& screen){ + if (!screen){ + return std::nullopt; + } + // The hexagon's orange, about (245, 175, 40) on the console: the largest + // hexagon-sized blob outside the minimap (its marker is there too). + const double scale = screen.height() / 1080.0; + PackedBinaryMatrix matrix = compress_rgb32_to_binary_range(screen, 220, 255, 140, 215, 0, 90); + std::unique_ptr session = make_WaterfillSession(matrix); + std::unique_ptr iter = session->make_iterator((size_t)(500 * scale * scale)); + std::optional best; + WaterfillObject object; + while (iter->find_next(object, false)){ + // The minimap: a circle of radius 125 around (180, 180) on + // 1920x1080; 140 leaves margin for its rim. A box around it also + // covered the real marker when it sat at the left edge, high up. + const double dx = object.center_of_gravity_x() / scale - 180; + const double dy = object.center_of_gravity_y() / scale - 180; + const bool in_minimap = dx * dx + dy * dy < 140 * 140; + const bool in_party_bar = object.center_of_gravity_y() / screen.height() > 0.70; + const double aspect = object.aspect_ratio(); + if (in_minimap || in_party_bar || object.area > 4000 * scale * scale || aspect < 0.6 || aspect > 1.4){ + continue; + } + if (!best || object.area > best->area){ + best = object; + } + } + if (!best){ + return std::nullopt; + } + // The distance: white digits just above the hexagon, up to its width + // out each side, about half its height tall. + const double w = (double)best->width(); + const double h = (double)best->height(); + const ImagePixelBox digits( + (size_t)std::max(0.0, best->min_x - w), (size_t)std::max(0.0, best->min_y - 0.95 * h), + (size_t)std::min((double)screen.width(), best->max_x + w), (size_t)std::max(1.0, best->min_y - 0.10 * h) + ); + ObjectiveMarker marker; + marker.x = (best->min_x + best->max_x) / 2.0 / screen.width(); + marker.distance = OCR::read_number_waterfill(logger, extract_box_reference(screen, digits), 0xffd0d0d0, 0xffffffff); + return marker; +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryObjectiveMarkerReader.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryObjectiveMarkerReader.h new file mode 100644 index 0000000000..5a216ef769 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryObjectiveMarkerReader.h @@ -0,0 +1,36 @@ +/* PokemonLZA AutoStory - Objective marker reader + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryObjectiveMarkerReader_H +#define PokemonAutomation_PokemonLZA_AutoStoryObjectiveMarkerReader_H + +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Logging/Logger.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The main mission's objective marker in the overworld: an orange hexagon +// "!" with the distance above it in white digits. +struct ObjectiveMarker{ + // The hexagon's middle, 0 (left edge) to 1 (right edge). + double x = 0; + // The digits read; -1 when they did not read. + int distance = -1; +}; + +// The marker on `screen`, if it shows (the minimap's own marker aside). +std::optional read_objective_marker(Logger& logger, const ImageViewRGB32& screen); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryObjectiveMarkerReader_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryObjectiveMarkerReader_Tests.cpp new file mode 100644 index 0000000000..f799cae553 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryObjectiveMarkerReader_Tests.cpp @@ -0,0 +1,158 @@ +/* PokemonLZA AutoStory - Objective marker reader Tests + * + * From: https://github.com/PokemonAutomation/ + * + * Synthetic 1920x1080 frames: the marker's orange drawn as squares and + * bars where the reader must take or skip them. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "PokemonLZA_AutoStoryObjectiveMarkerReader.h" +#include "PokemonLZA_AutoStoryObjectiveMarkerReader_Tests.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +constexpr uint32_t BACKGROUND = 0xff304030; +// The hexagon's orange on the console. +constexpr uint32_t ORANGE = 0xfff5af28; + +// A filled rectangle of the marker's orange, in pixels of a 1920x1080 +// frame. +struct Patch{ + size_t x; + size_t y; + size_t width; + size_t height; +}; + +ImageRGB32 frame_with(const std::vector& patches){ + ImageRGB32 image(1920, 1080); + image.fill(BACKGROUND); + for (const Patch& patch : patches){ + for (size_t y = patch.y; y < patch.y + patch.height; y++){ + for (size_t x = patch.x; x < patch.x + patch.width; x++){ + image.pixel(x, y) = ORANGE; + } + } + } + return image; +} + +std::string describe_x(std::optional x){ + return x ? std::format("a marker at x = {:.3f}", *x) : std::string("no marker"); +} + + +struct SyntheticCase{ + std::string name; + struct Input{ + std::function frame; + }; + Input input; + struct Expectation{ + // The marker's middle as a share of the width; none: no marker. + std::optional x; + }; + Expectation expectation; +}; + +const SyntheticCase SYNTHETIC_CASES[] = { + { + .name = "a marker in the middle", + .input = {.frame = []{ return frame_with({{930, 500, 60, 60}}); }}, + .expectation = {.x = 0.5}, + }, + { + .name = "a marker right of the middle", + .input = {.frame = []{ return frame_with({{1506, 300, 60, 60}}); }}, + .expectation = {.x = 0.8}, + }, + { + .name = "the larger of two markers", + .input = {.frame = []{ return frame_with({{400, 400, 40, 40}, {1506, 400, 60, 60}}); }}, + .expectation = {.x = 0.8}, + }, + { + .name = "a marker beside the minimap's own", + .input = {.frame = []{ return frame_with({{150, 150, 60, 60}, {930, 500, 40, 40}}); }}, + .expectation = {.x = 0.5}, + }, + { + .name = "the minimap's marker only", + .input = {.frame = []{ return frame_with({{150, 150, 60, 60}}); }}, + .expectation = {.x = std::nullopt}, + }, + { + .name = "orange in the party bar only", + .input = {.frame = []{ return frame_with({{930, 800, 60, 60}}); }}, + .expectation = {.x = std::nullopt}, + }, + { + .name = "an orange blob too large", + .input = {.frame = []{ return frame_with({{900, 400, 100, 100}}); }}, + .expectation = {.x = std::nullopt}, + }, + { + .name = "an orange bar too wide", + .input = {.frame = []{ return frame_with({{900, 400, 90, 30}}); }}, + .expectation = {.x = std::nullopt}, + }, + { + .name = "an orange speck too small", + .input = {.frame = []{ return frame_with({{930, 500, 15, 15}}); }}, + .expectation = {.x = std::nullopt}, + }, + {.name = "no orange", .input = {.frame = []{ return frame_with({}); }}, .expectation = {.x = std::nullopt}}, + {.name = "an empty frame", .input = {.frame = []{ return ImageRGB32(); }}, .expectation = {.x = std::nullopt}}, + {.name = "a 1 x 1 frame", .input = {.frame = []{ return ImageRGB32(1, 1); }}, .expectation = {.x = std::nullopt}}, +}; + +class Test_ObjectiveMarkerSynthetic : public UnitTest{ +public: + explicit Test_ObjectiveMarkerSynthetic(const SyntheticCase& test) + : UnitTest(std::format("PokemonLZA::AutoStory - Objective marker reader: {}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ + const std::optional marker = read_objective_marker(logger, m_test.input.frame()); + const std::optional x = marker ? std::optional(marker->x) : std::nullopt; + const bool same = x.has_value() == m_test.expectation.x.has_value() + && (!x || std::abs(*x - *m_test.expectation.x) < 0.01); + if (!same){ + return std::format("Expected {}, read {}.", describe_x(m_test.expectation.x), describe_x(x)); + } + return true; + } + +private: + const SyntheticCase& m_test; +}; + +} // namespace + + +void add_tests_AutoStoryObjectiveMarkerReader(UnitTestDatabase& database){ + for (const SyntheticCase& test : SYNTHETIC_CASES){ + database.add(test); + } +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryObjectiveMarkerReader_Tests.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryObjectiveMarkerReader_Tests.h new file mode 100644 index 0000000000..b48fd0f702 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryObjectiveMarkerReader_Tests.h @@ -0,0 +1,25 @@ +/* PokemonLZA AutoStory - Objective marker reader Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryObjectiveMarkerReader_Tests_H +#define PokemonAutomation_PokemonLZA_AutoStoryObjectiveMarkerReader_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Registers the objective marker reader's tests, on synthetic frames. +void add_tests_AutoStoryObjectiveMarkerReader(UnitTestDatabase& database); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.cpp new file mode 100644 index 0000000000..f925f008df --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.cpp @@ -0,0 +1,39 @@ +/* PokemonLZA AutoStory - Tutorial card detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "PokemonLZA_AutoStoryTutorialCardDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// A white panel in the middle over a dark, blurred, even screen. On 5 +// cards and 9 other screens (map, Pokedex, dialogs, overworld, night) at +// full resolution, the cards' panel is 241-246 bright (its text spreads it +// to 40-48) and their edges 61-76 with spread under 7; the others' panel +// is at most 185, or their edges brighter or busy. +bool tutorial_card_shows(const ImageViewRGB32& frame){ + const ImageStats panel = image_stats(extract_box_reference(frame, ImageFloatBox(0.32, 0.62, 0.36, 0.20))); + if (panel.average.sum() / 3 < 225){ + return false; + } + for (const ImageFloatBox& side : {ImageFloatBox(0.03, 0.30, 0.15, 0.40), ImageFloatBox(0.82, 0.30, 0.15, 0.40)}){ + const ImageStats edge = image_stats(extract_box_reference(frame, side)); + if (edge.average.sum() / 3 > 100 || edge.stddev.sum() / 3 > 12){ + return false; + } + } + return true; +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.h new file mode 100644 index 0000000000..fcea31caf5 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.h @@ -0,0 +1,30 @@ +/* PokemonLZA AutoStory - Tutorial card detector + * + * From: https://github.com/PokemonAutomation/ + * + * The tutorial cards the story opens (Map & Missions, Main Missions, + * Pokedex, ...). Only A turns their pages (B turns back), so a scene mash + * checks for one before it presses its text button. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryTutorialCardDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryTutorialCardDetector_H + +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// A tutorial card on screen: a bright panel in the middle over dark, even +// edges. +bool tutorial_card_shows(const ImageViewRGB32& frame); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index c4fc6a09cc..fe36c18a4e 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -2373,22 +2373,40 @@ file(GLOB LIBRARY_SOURCES Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.h Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryCaughtMarkReader.cpp Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryCaughtMarkReader.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader_Tests.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader_Tests.h Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.cpp Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector.h Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDexRegisteredDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDexRegisteredDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox_Tests.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox_Tests.h Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.cpp Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.h Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMinimapZoneDetector.cpp Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMinimapZoneDetector.h Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.cpp Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryObjectiveMarkerReader.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryObjectiveMarkerReader.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryObjectiveMarkerReader_Tests.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryObjectiveMarkerReader_Tests.h Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.cpp Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.h Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.cpp Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.h Source/VideoGames/PokemonLZA/Programs/Farming/PokemonLZA_DonutMaker.cpp Source/VideoGames/PokemonLZA/Programs/Farming/PokemonLZA_DonutMaker.h Source/VideoGames/PokemonLZA/Programs/Farming/PokemonLZA_FriendshipFarmer.cpp