diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index d7fc0c3e76..f23153a8f7 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -226,6 +226,11 @@ include(cmake/CoreLib.cmake) # Create the GUI library add_library(SerialProgramsLib STATIC ${CORE_LIBRARY_SOURCES} ${LIBRARY_SOURCES}) +# Data files built into the program (see cmake/EmbedTextFiles.cmake). +include(cmake/EmbedTextFiles.cmake) +pa_embed_text_files(SerialProgramsLib Source/VideoGames/PokemonLZA/Programs/AutoStory/ReferenceData + VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h PokemonAutomation::NintendoSwitch::PokemonLZA autostory_reference_json) + # Create binary target if (APPLE) diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp index 9b71758d9f..757272eea3 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp @@ -21,6 +21,7 @@ #include "Inference/PokemonLZA_OverworldPartySelectionDetector.h" #include "Inference/PokemonLZA_PartyBarDetector.h" #include "Inference/PokemonLZA_SelectionArrowDetector.h" +#include "Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h" namespace PokemonAutomation{ namespace NintendoSwitch{ @@ -45,6 +46,7 @@ void add_tests(UnitTestDatabase& database){ add_tests_OverworldPartySelectionDetector(database); add_tests_PartyBarDetector(database); add_tests_SelectionArrowDetector(database); + add_tests_AutoStoryDetectors(database); } diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp new file mode 100644 index 0000000000..bc26a56772 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp @@ -0,0 +1,134 @@ +/* PokemonLZA AutoStory - Advance arrow detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "CommonFramework/ImageTypes/BinaryImage.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Images/BinaryImage_FilterRgb32.h" +#include "Kernels/Waterfill/Kernels_Waterfill_Session.h" +#include "PokemonLZA_AutoStoryAdvanceArrowDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + +using namespace Kernels::Waterfill; + + +namespace{ + +// How well `blob` (a Waterfill object kept with its pixels) fills a V +// across its bounds: its overlap with the V over their union. Each arm +// runs from 10% in at the top to the middle at the bottom, 22% of the +// width across. +double v_match(const WaterfillObject& blob){ + const double width = double(blob.width()); + const double height = double(blob.height()); + size_t both = 0; + size_t either = 0; + for (size_t y = blob.min_y; y < blob.max_y; y++){ + const double v = (double(y - blob.min_y) + 0.5) / height; + for (size_t x = blob.min_x; x < blob.max_x; x++){ + const double u = (double(x - blob.min_x) + 0.5) / width; + const bool in_v = std::abs(std::abs(u - 0.5) - (0.40 * (1 - v) + 0.03)) <= 0.11; + const bool in_blob = blob.object->get(x, y); + if (in_v && in_blob){ + both++; + } + if (in_v || in_blob){ + either++; + } + } + } + return either == 0 ? 0 : double(both) / double(either); +} + +// The pixels of `mask` outside `blob` within `margin` of its bounds, per +// pixel of the blob. The blob lies inside the window, so they are the +// window's pixels less its own. +double stray_share(const PackedBinaryMatrix& mask, const WaterfillObject& blob, size_t margin){ + const size_t min_x = blob.min_x > margin ? blob.min_x - margin : 0; + const size_t min_y = blob.min_y > margin ? blob.min_y - margin : 0; + const size_t max_x = std::min(blob.max_x + margin, mask.width()); + const size_t max_y = std::min(blob.max_y + margin, mask.height()); + size_t in_window = 0; + for (size_t y = min_y; y < max_y; y++){ + for (size_t x = min_x; x < max_x; x++){ + if (mask.get(x, y)){ + in_window++; + } + } + } + return double(in_window - blob.area) / double(blob.area); +} + +// Whether `blob` of `mask` is the chevron, on a screen `scale` times 1080p: +// 32x19 measured, 30-33 x 18-20 on every frame with it. +bool chevron_blob(const PackedBinaryMatrix& mask, const WaterfillObject& blob, double scale){ + const double width = double(blob.width()) / scale; + const double height = double(blob.height()) / scale; + if (width < 26 || width > 40 || height < 15 || height > 24){ + return false; + } + const double aspect = width / height; + if (aspect < 1.4 || aspect > 2.1){ + return false; + } + return stray_share(mask, blob, size_t(4 * scale + 0.5)) <= 0.09 && v_match(blob) >= 0.45; +} + +// Whether a 4-connected blob of the pixels of `area` in the colour range +// `mins`-`maxs` is the chevron. +bool chevron_in(const ImageViewRGB32& area, double scale, uint32_t mins, uint32_t maxs){ + const PackedBinaryMatrix mask = compress_rgb32_to_binary_range(area, mins, maxs); + PackedBinaryMatrix unfilled = mask.copy(); + const std::unique_ptr session = make_WaterfillSession(unfilled); + const std::unique_ptr blobs = session->make_iterator(1); + WaterfillObject blob; + while (blobs->find_next(blob, true)){ + if (chevron_blob(mask, blob, scale)){ + return true; + } + } + return false; +} + +} // namespace + + +AdvanceArrowDetector::AdvanceArrowDetector(Color color) + : m_color(color) + , m_area(1370.0 / 1920, 930.0 / 1080, 110.0 / 1920, 120.0 / 1080) +{} + +void AdvanceArrowDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_area); +} + +// On 1920x1080 frames saved as JPEG (quality 80) the chevron (on the ad's +// subtitles, on white boxes, on the opening's blue box) fills the V +// 0.51-0.71 with at most 6% stray pixels. The closest other shape in size +// and shape, a dark blade of scenery, fills the V 0.48 but has 13% stray +// pixels, so stray pixels are allowed up to 9%; shapes with nothing +// around them fill the V 0.43 or less and are out of the chevron's size. +// JPEG keeps colour at half resolution, so a white chevron over orange +// (the player's shorts) loses blue below 235 at its edges: white asks +// for 220 in blue (any floor from 190 to 230 reads every frame right). +bool AdvanceArrowDetector::detect(const ImageViewRGB32& screen){ + const double scale = double(screen.height()) / 1080; + const ImageViewRGB32 area = extract_box_reference(screen, m_area); + // White: red and green 235 or more, blue 220 or more; dark: every + // channel 110 or under. + return chevron_in(area, scale, 0x00ebebdc, 0xffffffff) || chevron_in(area, scale, 0x00000000, 0xff6e6e6e); +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h new file mode 100644 index 0000000000..fb27998a3b --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h @@ -0,0 +1,46 @@ +/* PokemonLZA AutoStory - Advance arrow detector + * + * From: https://github.com/PokemonAutomation/ + * + * The "press A to go on" chevron at the bottom right of a dialogue box or + * of the boxless subtitles of the ad early in the story. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryAdvanceArrowDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryAdvanceArrowDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The "press A to go on" chevron (a V about 32x19 on 1920x1080) at the +// right end of a dialogue box and of boxless subtitles: dark on a white +// box, white on a blue box and on the ad's subtitles. Found by shape in +// x 1370-1480, y 930-1050: a blob of white (red and green 235+, blue 220+) or dark +// (every channel 110 or under) pixels the size of the chevron, filling a +// V, with no other pixel of its colour around it. Text is not read, so +// any language matches. +class AdvanceArrowDetector : public StaticScreenDetector{ +public: + explicit AdvanceArrowDetector(Color color = COLOR_GREEN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_area; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.cpp new file mode 100644 index 0000000000..354417e7af --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.cpp @@ -0,0 +1,86 @@ +/* PokemonLZA AutoStory - Autosave detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStoryAutosaveDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +// The icon (white, with a cyan glow, dim or bright), and the strips around +// it (left, above, below) clear of white and bright cyan, `dx` pixels (of +// 1920) right of where the Chinese badge has them. +bool autosave_icon_at( + const ImageViewRGB32& screen, const ImageFloatBox& icon, + std::initializer_list around, double dx +){ + const ImageFloatBox icon_here = shifted(icon, dx); + const double white = white_share(screen, icon_here); + if (white < 0.08 || white > 0.40 || dim_cyan_share(screen, icon_here) < 0.25){ + return false; + } + for (const ImageFloatBox& box : around){ + const ImageFloatBox here = shifted(box, dx); + if (white_share(screen, here) >= 0.05 || cyan_share(screen, here) >= 0.05){ + return false; + } + } + return true; +} + +} // namespace + + +AutosaveDetector::AutosaveDetector(Color color) + : m_color(color) + , m_icon(1603.0 / 1920, 73.0 / 1080, 27.0 / 1920, 31.0 / 1080) + , m_left(1570.0 / 1920, 73.0 / 1080, 28.0 / 1920, 31.0 / 1080) + , m_above(1603.0 / 1920, 56.0 / 1080, 27.0 / 1920, 14.0 / 1080) + , m_below(1603.0 / 1920, 108.0 / 1080, 27.0 / 1920, 14.0 / 1080) +{} + +void AutosaveDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_icon); + items.add(m_color, m_left); + items.add(m_color, m_above); + items.add(m_color, m_below); +} + +// On the badge the icon is 19% white and 48% cyan, with nothing bright +// left of it. A bright cutscene flash passes the icon's bounds too, but is +// 70%+ white and 39%+ cyan left of the icon as well. +// Only the Chinese badge is checked; with a shorter text (English +// "Saving...") a right-aligned badge moves the icon right, so the icon is +// looked for from where it is in Chinese to 240 px right of it. +// Glowing scenery (a lit blade, a bright sky) can match the icon somewhere +// in that span, but glows on past it: the strips 14 px above and below the +// icon are 22%+ white or cyan there and empty around the badge; they must +// stay under 5%. +// The icon's glow is dim over a dark or lamp-lit scene (about 28, 70, 85 +// on 02.04's check-in wall, frame 0075 of a +// run in Japanese, icon at dx 36), so the icon's glow is dim_cyan_share(): +// of the 3569 PLZA captures on the console it finds 11 more +// badges, every one a real badge, and no new false one. +bool AutosaveDetector::detect(const ImageViewRGB32& screen){ + for (double dx = 0; dx <= 240; dx += 4){ + if (autosave_icon_at(screen, m_icon, {m_left, m_above, m_below}, dx)){ + return true; + } + } + return false; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.h new file mode 100644 index 0000000000..555bb4c1bc --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.h @@ -0,0 +1,46 @@ +/* PokemonLZA AutoStory - Autosave detector + * + * From: https://github.com/PokemonAutomation/ + * + * The autosave badge, a pencil and "Saving...", which shows for 1-3 s + * after the game autosaves. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryAutosaveDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryAutosaveDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The autosave badge at the top right: a white pencil with a cyan glow at +// x 1603-1630, y 73-104 on 1920x1080 (the glow dim over a dark scene), +// and nothing white or cyan just left of it, above it or below it. Only +// the icon is read, so any language matches. +class AutosaveDetector : public StaticScreenDetector{ +public: + explicit AutosaveDetector(Color color = COLOR_YELLOW); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_icon; + ImageFloatBox m_left; + ImageFloatBox m_above; + ImageFloatBox m_below; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryCaughtMarkReader.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryCaughtMarkReader.cpp new file mode 100644 index 0000000000..0452b598a1 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryCaughtMarkReader.cpp @@ -0,0 +1,110 @@ +/* PokemonLZA AutoStory - Caught mark reader + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Images/BinaryImage_FilterRgb32.h" +#include "Kernels/Waterfill/Kernels_Waterfill_Session.h" +#include "PokemonLZA_AutoStoryCaughtMarkReader.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + +using namespace Kernels::Waterfill; + + +namespace{ + +// The name plate's HP bar: the widest flat green bar in the upper two +// thirds of the screen, 80 px wide or more at 1080p. The green is kept +// tight (red 0-90, green 175+, blue 90-145; the bar is about 45, 198, +// 115): a wider range took in the bright grass around Bunnelby's plate +// (about 68, 160, 80), so the bar was no longer flat and the lock-on was +// missed (03.08). The player's own HP bar, over the +// head while a wild Pokemon attacks, is about 63 px wide; the plate's +// some 105-115 (Bunnelby's and Scatterbug's, the same frames). +std::optional find_plate_bar(const ImageViewRGB32& screen){ + const double scale = screen.height() / 1080.0; + const ImageViewRGB32 upper = extract_box_reference(screen, ImageFloatBox(0, 0, 1, 0.68)); + PackedBinaryMatrix matrix = compress_rgb32_to_binary_range(upper, 0, 90, 175, 255, 90, 145); + std::unique_ptr session = make_WaterfillSession(matrix); + std::unique_ptr iter = session->make_iterator((size_t)(150 * scale * scale)); + std::optional best; + WaterfillObject object; + while (iter->find_next(object, false)){ + const size_t w = object.width() + 1; + const size_t h = object.height() + 1; + const bool flat = w >= 80 * scale && h <= 24 * scale && w >= 4 * h; + if (!flat || object.area * 2 < w * h){ + continue; + } + if (!best || w > best->width()){ + best = ImagePixelBox(object.min_x, object.min_y, object.max_x + 1, object.max_y + 1); + } + } + return best; +} + +} // namespace + + +std::optional lock_target_plate_bar(const ImageViewRGB32& screen){ + const std::optional bar = find_plate_bar(screen); + if (!bar || screen.width() == 0 || screen.height() == 0){ + return std::nullopt; + } + return ImageFloatBox( + (double)bar->min_x / screen.width(), (double)bar->min_y / screen.height(), + (double)bar->width() / screen.width(), (double)bar->height() / screen.height() + ); +} + +LockTargetPlateDetector::LockTargetPlateDetector(Color color) + : m_color(color) +{} + +void LockTargetPlateDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, ImageFloatBox(0, 0, 1, 0.68)); +} + +bool LockTargetPlateDetector::detect(const ImageViewRGB32& screen){ + return lock_target_plate_bar(screen).has_value(); +} + +std::optional lock_target_caught(const ImageViewRGB32& screen){ + const std::optional bar = find_plate_bar(screen); + if (!bar){ + return std::nullopt; + } + const double w = (double)bar->width(); + const size_t min_x = (size_t)(bar->min_x + 0.6 * w); + const size_t max_x = std::min(screen.width(), (size_t)(bar->min_x + 1.5 * w)); + const size_t min_y = (size_t)std::max(0.0, bar->min_y - 0.85 * w); + const size_t max_y = (size_t)std::max(0.0, bar->min_y - 0.45 * w); + size_t red = 0; + size_t white = 0; + for (size_t y = min_y; y < max_y; y++){ + for (size_t x = min_x; x < max_x; x++){ + const Color pixel(screen.pixel(x, y)); + const int r = pixel.red(); + // The ball's red is pinkish (blue about green); the gender + // mark next to it, on every plate, is orange (green well over + // blue) and is not counted (Scatterbug). + red += r > 190 && r > pixel.green() + 70 && r > pixel.blue() + 50 && pixel.blue() + 15 >= pixel.green(); + white += r > 215 && pixel.green() > 215 && pixel.blue() > 215; + } + } + // 37 red and 42 white at 1280x720 on the caught Mareep. + const double area_scale = (screen.height() / 720.0) * (screen.height() / 720.0); + return red >= 15 * area_scale && white >= 15 * area_scale; +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryCaughtMarkReader.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryCaughtMarkReader.h new file mode 100644 index 0000000000..adda466272 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryCaughtMarkReader.h @@ -0,0 +1,59 @@ +/* PokemonLZA AutoStory - Caught mark reader + * + * From: https://github.com/PokemonAutomation/ + * + * Whether the wild Pokemon locked on to is of a species already caught: + * its name plate then shows a Poke Ball after the level and gender. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryCaughtMarkReader_H +#define PokemonAutomation_PokemonLZA_AutoStoryCaughtMarkReader_H + +#include +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// While ZL locks on to a wild Pokemon: true when its name plate shows the +// caught mark, false when the plate shows none, nothing when no plate was +// found. The plate is found by its HP bar, a green bar (about 45, 198, 115) +// at least 80 px wide on 1920x1080, in the upper two thirds of the +// screen; the mark, pinkish red over white, sits above the bar's right end +// (0.6-1.5 bar widths from its left, 0.45-0.85 above it), right of the +// orange gender mark. Tuned on Mareep, caught and not, and +// Bunnelby and Scatterbug, on the console. The plate shows +// whether or not the partner is out; the move symbols only when it is. +std::optional lock_target_caught(const ImageViewRGB32& screen); + +// While ZL locks on to a wild Pokemon: its name plate's HP bar (as +// above) as shares of the screen, nothing when no plate was found. The +// plate stands above the target, so a lower bar is a nearer target. +std::optional lock_target_plate_bar(const ImageViewRGB32& screen); + +// Whether ZL holds a lock on a wild Pokemon: its name plate shows +// (lock_target_plate_bar()). Only the bar's shape and color are read, so +// any language matches. +class LockTargetPlateDetector : public StaticScreenDetector{ +public: + explicit LockTargetPlateDetector(Color color = COLOR_GREEN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.cpp new file mode 100644 index 0000000000..5d5ad4b33d --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.cpp @@ -0,0 +1,56 @@ +/* PokemonLZA AutoStory - Controls hint detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStoryControlsHintDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// On 1920x1080: the card at x 946-971, +// y 868-901 with its spine at x 974-976; the "?" at x 953-965, y 876-894. +ControlsHintDetector::ControlsHintDetector(Color color) + : m_color(color) + , m_card(947.0 / 1920, 872.0 / 1080, 24.0 / 1920, 28.0 / 1080) + , m_glyph(955.0 / 1920, 879.0 / 1080, 8.0 / 1920, 7.0 / 1080) + , m_around{ + {928.0 / 1920, 872.0 / 1080, 14.0 / 1920, 28.0 / 1080}, + {980.0 / 1920, 872.0 / 1080, 14.0 / 1920, 28.0 / 1080}, + {945.0 / 1920, 856.0 / 1080, 30.0 / 1920, 8.0 / 1080}, + {945.0 / 1920, 906.0 / 1080, 30.0 / 1920, 8.0 / 1080}, + } +{} + +void ControlsHintDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_card); + for (const ImageFloatBox& box : m_around){ + items.add(m_color, box); + } +} + +// With the hint up the card is 88%+ sky blue, the "?" at most 48%, and +// nothing around it is. A sky behind the card comes closest (card 76%), +// but then the "?" and the area around it are blue too (100%, 54%). +bool ControlsHintDetector::detect(const ImageViewRGB32& screen){ + if (sky_blue_share(screen, m_card) < 0.80 || sky_blue_share(screen, m_glyph) > 0.70){ + return false; + } + for (const ImageFloatBox& box : m_around){ + if (sky_blue_share(screen, box) > 0.10){ + return false; + } + } + return true; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.h new file mode 100644 index 0000000000..5dfe675d07 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.h @@ -0,0 +1,47 @@ +/* PokemonLZA AutoStory - Controls hint detector + * + * From: https://github.com/PokemonAutomation/ + * + * The tutorial's controls hint under the player, a "?" card over + * "(L) Move" and "(R) Control Camera", which shows for ~10 s when the + * player gets control. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryControlsHintDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryControlsHintDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The controls hint the tutorial shows under the player when they get +// control: its "?" card, sky blue with a dark "?", at x 947-971, y 872-900 +// on 1920x1080, centred, with no sky blue around it. Only the card is +// read: the rows under it ("(L) Move", "(R) Control Camera") are centred +// too, so their icons move with the text's length. +class ControlsHintDetector : public StaticScreenDetector{ +public: + explicit ControlsHintDetector(Color color = COLOR_BLUE); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_card; + ImageFloatBox m_glyph; + ImageFloatBox m_around[4]; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp new file mode 100644 index 0000000000..629d8896ba --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp @@ -0,0 +1,821 @@ +/* PokemonLZA AutoStory - Detectors Tests + * + * From: https://github.com/PokemonAutomation/ + * + * Synthetic frames, and console frames (1920x1080 JPEG or PNG) from + * CommandLineTests/PokemonLZA/AutoStoryDetectors/, one folder per scene + * sequence. + * + */ + + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "PokemonLZA_AutoStoryAdvanceArrowDetector.h" +#include "PokemonLZA_AutoStoryAutosaveDetector.h" +#include "PokemonLZA_AutoStoryCaughtMarkReader.h" +#include "PokemonLZA_AutoStoryControlsHintDetector.h" +#include "PokemonLZA_AutoStoryDetectors_Tests.h" +#include "PokemonLZA_AutoStoryMainMissionBannerDetector.h" +#include "PokemonLZA_AutoStoryMinimapZoneDetector.h" +#include "PokemonLZA_AutoStoryMissionCompleteDetector.h" +#include "PokemonLZA_AutoStoryPlatformArrivalDetector.h" +#include "PokemonLZA_AutoStorySkipPromptDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +constexpr size_t WIDTH = 1920; +constexpr size_t HEIGHT = 1080; + + +// What each detector reads on a frame: true or false, or none when a test +// does not check that detector. +struct DetectorReads{ + std::optional platform = {}; + std::optional skip_prompt = {}; + std::optional autosave = {}; + std::optional controls_hint = {}; + std::optional advance_arrow = {}; + std::optional mission_banner = {}; + std::optional mission_complete = {}; + std::optional minimap_zone = {}; +}; + +struct DetectorField{ + std::string_view name; + std::optional DetectorReads::* read; +}; +const DetectorField DETECTOR_FIELDS[] = { + {"platform", &DetectorReads::platform}, + {"skip prompt", &DetectorReads::skip_prompt}, + {"autosave", &DetectorReads::autosave}, + {"controls hint", &DetectorReads::controls_hint}, + {"advance arrow", &DetectorReads::advance_arrow}, + {"mission banner", &DetectorReads::mission_banner}, + {"mission complete", &DetectorReads::mission_complete}, + {"minimap zone", &DetectorReads::minimap_zone}, +}; + +// `reads` with every detector checked: one it leaves unset must not fire. +DetectorReads every_detector(DetectorReads reads){ + for (const DetectorField& field : DETECTOR_FIELDS){ + reads.*field.read = (reads.*field.read).value_or(false); + } + return reads; +} + +// The detectors that fire in `got`, for the log: ", platform, autosave". +std::string firing(const DetectorReads& got){ + std::string text; + for (const DetectorField& field : DETECTOR_FIELDS){ + if ((got.*field.read).value_or(false)){ + text += std::format(", {}", field.name); + } + } + return text; +} + +// The detectors `want` checks that `got` reads otherwise: " platform +// (expected yes)"; empty when all agree. +std::string mismatches(const DetectorReads& got, const DetectorReads& want){ + std::string text; + for (const DetectorField& field : DETECTOR_FIELDS){ + const std::optional& expected = want.*field.read; + if (expected && got.*field.read != expected){ + text += std::format(" {} (expected {})", field.name, *expected ? "yes" : "no"); + } + } + return text; +} + +// Every detector of this module, one of each. +struct StoryDetectors{ + PlatformArrivalDetector platform; + SkipPromptDetector skip; + AutosaveDetector autosave; + ControlsHintDetector hint; + AdvanceArrowDetector arrow; + MainMissionBannerDetector banner; + MissionCompleteDetector complete; + MinimapZoneDetector zone; + + DetectorReads read(const ImageViewRGB32& frame){ + return DetectorReads{ + .platform = platform.detect(frame), .skip_prompt = skip.detect(frame), + .autosave = autosave.detect(frame), .controls_hint = hint.detect(frame), + .advance_arrow = arrow.detect(frame), .mission_banner = banner.detect(frame), + .mission_complete = complete.detect(frame), .minimap_zone = zone.detect(frame), + }; + } +}; + + +// Synthetic frames. + +void fill_box(size_t min_x, size_t min_y, size_t max_x, size_t max_y, uint32_t color, ImageRGB32& image){ + for (size_t y = min_y; y < max_y; y++){ + for (size_t x = min_x; x < max_x; x++){ + image.pixel(x, y) = color; + } + } +} + +// A disk of radius `radius` centered on (`center_x`, `center_y`). +void fill_disk(size_t center_x, size_t center_y, size_t radius, uint32_t color, ImageRGB32& image){ + for (size_t y = center_y - radius; y <= center_y + radius; y++){ + for (size_t x = center_x - radius; x <= center_x + radius; x++){ + const double dx = static_cast(x) - static_cast(center_x); + const double dy = static_cast(y) - static_cast(center_y); + if (dx * dx + dy * dy <= static_cast(radius * radius)){ + image.pixel(x, y) = color; + } + } + } +} + +ImageRGB32 solid_frame(uint32_t color){ + ImageRGB32 image(WIDTH, HEIGHT); + image.fill(color); + return image; +} + +// The skip badge drawn on black: the (+) icon (a white disk, its plus +// dark, as the ButtonPlus template), white text strokes (one column in +// four), the dark strip under them. +ImageRGB32 skip_badge_frame(){ + ImageRGB32 image = solid_frame(0xff000000); + fill_box(1500, 100, 1920, 175, 0xff2c2c30, image); + fill_disk(1575, 139, 11, 0xffffffff, image); + fill_box(1569, 138, 1582, 141, 0xff2c2c30, image); + fill_box(1574, 133, 1577, 146, 0xff2c2c30, image); + for (size_t x = 1630; x < 1875; x += 4){ + fill_box(x, 124, x + 1, 156, 0xffffffff, image); + } + return image; +} + +// The controls hint's card drawn on black: sky blue with a dark block for +// the top of its "?". +ImageRGB32 controls_hint_frame(){ + ImageRGB32 image = solid_frame(0xff000000); + fill_box(946, 868, 972, 902, 0xff7dd2fa, image); + fill_box(956, 879, 962, 886, 0xff1c7db0, image); + return image; +} + +// The platform reference with the player's area (x 780-1200, y 330-1080), +// which is in no region, painted white. +ImageRGB32 platform_with_player_frame(){ + ImageRGB32 image = platform_reference_image(WIDTH, HEIGHT); + fill_box(780, 330, 1200, HEIGHT, 0xffffffff, image); + return image; +} + +// The advance chevron drawn in `ink` on `background`: a V 32x19 at +// x 1400, y 1004, each arm 18% of the width across. +ImageRGB32 chevron_frame(uint32_t background, uint32_t ink){ + ImageRGB32 image = solid_frame(background); + for (size_t y = 0; y < 19; y++){ + const double v = (double(y) + 0.5) / 19; + for (size_t x = 0; x < 32; x++){ + const double u = (double(x) + 0.5) / 32; + if (std::abs(std::abs(u - 0.5) - (0.40 * (1 - v) + 0.03)) <= 0.09){ + image.pixel(1400 + x, 1004 + y) = ink; + } + } + } + return image; +} + +// A solid white block the chevron's size on the ad's light blue. +ImageRGB32 chevron_sized_block_frame(){ + ImageRGB32 image = solid_frame(0xffc1ecf0); + fill_box(1400, 1004, 1432, 1023, 0xffffffff, image); + return image; +} + +// The autosave badge's pencil, 48 px right of the Chinese badge's (where +// the shorter Japanese text puts it), over `background`: a white stroke 4 px +// wide in a `glow` band 12 px wide. +ImageRGB32 autosave_badge_frame(uint32_t background, uint32_t glow){ + ImageRGB32 image(WIDTH, HEIGHT); + image.fill(background); + fill_box(1655, 73, 1667, 104, glow, image); + fill_box(1659, 73, 1663, 104, 0xffffffff, image); + return image; +} + +// How mission_banner_frame() draws the icon. +enum class BannerIcon{ + NONE, + DRAWN, +}; + +// The mission banner drawn on a grey scene: the yellow bar, and with +// BannerIcon::DRAWN the icon's white hexagon (as a block) with the orange +// stroke, `dx` right of where the detector expects it. +ImageRGB32 mission_banner_frame(size_t dx, BannerIcon icon){ + ImageRGB32 image = solid_frame(0xff606060); + fill_box(345, 410, 1590, 600, 0xfffad228, image); + if (icon == BannerIcon::DRAWN){ + fill_box(700 + dx, 327, 738 + dx, 370, 0xffffffff, image); + fill_box(716 + dx, 336, 720 + dx, 357, 0xfff07020, image); + } + return image; +} + +// The mission complete card drawn on a grey scene: the yellow bar, and +// "COMPLETE" as green strokes (three columns in four) above it. +ImageRGB32 mission_complete_frame(){ + ImageRGB32 image = solid_frame(0xff606060); + fill_box(345, 390, 1590, 585, 0xfffad228, image); + for (size_t x = 615; x < 1300; x += 4){ + fill_box(x, 275, x + 3, 345, 0xffa5f07d, image); + } + return image; +} + +constexpr uint32_t WHITE = 0xffffffff; +constexpr uint32_t BLACK = 0xff000000; +constexpr uint32_t GRAY = 0xff808080; +constexpr uint32_t AD_LIGHT_BLUE = 0xffc1ecf0; +constexpr uint32_t BOX_WHITE = 0xfffcfcf7; +constexpr uint32_t BOX_TEXT_DARK = 0xff323648; +constexpr uint32_t HINT_SKY_BLUE = 0xff7dd2fa; +constexpr uint32_t BANNER_YELLOW = 0xfffad228; +constexpr uint32_t COMPLETE_GREEN = 0xffa5f07d; +constexpr uint32_t ZONE_GREEN = 0xffa0f0c0; +constexpr uint32_t ARROW_BLUE = 0xff00a0f0; +constexpr uint32_t MAP_BEIGE = 0xffe0d0b0; + +// The minimap's middle on `ground` (the zone's translucent green, or the +// city map's beige), with the player's arrow (a 10 x 20 blue block at the +// middle) when `arrow`. +ImageRGB32 minimap_frame(uint32_t ground, bool arrow){ + ImageRGB32 image = solid_frame(GRAY); + fill_box(40, 40, 320, 320, ground, image); + if (arrow){ + fill_box(175, 170, 185, 190, ARROW_BLUE, image); + } + return image; +} + +struct SyntheticFrameCase{ + std::string name; + struct Input{ + std::function frame; + }; + Input input; + struct Expectation{ + DetectorReads reads = {}; + }; + Expectation expectation; +}; + +const SyntheticFrameCase SYNTHETIC_FRAME_CASES[] = { + { + .name = "an empty image", + .input = {.frame = []{ return ImageRGB32(); }}, + .expectation = {.reads = every_detector({})}, + }, + { + .name = "a black frame", + .input = {.frame = []{ return solid_frame(BLACK); }}, + .expectation = {.reads = { + .skip_prompt = false, .autosave = false, .controls_hint = false, .advance_arrow = false, .minimap_zone = false, + }}, + }, + { + .name = "a white frame", + .input = {.frame = []{ return solid_frame(WHITE); }}, + .expectation = {.reads = { + .skip_prompt = false, .autosave = false, .controls_hint = false, .advance_arrow = false, .mission_banner = false, + .minimap_zone = false, + }}, + }, + { + .name = "a gray frame", + .input = {.frame = []{ return solid_frame(GRAY); }}, + .expectation = {.reads = {.platform = false}}, + }, + { + .name = "the platform reference", + .input = {.frame = []{ return platform_reference_image(WIDTH, HEIGHT); }}, + .expectation = {.reads = {.platform = true}}, + }, + { + .name = "the platform reference with the player's area painted", + .input = {.frame = platform_with_player_frame}, + .expectation = {.reads = {.platform = true}}, + }, + { + .name = "the skip badge", + .input = {.frame = skip_badge_frame}, + .expectation = {.reads = {.skip_prompt = true, .autosave = false}}, + }, + // The autosave badge's glow is bright over a dark sky and dim over a + // lamp-lit wall (about 28, 70, 85: 02.04's check-in, + // frame 0075, which the badge detector missed). + { + .name = "the autosave badge with a bright glow", + .input = {.frame = []{ return autosave_badge_frame(0xff101418, 0xff28c8e6); }}, + .expectation = {.reads = {.autosave = true}}, + }, + { + .name = "the autosave badge with a dim glow over a dark wall", + .input = {.frame = []{ return autosave_badge_frame(0xff252722, 0xff1c4655); }}, + .expectation = {.reads = {.autosave = true}}, + }, + { + .name = "the autosave badge's white stroke with no glow", + .input = {.frame = []{ return autosave_badge_frame(0xff252722, 0xff252722); }}, + .expectation = {.reads = {.autosave = false}}, + }, + { + .name = "the controls hint card", + .input = {.frame = controls_hint_frame}, + .expectation = {.reads = {.controls_hint = true}}, + }, + { + .name = "a frame all in the hint card's blue (a sky)", + .input = {.frame = []{ return solid_frame(HINT_SKY_BLUE); }}, + .expectation = {.reads = {.controls_hint = false}}, + }, + { + .name = "a white chevron on the ad's light blue", + .input = {.frame = []{ return chevron_frame(AD_LIGHT_BLUE, WHITE); }}, + .expectation = {.reads = {.advance_arrow = true}}, + }, + { + .name = "a dark chevron on a white box", + .input = {.frame = []{ return chevron_frame(BOX_WHITE, BOX_TEXT_DARK); }}, + .expectation = {.reads = {.advance_arrow = true}}, + }, + { + .name = "a solid block the chevron's size", + .input = {.frame = chevron_sized_block_frame}, + .expectation = {.reads = {.advance_arrow = false}}, + }, + { + .name = "the mission banner", + .input = {.frame = []{ return mission_banner_frame(0, BannerIcon::DRAWN); }}, + .expectation = {.reads = {.mission_banner = true, .mission_complete = false}}, + }, + { + .name = "the mission banner with its icon 40 px right", + .input = {.frame = []{ return mission_banner_frame(40, BannerIcon::DRAWN); }}, + .expectation = {.reads = {.mission_banner = true}}, + }, + { + .name = "the mission banner's bar without the icon", + .input = {.frame = []{ return mission_banner_frame(0, BannerIcon::NONE); }}, + .expectation = {.reads = {.mission_banner = false}}, + }, + { + .name = "a frame all in the bar's yellow", + .input = {.frame = []{ return solid_frame(BANNER_YELLOW); }}, + .expectation = {.reads = {.mission_banner = false, .mission_complete = false}}, + }, + { + .name = "the mission complete card", + .input = {.frame = mission_complete_frame}, + .expectation = {.reads = {.mission_banner = false, .mission_complete = true}}, + }, + { + .name = "a frame all in the card's green", + .input = {.frame = []{ return solid_frame(COMPLETE_GREEN); }}, + .expectation = {.reads = {.mission_complete = false}}, + }, + { + .name = "the minimap in a wild zone: the player's arrow on the zone's green", + .input = {.frame = []{ return minimap_frame(ZONE_GREEN, true); }}, + .expectation = {.reads = {.minimap_zone = true}}, + }, + { + .name = "the zone's green on the minimap without the player's arrow", + .input = {.frame = []{ return minimap_frame(ZONE_GREEN, false); }}, + .expectation = {.reads = {.minimap_zone = false}}, + }, + { + .name = "the minimap in the city: the player's arrow on the map's beige", + .input = {.frame = []{ return minimap_frame(MAP_BEIGE, true); }}, + .expectation = {.reads = {.minimap_zone = false}}, + }, + { + .name = "a frame all in the zone's green", + .input = {.frame = []{ return solid_frame(ZONE_GREEN); }}, + .expectation = {.reads = {.minimap_zone = false}}, + }, +}; + +// The lock-on name plate's HP bar (lock_target_plate_bar()), drawn as on +// the console's 03.08 frames (640 px, scaled here to 1920): +// the bar about 45, 198, 115 and 108 x 18 under the target's name; the +// bright grass around it (68, 160, 80, Bunnelby's plate at 60) and the +// player's own HP bar (63 x 12, over the head while a wild Pokemon +// attacks). Above the bar's right end, the gender mark (an orange disk +// with a white sign, about 255, 118, 68) and, for a species already +// caught, a Poke Ball (pinkish red over white, about 255, 107, 110): +// Scatterbug's plate shows only the first, Bunnelby's both. +constexpr uint32_t PLATE_BAR_GREEN = 0xff2dc673; +constexpr uint32_t GENDER_ORANGE = 0xffff7644; +constexpr uint32_t BALL_RED = 0xffff6b6e; +constexpr uint32_t BRIGHT_GRASS = 0xff44a050; +constexpr uint32_t DARK_GROUND = 0xff303030; + +// What plate_frame() draws besides `ground`. +struct PlateDrawing{ + bool plate_bar = false; // at x 950, y 515 + bool player_bar = false; // at x 990, y 560 + bool gender_mark = false; // at x 1020, y 430 + bool ball_mark = false; // at x 1056, y 430 +}; + +ImageRGB32 plate_frame(uint32_t ground, PlateDrawing drawing){ + ImageRGB32 image = solid_frame(ground); + if (drawing.plate_bar){ + fill_box(950, 515, 1058, 533, PLATE_BAR_GREEN, image); + } + if (drawing.player_bar){ + fill_box(990, 560, 1053, 572, PLATE_BAR_GREEN, image); + } + if (drawing.gender_mark){ + fill_box(1020, 430, 1044, 454, GENDER_ORANGE, image); + fill_box(1028, 438, 1036, 446, WHITE, image); + } + if (drawing.ball_mark){ + fill_box(1056, 430, 1080, 442, BALL_RED, image); + fill_box(1056, 442, 1080, 454, WHITE, image); + } + return image; +} + +struct PlateFrameCase{ + std::string name; + struct Input{ + std::function frame; + }; + Input input; + struct Expectation{ + // The bar's left edge as a share of the width; none: no plate. + std::optional bar_x = {}; + // lock_target_caught(); none: not checked. + std::optional caught = {}; + }; + Expectation expectation; +}; + +const PlateFrameCase PLATE_FRAME_CASES[] = { + { + .name = "the plate's bar on dark ground", + .input = {.frame = []{ return plate_frame(DARK_GROUND, {.plate_bar = true}); }}, + .expectation = {.bar_x = 950.0 / WIDTH}, + }, + { + .name = "the plate's bar on the bright grass it touches", + .input = {.frame = []{ return plate_frame(BRIGHT_GRASS, {.plate_bar = true}); }}, + .expectation = {.bar_x = 950.0 / WIDTH}, + }, + { + .name = "the plate's bar and the player's HP bar", + .input = {.frame = []{ return plate_frame(BRIGHT_GRASS, {.plate_bar = true, .player_bar = true}); }}, + .expectation = {.bar_x = 950.0 / WIDTH}, + }, + { + .name = "a plate with the gender mark only (Scatterbug, not caught)", + .input = {.frame = []{ return plate_frame(DARK_GROUND, {.plate_bar = true, .gender_mark = true}); }}, + .expectation = {.bar_x = 950.0 / WIDTH, .caught = false}, + }, + { + .name = "a plate with the gender mark and the Poke Ball (Bunnelby, caught)", + .input = {.frame = []{ return plate_frame(DARK_GROUND, {.plate_bar = true, .gender_mark = true, .ball_mark = true}); }}, + .expectation = {.bar_x = 950.0 / WIDTH, .caught = true}, + }, + { + .name = "the player's HP bar alone", + .input = {.frame = []{ return plate_frame(DARK_GROUND, {.player_bar = true}); }}, + .expectation = {}, + }, + { + .name = "bright grass alone", + .input = {.frame = []{ return plate_frame(BRIGHT_GRASS, {}); }}, + .expectation = {}, + }, + { + .name = "a frame all in the bar's green", + .input = {.frame = []{ return solid_frame(PLATE_BAR_GREEN); }}, + .expectation = {}, + }, +}; + +UnitTestResult check_plate_frame(const PlateFrameCase& test, Logger& logger){ + const ImageRGB32 frame = test.input.frame(); + const std::optional bar = lock_target_plate_bar(frame); + const std::optional want = test.expectation.bar_x; + if (bar){ + logger.log(std::format("{}: bar at ({:.3f}, {:.3f}), {:.3f} wide.", test.name, bar->x, bar->y, bar->width)); + }else{ + logger.log(std::format("{}: no bar.", test.name)); + } + if (bar.has_value() != want.has_value() || (bar && std::abs(bar->x - *want) > 0.002)){ + return std::format("Wrong on {}: expected {}.", test.name, want ? std::format("a bar at x = {:.3f}", *want) : std::string("no bar")); + } + if (test.expectation.caught && lock_target_caught(frame) != test.expectation.caught){ + return std::format("Wrong on {}: expected the caught mark {}.", test.name, *test.expectation.caught ? "read" : "not read"); + } + return true; +} + +UnitTestResult check_synthetic_frame(const SyntheticFrameCase& test, Logger& logger){ + StoryDetectors detectors; + const DetectorReads got = detectors.read(test.input.frame()); + logger.log(std::format("{}: {}{}", test.name, to_string(detectors.platform.last_distances()), firing(got))); + const std::string wrong = mismatches(got, test.expectation.reads); + if (!wrong.empty()){ + return std::format("Wrong on {}:{}", test.name, wrong); + } + return true; +} + + +// Console frames, from folders in CommandLineTests. + +// The number of `digits` digits at `offset` in a frame's file name; none +// if they are not all digits. +std::optional frame_number(std::string_view name, size_t offset, size_t digits){ + if (name.size() < offset + digits){ + return std::nullopt; + } + const std::string_view text = name.substr(offset, digits); + int number = 0; + const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), number); + if (error != std::errc() || end != text.data() + text.size()){ + return std::nullopt; + } + return number; +} + +// What a frame shows, from the readings in its name +// ("0133-nothing+mission_banner.jpg"). +DetectorReads as_named(std::string_view name){ + const auto has = [name](std::string_view reading){ + return name.find(reading) != std::string_view::npos; + }; + return DetectorReads{ + .platform = has("+platform"), .skip_prompt = has("+skip_prompt"), + .autosave = has("+autosave"), .controls_hint = has("+controls_hint"), + .advance_arrow = has("+advance_arrow"), .mission_banner = has("+mission_banner"), + }; +} + +// The opening cutscene up to the platform: only 0050 shows the skip +// prompt, and none is the platform. +std::optional opening(std::string_view name){ + return DetectorReads{.skip_prompt = name.starts_with("0050-")}; +} + +// The platform (as named) with the controls hint (0001-0006), the walk, +// the white fade, the title card, the text-free flyover, the starters, the +// tourist choice and the start of the ad: no autosave badge, the advance +// chevron only on the one box with its text done (0040). +std::optional arrival_to_ad(std::string_view name){ + const std::optional index = frame_number(name, 0, 4); + if (!index){ + return std::nullopt; + } + return DetectorReads{ + .platform = name.find("+platform") != std::string_view::npos, + .controls_hint = *index <= 6, .advance_arrow = *index == 40, + }; +} + +// As arrival_to_ad(), on to the ad's subtitles (0090-0115). The advance +// chevron on the boxes with their text done (0044, 0049, 0053, 0078) and +// on the subtitles once typed (0093-0115). +std::optional arrival_to_ad_subtitles(std::string_view name){ + const std::optional index = frame_number(name, 0, 4); + if (!index){ + return std::nullopt; + } + const bool box_arrow = *index == 44 || *index == 49 || *index == 53 || *index == 78; + return DetectorReads{ + .platform = name.find("+platform") != std::string_view::npos, + .controls_hint = *index <= 6, + .advance_arrow = box_arrow || (*index >= 93 && *index <= 115), + }; +} + +// As arrival_to_ad_subtitles(), on through the ad and the bag scene. The +// platform, the controls hint and the advance chevron as named; the +// mission banner on 0181 only, the last frame. +std::optional arrival_to_mission_banner(std::string_view name){ + const std::optional index = frame_number(name, 0, 4); + if (!index){ + return std::nullopt; + } + return DetectorReads{ + .platform = name.find("+platform") != std::string_view::npos, + .controls_hint = name.find("+controls_hint") != std::string_view::npos, + .advance_arrow = name.find("+advance_arrow") != std::string_view::npos, + .mission_banner = *index == 181, + }; +} + +// Every detector as named; the banner on 0181-0189, no complete card. The +// autosave badge on 0197 too: it is named "nothing" but shows the badge +// (Traditional Chinese, top right), right after the black fade. +std::optional arrival_past_mission_banner(std::string_view name){ + DetectorReads want = as_named(name); + if (name.starts_with("0197-")){ + want.autosave = true; + } + return want; +} + +// As named, and the complete card on 0045 and 0046 (named "nothing"); the +// next mission's banner (0133-0139) is not the card. +std::optional mission_complete_to_next_banner(std::string_view name){ + DetectorReads want = as_named(name); + want.mission_complete = name.starts_with("0045-") || name.starts_with("0046-"); + return want; +} + +// 03.07's last catches to 03.08's start in Wild Zone 1 (PNG): three black +// fades, one city frame and the autosave as named; the minimap shows the +// player in the zone's green on the last three (0075-0077), 0077 being +// where 03.08 starts (the far gate at 127). +std::optional catches_to_zone_1_start(std::string_view name){ + const std::optional index = frame_number(name, 0, 4); + if (!index){ + return std::nullopt; + } + DetectorReads want = as_named(name); + want.minimap_zone = *index >= 75; + return want; +} + +struct FrameFolderCase{ + std::string name; + struct Input{ + std::string_view folder; // In CommandLineTests/PokemonLZA/AutoStoryDetectors/. + }; + Input input; + struct Expectation{ + size_t frames = 0; // How many frames of the folder `reads` has an expectation for. + // Each frame's reads, by file name; none for a file not in the set. + // A detector it leaves unset must not fire. + std::function(std::string_view name)> reads = {}; + }; + Expectation expectation; +}; + +const FrameFolderCase FRAME_FOLDER_CASES[] = { + { + .name = "the opening cutscene", + .input = {.folder = "Opening"}, + .expectation = {.frames = 56, .reads = opening}, + }, + { + .name = "the arrival to the ad", + .input = {.folder = "ArrivalToAd"}, + .expectation = {.frames = 89, .reads = arrival_to_ad}, + }, + { + .name = "the arrival to the ad's subtitles", + .input = {.folder = "ArrivalToAdSubtitles"}, + .expectation = {.frames = 115, .reads = arrival_to_ad_subtitles}, + }, + { + .name = "the arrival to the mission banner", + .input = {.folder = "ArrivalToMissionBanner"}, + .expectation = {.frames = 181, .reads = arrival_to_mission_banner}, + }, + { + .name = "the arrival past the mission banner", + .input = {.folder = "ArrivalPastMissionBanner"}, + .expectation = {.frames = 221, .reads = arrival_past_mission_banner}, + }, + { + .name = "the mission complete card to the next mission's banner", + .input = {.folder = "MissionCompleteToNextBanner"}, + .expectation = {.frames = 139, .reads = mission_complete_to_next_banner}, + }, + { + .name = "the catches to 03.08's start in Wild Zone 1", + .input = {.folder = "CatchesToZone1Start"}, + .expectation = {.frames = 8, .reads = catches_to_zone_1_start}, + }, +}; + +UnitTestResult check_frame_folder(const FrameFolderCase& test, Logger& logger){ + const Filesystem::Path folder(std::format("{}PokemonLZA/AutoStoryDetectors/{}/", UNIT_TEST_RESOURCE_PATH(), test.input.folder)); + if (!Filesystem::is_directory(folder)){ + return std::format("{} is not a folder.", folder.string()); + } + StoryDetectors detectors; + size_t frames = 0; + std::string failures; + for (const Filesystem::Path& path : Filesystem::list_directory(folder)){ + const std::string name = path.filename().string(); + if (!name.ends_with(".jpg") && !name.ends_with(".png")){ + continue; + } + const std::optional want = test.expectation.reads(name); + if (!want){ + continue; + } + const DetectorReads got = detectors.read(ImageRGB32(path.string())); + logger.log(std::format("{}: {}{}", name, to_string(detectors.platform.last_distances()), firing(got))); + const std::string wrong = mismatches(got, every_detector(*want)); + if (!wrong.empty()){ + failures += std::format(" {}:{};", name, wrong); + } + frames++; + } + if (frames != test.expectation.frames){ + return std::format("Expected {} frames in {}, found {}.", test.expectation.frames, folder.string(), frames); + } + if (!failures.empty()){ + return std::format("Wrong on:{}", failures); + } + return true; +} + + +class Test_SyntheticFrame : public UnitTest{ +public: + explicit Test_SyntheticFrame(const SyntheticFrameCase& test) + : UnitTest(std::format("PokemonLZA::AutoStoryDetectors - synthetic frame: {}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ + return check_synthetic_frame(m_test, logger); + } + +private: + const SyntheticFrameCase& m_test; +}; + +class Test_PlateFrame : public UnitTest{ +public: + explicit Test_PlateFrame(const PlateFrameCase& test) + : UnitTest(std::format("PokemonLZA::AutoStoryDetectors - lock-on plate: {}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ + return check_plate_frame(m_test, logger); + } + +private: + const PlateFrameCase& m_test; +}; + +class Test_FrameFolder : public UnitTest{ +public: + explicit Test_FrameFolder(const FrameFolderCase& test) + : UnitTest(std::format("PokemonLZA::AutoStoryDetectors - console frames: {}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ + return check_frame_folder(m_test, logger); + } + +private: + const FrameFolderCase& m_test; +}; + +} // namespace + + +void add_tests_AutoStoryDetectors(UnitTestDatabase& database){ + for (const SyntheticFrameCase& test : SYNTHETIC_FRAME_CASES){ + database.add(test); + } + for (const PlateFrameCase& test : PLATE_FRAME_CASES){ + database.add(test); + } + for (const FrameFolderCase& test : FRAME_FOLDER_CASES){ + database.add(test); + } +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h new file mode 100644 index 0000000000..bda582d5d1 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h @@ -0,0 +1,26 @@ +/* PokemonLZA AutoStory - Detectors Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryDetectors_Tests_H +#define PokemonAutomation_PokemonLZA_AutoStoryDetectors_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Registers the tests of the AutoStory detectors (Detectors/): synthetic +// frames, and console frames from CommandLineTests. +void add_tests_AutoStoryDetectors(UnitTestDatabase& database); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.cpp new file mode 100644 index 0000000000..0ea79661f1 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.cpp @@ -0,0 +1,74 @@ +/* PokemonLZA AutoStory - Main mission banner detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStoryMainMissionBannerDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// On 1920x1080: the bar's yellow fills x 430-820 and x 1100-1500 at +// y 425-585 (left and right of the name), none of it 15 px above or +// below the bar; the icon's hexagon is white at x 700-710 and 724-734, +// y 337-358, with the "!"'s orange stroke at x 716-719 between them. +MainMissionBannerDetector::MainMissionBannerDetector(Color color) + : m_color(color) + , m_bar_left(430.0 / 1920, 425.0 / 1080, 390.0 / 1920, 160.0 / 1080) + , m_bar_right(1100.0 / 1920, 425.0 / 1080, 400.0 / 1920, 160.0 / 1080) + , m_above(430.0 / 1920, 375.0 / 1080, 1070.0 / 1920, 25.0 / 1080) + , m_below(430.0 / 1920, 615.0 / 1080, 1070.0 / 1920, 25.0 / 1080) + , m_icon_left(700.0 / 1920, 337.0 / 1080, 10.0 / 1920, 21.0 / 1080) + , m_icon_right(724.0 / 1920, 337.0 / 1080, 10.0 / 1920, 21.0 / 1080) + , m_icon_stroke(716.0 / 1920, 337.0 / 1080, 3.0 / 1920, 11.0 / 1080) +{} + +void MainMissionBannerDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_bar_left); + items.add(m_color, m_bar_right); + items.add(m_color, m_above); + items.add(m_color, m_below); + items.add(m_color, m_icon_left); + items.add(m_color, m_icon_right); + items.add(m_color, m_icon_stroke); +} + +// On the banner both bar boxes are 100% yellow, the strips around the bar +// 0-1%, and the icon's three boxes 100% at dx 0. Other screens have the +// bar boxes 8% yellow at most and the icon's weakest box 27% at most (a +// city flyover). The icon is looked for 60 px either side, as a longer or +// shorter header moves it. +bool MainMissionBannerDetector::detect(const ImageViewRGB32& screen){ + if (banner_yellow_share(screen, m_bar_left) < 0.6 || banner_yellow_share(screen, m_bar_right) < 0.6){ + return false; + } + if (banner_yellow_share(screen, m_above) > 0.3 || banner_yellow_share(screen, m_below) > 0.3){ + return false; + } + const auto bright = [](Color pixel){ + return pixel.red() > 225 && pixel.green() > 225 && pixel.blue() > 225; + }; + const auto orange = [](Color pixel){ + return pixel.red() > 200 && pixel.blue() < 150; + }; + for (double dx = -60; dx <= 60; dx += 2){ + if (share_of(screen, shifted(m_icon_left, dx), bright) >= 0.7 && + share_of(screen, shifted(m_icon_right, dx), bright) >= 0.7 && + share_of(screen, shifted(m_icon_stroke, dx), orange) >= 0.7 + ){ + return true; + } + } + return false; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.h new file mode 100644 index 0000000000..17bc5d634c --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.h @@ -0,0 +1,52 @@ +/* PokemonLZA AutoStory - Main mission banner detector + * + * From: https://github.com/PokemonAutomation/ + * + * The "MAIN MISSION 01" banner across the middle when a main mission + * starts. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryMainMissionBannerDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryMainMissionBannerDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The banner a main mission starts with: a see-through yellow bar at +// x 345-1590, y 410-600 on 1920x1080 with the mission's name in the +// middle, and above it, left of "MAIN MISSION 01", a white hexagon with +// an orange "!" at about x 700-738, y 327-370. The bar is read left and +// right of the name, with no yellow just above or below it, and the icon +// is looked for along the header (the header's text is centred). Text is +// not read, so any language matches. +class MainMissionBannerDetector : public StaticScreenDetector{ +public: + explicit MainMissionBannerDetector(Color color = COLOR_ORANGE); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_bar_left; + ImageFloatBox m_bar_right; + ImageFloatBox m_above; + ImageFloatBox m_below; + ImageFloatBox m_icon_left; + ImageFloatBox m_icon_right; + ImageFloatBox m_icon_stroke; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMinimapZoneDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMinimapZoneDetector.cpp new file mode 100644 index 0000000000..dbcd13abe0 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMinimapZoneDetector.cpp @@ -0,0 +1,57 @@ +/* PokemonLZA AutoStory - Minimap zone detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStoryMinimapZoneDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +// The wild zone's green on the minimap (about 160, 240, 192, and 144, +// 224, 160 where darker): green over 150 and over red and blue by 20. +bool zone_green(Color pixel){ + return pixel.green() > 150 && pixel.green() > pixel.red() + 20 && pixel.green() > pixel.blue() + 20; +} + +// The player's arrow's blue (0, 160, 240 and 0, 144, 240; its view cone, +// 112, 208, 240, is left out by red under 80 and green under 200). +bool arrow_blue(Color pixel){ + return pixel.blue() > 200 && pixel.red() < 80 && pixel.green() > 100 && pixel.green() < 200; +} + +} // namespace + + +MinimapZoneDetector::MinimapZoneDetector(Color color) + : m_color(color) + , m_middle(150.0 / 1920, 150.0 / 1080, 60.0 / 1920, 60.0 / 1080) +{} + +void MinimapZoneDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_middle); +} + +// In Wild Zone 1 at 03.08's start the 60 x 60 middle is 46-47% zone green +// and 10% arrow blue (CatchesToZone1Start/0075-0077). +// Of 82 frames in the city and its cutscenes none has over +// 0.1% of either, except one cutscene (ArrivalPastMissionBanner/0114) +// with 29% of a scenery green and no arrow, and the ad's blue sky +// (ArrivalToAd/0043) with 10% arrow blue and no green: both readings are +// needed, green at a quarter and the arrow at 4%. +bool MinimapZoneDetector::detect(const ImageViewRGB32& screen){ + return share_of(screen, m_middle, zone_green) >= 0.25 && share_of(screen, m_middle, arrow_blue) >= 0.04; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMinimapZoneDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMinimapZoneDetector.h new file mode 100644 index 0000000000..161c36e8a0 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMinimapZoneDetector.h @@ -0,0 +1,44 @@ +/* PokemonLZA AutoStory - Minimap zone detector + * + * From: https://github.com/PokemonAutomation/ + * + * Whether the minimap shows the player inside a wild zone: its blue + * arrow on the zone's green. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryMinimapZoneDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryMinimapZoneDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The middle of the minimap at the top left (a circle of radius 140 +// around (180, 180) on 1920x1080), where the player's blue arrow always +// is: inside a wild zone the map there is the zone's translucent green, +// in the city the map's beige and white. No text is read, so any language +// matches. The minimap is not read when a scene or a menu hides it. +class MinimapZoneDetector : public StaticScreenDetector{ +public: + explicit MinimapZoneDetector(Color color = COLOR_GREEN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_middle; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.cpp new file mode 100644 index 0000000000..5485103435 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.cpp @@ -0,0 +1,53 @@ +/* PokemonLZA AutoStory - Mission complete detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStoryMissionCompleteDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// On 1920x1080: the word's green fills x 615-1300, +// y 271-349, none of it 20 px above or below; the bar's yellow fills +// y 390-585, read left and right of the name as for the banner. +MissionCompleteDetector::MissionCompleteDetector(Color color) + : m_color(color) + , m_word(640.0 / 1920, 285.0 / 1080, 640.0 / 1920, 55.0 / 1080) + , m_above(620.0 / 1920, 245.0 / 1080, 680.0 / 1920, 20.0 / 1080) + , m_below(620.0 / 1920, 358.0 / 1080, 680.0 / 1920, 20.0 / 1080) + , m_bar_left(430.0 / 1920, 440.0 / 1080, 390.0 / 1920, 140.0 / 1080) + , m_bar_right(1100.0 / 1920, 440.0 / 1080, 400.0 / 1920, 140.0 / 1080) +{} + +void MissionCompleteDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_word); + items.add(m_color, m_above); + items.add(m_color, m_below); + items.add(m_color, m_bar_left); + items.add(m_color, m_bar_right); +} + +// On the card the word is 70% green, nothing around it is green and both +// bar boxes are 100% yellow. Other screens have the word 11% green at +// most; the main mission banners, which have the same bar, 2% at most. +bool MissionCompleteDetector::detect(const ImageViewRGB32& screen){ + if (complete_green_share(screen, m_word) < 0.40){ + return false; + } + if (complete_green_share(screen, m_above) > 0.10 || complete_green_share(screen, m_below) > 0.10){ + return false; + } + return banner_yellow_share(screen, m_bar_left) >= 0.6 && banner_yellow_share(screen, m_bar_right) >= 0.6; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.h new file mode 100644 index 0000000000..e337d5ff41 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.h @@ -0,0 +1,49 @@ +/* PokemonLZA AutoStory - Mission complete detector + * + * From: https://github.com/PokemonAutomation/ + * + * The "COMPLETE" card when a main mission ends, which waits for A. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryMissionCompleteDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryMissionCompleteDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The card a main mission ends with, which waits for A: a big green +// "COMPLETE" at x 615-1300, y 271-349 on 1920x1080 over the mission's +// yellow bar (x 345-1590, y 390-585, "MAIN MISSION 01" and the mission's +// name), in a yellow hexagon with the mission's item. The word is read +// with nothing green just above or below it, and the bar left and right +// of the name. The word is the same in every language seen (the header +// stays English in Chinese), so its text is not read. +class MissionCompleteDetector : public StaticScreenDetector{ +public: + explicit MissionCompleteDetector(Color color = COLOR_GREEN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_word; + ImageFloatBox m_above; + ImageFloatBox m_below; + ImageFloatBox m_bar_left; + ImageFloatBox m_bar_right; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.cpp new file mode 100644 index 0000000000..f120a7b91d --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.cpp @@ -0,0 +1,67 @@ +/* PokemonLZA AutoStory - Pixel shares + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "PokemonLZA_AutoStoryPixelShares.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +double cyan_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.blue() > pixel.red() + 50 && pixel.green() > pixel.red() + 30 && pixel.blue() > 120; + }); +} + +double dim_cyan_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.blue() > pixel.red() + 40 && pixel.green() > pixel.red() + 25 && pixel.blue() > 70; + }); +} + +// The share of `box`'s pixels whose every channel is above 200. +double white_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.red() > 200 && pixel.green() > 200 && pixel.blue() > 200; + }); +} + +// The share of `box`'s pixels in the controls hint card's sky blue (about +// 125, 210, 250): blue over 200, green over 160, blue over red by 60. +double sky_blue_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.blue() > 200 && pixel.green() > 160 && pixel.blue() > pixel.red() + 60; + }); +} + +// The share of `box`'s pixels in the mission banner's yellow (about 250, +// 210, 40 over a scene): red over 200, green over 140, blue under 130, +// red over blue by 110. +double banner_yellow_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.red() > 200 && pixel.green() > 140 && pixel.blue() < 130 && + pixel.red() > pixel.blue() + 110; + }); +} + +// The share of `box`'s pixels in the mission complete card's green (about +// 165, 240, 125): green over 180, over red by 40 and over blue by 70. +double complete_green_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.green() > 180 && pixel.green() > pixel.red() + 40 && pixel.green() > pixel.blue() + 70; + }); +} + +ImageFloatBox shifted(ImageFloatBox box, double dx){ + box.x += dx / 1920; + return box; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.h new file mode 100644 index 0000000000..e5bcdd490c --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.h @@ -0,0 +1,78 @@ +/* PokemonLZA AutoStory - Pixel shares + * + * From: https://github.com/PokemonAutomation/ + * + * How much of a box's pixels are of a color: what the badge detectors + * (Detectors/) read. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryPixelShares_H +#define PokemonAutomation_PokemonLZA_AutoStoryPixelShares_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The share of `box`'s pixels that `matches` (a Color -> bool). +template +double share_of(const ImageViewRGB32& screen, const ImageFloatBox& box, Matches&& matches); + +// The share of `box`'s pixels that glow cyan: blue over red by 50, green +// over red by 30, blue over 120. +double cyan_share(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// The share of `box`'s pixels that glow cyan, dim too: blue over red by +// 40, green over red by 25, blue over 70. The autosave badge's glow over a +// dark or warm scene (a lamp-lit wall: about 28, 70, 85) is too dim for +// cyan_share(). +double dim_cyan_share(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// The share of `box`'s pixels whose every channel is above 200. +double white_share(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// The share of `box`'s pixels in the controls hint card's sky blue (about +// 125, 210, 250): blue over 200, green over 160, blue over red by 60. +double sky_blue_share(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// The share of `box`'s pixels in the mission banner's yellow (about 250, +// 210, 40 over a scene): red over 200, green over 140, blue under 130, +// red over blue by 110. +double banner_yellow_share(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// The share of `box`'s pixels in the mission complete card's green (about +// 165, 240, 125): green over 180, over red by 40 and over blue by 70. +double complete_green_share(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// `box` moved `dx` pixels (of 1920) right. +ImageFloatBox shifted(ImageFloatBox box, double dx); + + +template +double share_of(const ImageViewRGB32& screen, const ImageFloatBox& box, Matches&& matches){ + const ImageViewRGB32 area = extract_box_reference(screen, box); + const size_t total = area.width() * area.height(); + if (total == 0){ + return 0; + } + size_t count = 0; + for (size_t y = 0; y < area.height(); y++){ + for (size_t x = 0; x < area.width(); x++){ + if (matches(Color(area.pixel(x, y)))){ + count++; + } + } + } + return (double)count / (double)total; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp new file mode 100644 index 0000000000..9f4a95bbf4 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp @@ -0,0 +1,221 @@ +/* PokemonLZA AutoStory - Opening detectors + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Json/JsonArray.h" +#include "Common/Cpp/Json/JsonObject.h" +#include "Common/Cpp/Json/JsonValue.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPlatformArrivalDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +// A region of the frame and the cells it is averaged into. On 1920x1080: +// the top band y 0-300 across; the left +// side x 0-640 and the right side x 1280-1920, both y 300-1080. The player +// (x 780-1200, y 330-1080) is in none of them. Each cell is 30x30 (top) or +// 40x39 pixels (sides). +struct PlatformRegion{ + std::string_view name; + ImageFloatBox box; + size_t cols; + size_t rows; +}; + +const PlatformRegion PLATFORM_REGIONS[] = { + {"top", {0.0, 0.0, 1.0, 300.0 / 1080}, 64, 10}, + {"left", {0.0, 300.0 / 1080, 640.0 / 1920, 780.0 / 1080}, 16, 20}, + {"right", {1280.0 / 1920, 300.0 / 1080, 640.0 / 1920, 780.0 / 1080}, 16, 20}, +}; + +// The bounds of platform_matches(), from this code's cell averages on +// 1920x1080, against the platform reference: +// - the closest cutscene scene, the train's interior just before the +// platform: top 112.8, left 82.6, right 71.9; the next, city flyovers: +// top 134.7+, left 95.6+; +// - the reference against itself moved 16 px: top 42.2, left 20.8, +// right 21.2; 10% darker: top 24.6, left 19.9, right 16.4; +// - the platform after loading its autosave (camera turned a little): +// top 67.9-68.1, left 25.8-41.7, right 16.5-16.9. +// So the bounds sit between the loaded view and the train's interior: +// top 90 (23 below it), left 55 (27 below). The right side holds a walking +// NPC and the companion, so it only has to be roughly the platform (120: +// a menu or another place is 140+). +constexpr double PLATFORM_TOP_MAX = 90.0; +constexpr double PLATFORM_LEFT_MAX = 55.0; +constexpr double PLATFORM_RIGHT_MAX = 120.0; + +// The largest distance: every channel of every cell 255 apart, sqrt(3) * 255. +constexpr double MAX_PLATFORM_DISTANCE = 441.7; + +using Cells = std::vector; + +// `region` of `screen`, averaged into its cells, row by row. +Cells region_cells(const ImageViewRGB32& screen, const PlatformRegion& region){ + Cells cells; + cells.reserve(region.cols * region.rows); + const double cell_width = region.box.width / (double)region.cols; + const double cell_height = region.box.height / (double)region.rows; + for (size_t row = 0; row < region.rows; row++){ + for (size_t col = 0; col < region.cols; col++){ + const ImageFloatBox cell( + region.box.x + cell_width * (double)col, region.box.y + cell_height * (double)row, + cell_width, cell_height + ); + cells.push_back(image_stats(extract_box_reference(screen, cell)).average); + } + } + return cells; +} + +// The "rrggbb" at `offset` of `hex` as a color. Throws ParseException for +// a character that is not a hex digit. +FloatPixel hex_pixel(std::string_view hex, size_t offset){ + const auto channel = [&](size_t index){ + const std::string_view digits = hex.substr(offset + index * 2, 2); + int value = 0; + const auto [end, error] = std::from_chars(digits.data(), digits.data() + digits.size(), value, 16); + if (error != std::errc() || end != digits.data() + digits.size()){ + throw ParseException(std::format("plza-platform.json: \"{}\" is not a hex color channel.", digits)); + } + return (double)value; + }; + return FloatPixel(channel(0), channel(1), channel(2)); +} + +// The reference's cells of `region` from ReferenceData/plza-platform.json: +// "regions": {: {"cols": n, "rows": ["rrggbb...", ...]}}. +Cells reference_cells(const JsonObject& regions, const PlatformRegion& region){ + const std::string source = "plza-platform.json"; + const JsonObject& entry = regions.get_object_throw(std::string(region.name), source); + const JsonArray& rows = entry.get_array_throw("rows", source); + if ((size_t)entry.get_integer_throw("cols", source) != region.cols || rows.size() != region.rows){ + throw ParseException(std::format("{}: region {} is not {}x{} cells.", source, region.name, region.cols, region.rows)); + } + Cells cells; + for (const JsonValue& row : rows){ + const std::string& hex = row.to_string_throw(source); + if (hex.size() != region.cols * 6){ + throw ParseException(std::format("{}: a row of region {} is not {} colors.", source, region.name, region.cols)); + } + for (size_t col = 0; col < region.cols; col++){ + cells.push_back(hex_pixel(hex, col * 6)); + } + } + return cells; +} + +// The reference's cells, per PLATFORM_REGIONS entry. +const std::vector& platform_reference(){ + static const std::vector reference = []{ + for (const auto& [name, text] : autostory_reference_json()){ + if (name != "plza-platform"){ + continue; + } + const JsonValue json = parse_json(text); + const JsonObject& regions = json.to_object_throw("plza-platform.json").get_object_throw("regions"); + std::vector cells; + for (const PlatformRegion& region : PLATFORM_REGIONS){ + cells.push_back(reference_cells(regions, region)); + } + return cells; + } + throw FileException(nullptr, PA_CURRENT_FUNCTION, "The platform reference is not built in.", "plza-platform.json"); + }(); + return reference; +} + +double cells_rmsd(const Cells& reference, const Cells& cells){ + double sum = 0; + for (size_t index = 0; index < cells.size(); index++){ + const double r = reference[index].r - cells[index].r; + const double g = reference[index].g - cells[index].g; + const double b = reference[index].b - cells[index].b; + sum += r * r + g * g + b * b; + } + return std::sqrt(sum / (double)cells.size()); +} + +} // namespace + + +PlatformDistances platform_distances(const ImageViewRGB32& screen){ + const std::vector& reference = platform_reference(); + double distance[std::size(PLATFORM_REGIONS)] = {}; + for (size_t index = 0; index < std::size(PLATFORM_REGIONS); index++){ + distance[index] = cells_rmsd(reference[index], region_cells(screen, PLATFORM_REGIONS[index])); + } + return PlatformDistances{.top = distance[0], .left = distance[1], .right = distance[2]}; +} + +bool platform_matches(const PlatformDistances& distances){ + return distances.top <= PLATFORM_TOP_MAX + && distances.left <= PLATFORM_LEFT_MAX + && distances.right <= PLATFORM_RIGHT_MAX; +} + +ImageRGB32 platform_reference_image(size_t width, size_t height){ + ImageRGB32 image(width, height); + image.fill(0xff000000); + const std::vector& reference = platform_reference(); + for (size_t index = 0; index < std::size(PLATFORM_REGIONS); index++){ + const PlatformRegion& region = PLATFORM_REGIONS[index]; + const ImagePixelBox box = floatbox_to_pixelbox(width, height, region.box); + for (size_t y = box.min_y; y < box.max_y; y++){ + for (size_t x = box.min_x; x < box.max_x; x++){ + const size_t col = (x - box.min_x) * region.cols / (box.max_x - box.min_x); + const size_t row = (y - box.min_y) * region.rows / (box.max_y - box.min_y); + const FloatPixel& cell = reference[index][row * region.cols + col]; + image.pixel(x, y) = (uint32_t)Color((uint8_t)std::lround(cell.r), (uint8_t)std::lround(cell.g), (uint8_t)std::lround(cell.b)); + } + } + } + return image; +} + +std::string to_string(const PlatformDistances& distances){ + return std::format("top {:.1f}, left {:.1f}, right {:.1f}", distances.top, distances.left, distances.right); +} + + +PlatformArrivalDetector::PlatformArrivalDetector(Color color) + : m_color(color) +{} + +void PlatformArrivalDetector::make_overlays(VideoOverlaySet& items) const{ + for (const PlatformRegion& region : PLATFORM_REGIONS){ + items.add(m_color, region.box); + } +} + +bool PlatformArrivalDetector::detect(const ImageViewRGB32& screen){ + if (screen.width() == 0 || screen.height() == 0){ + // No frame: as far from the platform as a frame can be. + m_last = PlatformDistances{.top = MAX_PLATFORM_DISTANCE, .left = MAX_PLATFORM_DISTANCE, .right = MAX_PLATFORM_DISTANCE}; + return false; + } + m_last = platform_distances(screen); + return platform_matches(m_last); +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h new file mode 100644 index 0000000000..d8635ccc5e --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h @@ -0,0 +1,83 @@ +/* PokemonLZA AutoStory - Platform arrival detector + * + * From: https://github.com/PokemonAutomation/ + * + * The station platform once the new game's opening cutscene ends and the + * player has control (PlatformArrivalDetector): the frame's top and sides + * against a reference of that first frame, with the player cropped out. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryPlatformArrivalDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryPlatformArrivalDetector_H + +#include +#include +#include +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The checked-in reference files, ReferenceData/*.json, built into the +// program (cmake/EmbedTextFiles.cmake): (name, text), by name. +// plza-platform holds the platform reference's cell colors per region. +const std::vector>& autostory_reference_json(); + + +// How far a frame's regions are from the platform reference: each region +// is averaged into cells, and the distance is the RMSD over its cells of +// the summed squared channel differences (0 = the same, at most 441). +struct PlatformDistances{ + double top; + double left; + double right; +}; + +// The regions of a frame against the platform reference. +PlatformDistances platform_distances(const ImageViewRGB32& screen); + +// Whether `distances` are the platform: top and left within their bounds, +// and the right side within a looser one (the companion and the NPCs there +// move). The bounds and the measurements behind them are in the .cpp. +bool platform_matches(const PlatformDistances& distances); + +// The reference as a `width` x `height` frame: each cell painted its +// color, the player's area black. A frame the detector sees as the +// platform, for tests. +ImageRGB32 platform_reference_image(size_t width, size_t height); + +// "top 12.3, left 4.5, right 30.1", for the log. +std::string to_string(const PlatformDistances& distances); + + +// The station platform with free control, right after the opening +// cutscene, at the default camera. +class PlatformArrivalDetector : public StaticScreenDetector{ +public: + explicit PlatformArrivalDetector(Color color = COLOR_CYAN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + + // The distances of the last detect(). + const PlatformDistances& last_distances() const{ + return m_last; + } + +private: + Color m_color; + PlatformDistances m_last{.top = 0, .left = 0, .right = 0}; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.cpp new file mode 100644 index 0000000000..66ddbf9da8 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.cpp @@ -0,0 +1,52 @@ +/* PokemonLZA AutoStory - Skip prompt detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStorySkipPromptDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The badge on 1920x1080: the (+) icon, a white disk with a dark plus, +// at x 1564-1586, y 128-150 (its box 8 px wider all round); the text +// x 1630-1875, y 122-158; the strip under the text x 1640-1880, y 161-169. +SkipPromptDetector::SkipPromptDetector(Color color) + : m_color(color) + , m_icon(color, ButtonType::ButtonPlus, ImageFloatBox(1556.0 / 1920, 120.0 / 1080, 38.0 / 1920, 38.0 / 1080)) + , m_text(1630.0 / 1920, 122.0 / 1080, 245.0 / 1920, 36.0 / 1080) + , m_strip(1640.0 / 1920, 161.0 / 1080, 240.0 / 1920, 8.0 / 1080) +{} + +void SkipPromptDetector::make_overlays(VideoOverlaySet& items) const{ + m_icon.make_overlays(items); + items.add(m_color, m_text); + items.add(m_color, m_strip); +} + +// On the badge the text is 24% white and the strip is dark and even +// (average 44, deviation 2). The text and strip tests stay beside the +// icon's template match: the opening's white flash has the strip at 220. +bool SkipPromptDetector::detect(const ImageViewRGB32& screen){ + if (!m_icon.detect(screen)){ + return false; + } + const double text = white_share(screen, m_text); + if (text < 0.10 || text > 0.45){ + return false; + } + const ImageStats strip = image_stats(extract_box_reference(screen, m_strip)); + return strip.average.sum() < 3 * 100.0 && strip.stddev.sum() < 3 * 15.0; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.h new file mode 100644 index 0000000000..b49bcc92b9 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.h @@ -0,0 +1,44 @@ +/* PokemonLZA AutoStory - Skip prompt detector + * + * From: https://github.com/PokemonAutomation/ + * + * The opening cutscene's "(+) Hold button to skip" badge. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStorySkipPromptDetector_H +#define PokemonAutomation_PokemonLZA_AutoStorySkipPromptDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" +#include "VideoGames/PokemonLZA/Inference/PokemonLZA_ButtonDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The cutscene's "(+) Hold button to skip" badge at the top right: the +// (+) icon (ButtonDetector's ButtonType::ButtonPlus), white text on the +// badge, and the badge's dark, even strip under the text. +class SkipPromptDetector : public StaticScreenDetector{ +public: + explicit SkipPromptDetector(Color color = COLOR_MAGENTA); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ButtonDetector m_icon; + ImageFloatBox m_text; + ImageFloatBox m_strip; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/ReferenceData/plza-platform.json b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/ReferenceData/plza-platform.json new file mode 100644 index 0000000000..1d59ee8577 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/ReferenceData/plza-platform.json @@ -0,0 +1,70 @@ +{ + "source": "the station platform right after the opening cutscene, default camera, 1920x1080", + "regions": { + "top": { + "cols": 64, + "rows": [ + "96dbf56fa2a7364b27577a55a2e8ffa8edffadf1ffb2f4ffb6f6ffb3f7ffb0f6ff9ee2f090d3e6a6edfca8f1ffa6f0ffa3eeff73a7bb234156476f8774afc846698241657d21415a243a4047658321436244668048687e4a6b8248698047687e4869804c6e854b6d8548697f45647a45667c39586c2c4557213a4c26466726475f1f3b554756665e65736d70797679817c7d856c70786e71796d7381777d8f74798773767e6f727963676f6c717c61656d5b606b61646b676a72666a7254585e", + "719ba7374b263245245576645c816a85b6a297cec6c0faffc4fdffc1fcffbdfbffb9faffadeef780b7ca86bfd09cdbe9aff3ff74a8b924455b466a7da7eeff6797ab2b475b25486022384139566e36587441608046677d3c5b6f3753673450632e495b2d485a2d485a2f4a5d355163355164294456243d4e1f3a51315d8b3a65911e3d5554616f6c7078797b8274767e797b826569716b6f79747a8c6e7586767b8b70747e6d707873757d6a6e775e626a5b606955596065697362677152565c", + "2d3849343e4c39474636442f4057323f553396c6b9c8feffc9ffffc9ffffc8ffffc2fafca5dbe37eabb76e97a587bac8afedf67fb3bf25475b486e7fb4f5ff82bbce385e782f546e223843324e63365a7745617f253f54314d6233516838576e33557030516b30526b33516738566b2942562840531f394b2e567e4c79ae5f769121435e54606d73767f75787f6c707a686c756c6f767479875d636d4f58666870816b707a77798183848b737781676a725d626c6669706f727a686b73545961", + "364f742a36482c3848313d4e3743504b5e5dacd7dcccfeffd1ffffd3ffffd3ffffcdfdfdb1dfe391bac3b4e6eaccffffb9edf27daab7224555456876c2fbffc0f9fdb0e9f16596a9223d472945542e516d3a5875213c513350663456723d607c46687f4a6c84496a813e627a2a4a62274255223a4c2546624671a4596d865e70885563735b626e6367706b6f78777f93777d8d676b746e73814a50583f464d4d545e6a6d7773757d84848b787c85676b746569737d7e8371757d60646c555a62", + "7da9f775a6f85b86ca4566943648622f3b4b313e524e5e6d83a0a9bce2e5d6ffffc9f1f350677192b5bbd8ffffdaffffd7ffff96c2c922414e43626ecdfeffd2ffffd1ffffc1f4f629455026404b29496237516326405726455c2e4a5b2f4b5c2c47592b47592c47582d48592e475927455a223b4f39536e4860773e5a72405b736d7179676c756c6f78767a856067766f76886c717c646a7442494e3d434a3c434b70747e7878807a7c837a7d8773757d666a7475777e6e7077696d744f545d", + "9cb8e59ebceba0c0f49bbffb8cb8fb7aa6e75b6281434559313c4b313f505366748da9b1aed1d594b2b6dfffffe1ffffdfffffa1c7cc2543523e5b67b1d9e3bdebf4b3efff97e2fe45768f28485a284b622c455229475f2c475d38576d2d4d6432526934546b3150662d4b61233b4d243f54304d664353626268725f68715c656e6a6f796a6e776e7179717581484e56565d6a757b89565c6341474ea4a7a9c7c9cbd9dadcd9dadcd9dadbdadbddd8d8da6e727b71747b7879806d70774d525d", + "9eb9e19fbae3a2bde6aac5ebacc8eeaecaf39fafd36b56606e5d705f67845470963b4d66323e4f47576494acb2a9c3c8defefea4c9d025414d39566290deff74c3ee70bae478cdf761a3c433556a3d6784243e4a35546a233d5235556c34546b305168315269325168284255254256374b5e546578495764767881787a8275777f6f727b6c70796b6f78676c7841484d4c535e757b8b545a62434950cfd0d2d2d3d6d2d3d7cbced2d5d7dbc6c8ccf7f7f76f737c7f808787878c63676e4f555f", + "9ab5e0a3bee4a6c0e7a2bde6a5c0e7a7c2e8aabfe46b54516c56546d5a5ea6c4f59fc3fa8aaee56483b24b5e783341544661734f7678264550344f5b70b8d876c0e774bee56cadd44c6f863e657966afd72a48562d49582742562b495f355870365a743456702f4e6528455a3f55673a4f5d75777f676b747a7c838281887d7e8674777f72767e676b746167723f4e5c4a5b70696f7c535963464c56d0d1d4c7c9ced7dcdeced1d5dadde0e3e5e6efeff06468707e808786868c64686f4c515c", + "9bb4dd9eb8df9fbbe1a3bce3a9c2e6a4bfe4a9c1e66a53506b54516a53519db6dd7e8a9e79889da6c3f18eb6f18ab5f57595c75a759425424c304952507c6c5ea09e7fd6ff7dd5ff5077903c61727ad1fc487c972b47532643532d4e645490b565aed65a98be47799a364f62495c69626a73787a8070737c7e7e8585848b626e7c707580787b8371747c606672223f5c3e50664e5c6c525762474d59b0b2b7d1d3d5ced3d8dfe0e1d2d5d7dcdddedadbdc575c6462666c696c735f636b494f5c", + "6b81a37088aa758db087a0c3a1b9dd9fb9df9ab2d54c4d5e524d5a584f5793abd28086927e756b9bb8e68cacd980a7df96b3e08397b3233b442b404a6882a84f66834a6e8466aacf4f77903b5e6e69add94a799540687c2946553c647e7fb6ca8ecbdf7ab9d472c2eb4c62726e747c7f80866f737b6f737b7e7f8586858b7d838c6f77846d707b7679815d66715870824c5a6758616c515761454c574147524b515c4a566c585b694c535e6f727b6d70795459626c6f77686c7460656d494f5c" + ] + }, + "left": { + "cols": 16, + "rows": [ + "6c7e92596a7d49618167604c6c685a474c443b413f3c4959808c9a7c92b17b95bc7ea2d38da8d54b5a64263a418caee2", + "6b809a4d5d6b486488645d48655e473841442d353f31384067604e6b7b8553667e799cc9708296314a5c273b41667993", + "768aa24856644a668a605a465f5a44323b402b343a2c353e645f4f5f6c7252667f7496c16f849a3a596c244251576c82", + "7692b5596d834f6e945b564452504329343c28323a353d455151496f776f5469817292b9626d8036536720373d4a5f77", + "6b748351535d506d915651414d4c433638432c313b3038454649465c5b624d63806f8eb16d7f9a4b68741f3d4e5c7a9a", + "58555f50505e4f6c8f524e414d4c42282a3222262e3d3c444b59697188a17594b86a8db780a1cb40576b1d2c30547293", + "5455605a5d6c4863895e666b626c762e37442b34403c3a425a759680a0c87e9fc8678bb77b9dc843647d1e425341596a", + "667e9d6f8aad81a0c683a2c97a9ac02e3a4827333d2a3240708fb57a9ac57698c457759a4c65802d434f1f323634485d", + "84a5ca81a3c880a2c77c9ec57899c15c7a9e486e9f252e396486b1567499455e77485f73475d722d425220313931455f", + "7fa1c87c9fc77a9dc6779ac47395c05f7ca3547cb8456896425b74485e72495f74405670364c6f354c70364c714b5c7a", + "7a9cc5789ac47698c37295c05c7ea6435d7b43596e495f74495f73445a71395071344a6d344a6e344b6f526280828898", + "7497c17094bf5a79a0445d7942586c475d704a6075485e743c526f344a6e344c70344b70344b6f58668284899a838999", + "506e903f5871445a6e4a6075495f73495e72405671354c6e334a6d344c71344b70354c705f6c85828898868b9c838898", + "44596d495f734a6174495f73435970384f70334a6e32496d334b6f344c72364d72647086828797868c9d858b9c70798e", + "465c70465c70455a6f3b516e344b6f344b6f32496c344c71334a6f394f716e778c848998848a9983889778809370798d", + "465c6f3f556e33496b334b6f334b6f32496d32486c31486b3a4d6d747c8e848a988187948186947d83926b74887f8492", + "3247642e43632f456531476a31466930466830476a40526f777d8c8388938489957f84907e8390677083797f8d808593", + "2c405d2b405d2d42612f45663046683046674958747d828f878b95858994858a958387926f7685737a888085907f838e", + "2a3c572a3d592c405e2e42612f45654b5a747f8490858992848890858891848892747986666e7f7e838d7f838d7f848e", + "293b55293b55293b562e415e545f7381858e878a92868a9283868f8488917a7e89626a7a797d8882868e7f838c7d828b" + ] + }, + "right": { + "cols": 16, + "rows": [ + "666c785d636e62666f474d59545963595e68464c573e4450485364494e60636872797a81656a7387868c6e7178505663", + "686e796b6e78676b74464c57555a64585e67454b553c424d4c4e5d4b4048686b74797b8154596271747c6a6e754d535f", + "5352526c6f785f646e464c58555a64565c65464c563b414c5559644e332c4d4a4d70737a5a5f676b6f76656a724b515f", + "5b5e57656a73555c674248565e626c585d664349533f42496d6b6b83623e747367676b73575c646c6e75646871484e5c", + "515b63656a75545d6d404653565d6b4e54603f4651473c3e958b63af98357e7235616671484d5565686e686c74454c59", + "4050694c5769485367383f4c545b694c525d3b434d403a3fb4a25bb0972e9e8a325b626d4b515c595f68585e65424958", + "3f51713f52714051714052724c597240495a3440522c2c3281765081712c836e2e6469714f555f60666e555b66434a59", + "3f52733f52734052744052734153744053744d5d80515c772f333e12192052464860656e5056605b616a505761434a58", + "3e52733e52744053753f52743f5273405374415475435473283344121b264d545f4f56604a515d5259624e545f404855", + "3e52743e52744054773f52744154773f53744154764255772a374c151e293f48543d444b424d5f4b5462474d5a3b4251", + "3e53773d52753e52743f53774054774256793f5375415476303b50171e278a74407e735f474748435678415371415271", + "3d52753f54783f54783e53763f53764054774054765061825e505f6b5a61615e59ad83334747423f4e6845587a425475", + "3e54793c52753c52753e54783e54773e53763e53764055794155784255784f5a6f71644f8b7243445571425577415374", + "3e53783d52763c51753b51753d53763e53773e53763f54763e537641557942567943577a40547643577a425678415475", + "384e713c51753c51743c50733c50723f53773e53773e52753e52753e51744054784154784053764054764052743f5071", + "3a4e6f374c6d394d6f3b4f723c50733b4e703c4f713d50723c50723b4f703c4f703e51734053763e50713c4e6f3e4f6f", + "4f5d76374b6c374a6c374a6c374a6b374a6a394c6c3b4e703a4d6e3a4d6d3a4d6d3b4e6e3d4e6f3c4e6d394b69384966", + "6d75863d4e6c364869374969364767364867374968384a6a384a6a3a4c6c3b4d6d3c4d6d3a4c6a3b4c6b384865374763", + "7b808c576279344563354766354766344665354766394a69354765374967374967394b6938496637486436466135455e", + "7b7f8b707683404f6835466535476635476535466435476533456136476536476435466234445f32425b32425b324158" + ] + } + } +} diff --git a/SerialPrograms/cmake/EmbedTextFiles.cmake b/SerialPrograms/cmake/EmbedTextFiles.cmake new file mode 100644 index 0000000000..943c6f25ab --- /dev/null +++ b/SerialPrograms/cmake/EmbedTextFiles.cmake @@ -0,0 +1,88 @@ +# pa_embed_text_files(): builds the text of data files into the program. +# +# pa_embed_text_files(
) +# +# Reads every /*.json (relative to the calling CMakeLists.txt) and +# writes a C++ file to the build folder, compiled into , that defines +# in +# +# const std::vector>& (); +# +# returning (file name without ".json", file text) for each file, sorted by +# name.
, an include path, declares that function. The files stay +# plain files in the repo; nothing generated is committed. Adding, removing or +# changing a file reruns CMake at the next build. +# +# Each text is split into raw string literals of at most 16000 bytes, joined +# at run time, so no literal exceeds MSVC's limit of 16380 bytes. + +function(pa_embed_text_files TARGET FOLDER HEADER NAMESPACE FUNCTION) + # Canonicalize once so resource verification and embedding watch the same + # paths. Mixed absolute/../ aliases can produce duplicate Ninja outputs. + file(REAL_PATH "${FOLDER}" MAP_DIRECTORY BASE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + file(GLOB FILES CONFIGURE_DEPENDS "${MAP_DIRECTORY}/*.json") + if(NOT FILES) + message(FATAL_ERROR "pa_embed_text_files(): no .json files in ${FOLDER}.") + endif() + list(SORT FILES) + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${FILES}) + + set(ENTRIES "") + foreach(FILE IN LISTS FILES) + get_filename_component(NAME "${FILE}" NAME_WLE) + file(READ "${FILE}" TEXT) + string(FIND "${TEXT}" ")json\"" CLOSE) + string(LENGTH "${TEXT}" LENGTH) + if(NOT CLOSE EQUAL -1 OR LENGTH EQUAL 0) + message(FATAL_ERROR "pa_embed_text_files(): ${FILE} is empty or contains )json\".") + endif() + set(PARTS "") + set(START 0) + while(START LESS LENGTH) + string(SUBSTRING "${TEXT}" ${START} 16000 PART) + string(APPEND PARTS "\n R\"json(${PART})json\",") + math(EXPR START "${START} + 16000") + endwhile() + string(APPEND ENTRIES " {\"${NAME}\", join({${PARTS}\n })},\n") + endforeach() + + set(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/EmbeddedText/${FUNCTION}.cpp") + file(WRITE "${OUTPUT}.new" +"// Generated by SerialPrograms/cmake/EmbedTextFiles.cmake from +// ${FOLDER}/*.json. Do not edit. + +#include +#include +#include +#include +#include \"${HEADER}\" + +namespace ${NAMESPACE}{ + + +namespace{ + +std::string join(std::initializer_list parts){ + std::string text; + for (const char* part : parts){ + text += part; + } + return text; +} + +} + + +const std::vector>& ${FUNCTION}(){ + static const std::vector> FILES{ +${ENTRIES} }; + return FILES; +} + + +} +") + # Copied only if it changed, so an unchanged file isn't recompiled. + configure_file("${OUTPUT}.new" "${OUTPUT}" COPYONLY) + target_sources(${TARGET} PRIVATE "${OUTPUT}") +endfunction() diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 53f24be4cb..c4fc6a09cc 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -2367,6 +2367,28 @@ file(GLOB LIBRARY_SOURCES Source/VideoGames/PokemonLZA/PokemonLZA_Settings.h Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp Source/VideoGames/PokemonLZA/PokemonLZA_Tests.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryCaughtMarkReader.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryCaughtMarkReader.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMinimapZoneDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMinimapZoneDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.h Source/VideoGames/PokemonLZA/Programs/Farming/PokemonLZA_DonutMaker.cpp Source/VideoGames/PokemonLZA/Programs/Farming/PokemonLZA_DonutMaker.h Source/VideoGames/PokemonLZA/Programs/Farming/PokemonLZA_FriendshipFarmer.cpp