From 9a4f75479867f198b6a0a859cc93c46113cf1c3d Mon Sep 17 00:00:00 2001 From: dayfine Date: Wed, 11 Nov 2026 22:25:56 -0800 Subject: [PATCH 1/5] PokemonLZA: AutoStory: detectors and their tests --- SerialPrograms/CMakeLists.txt | 2 + .../PokemonLZA/PokemonLZA_Tests.cpp | 2 + ...kemonLZA_AutoStoryAdvanceArrowDetector.cpp | 217 +++++++ ...PokemonLZA_AutoStoryAdvanceArrowDetector.h | 46 ++ .../PokemonLZA_AutoStoryAutosaveDetector.cpp | 83 +++ .../PokemonLZA_AutoStoryAutosaveDetector.h | 46 ++ ...kemonLZA_AutoStoryControlsHintDetector.cpp | 58 ++ ...PokemonLZA_AutoStoryControlsHintDetector.h | 47 ++ .../PokemonLZA_AutoStoryDetectors_Tests.cpp | 542 ++++++++++++++++++ .../PokemonLZA_AutoStoryDetectors_Tests.h | 28 + ...LZA_AutoStoryMainMissionBannerDetector.cpp | 77 +++ ...onLZA_AutoStoryMainMissionBannerDetector.h | 52 ++ ...onLZA_AutoStoryMissionCompleteDetector.cpp | 58 ++ ...emonLZA_AutoStoryMissionCompleteDetector.h | 49 ++ .../PokemonLZA_AutoStoryPixelShares.cpp | 61 ++ .../PokemonLZA_AutoStoryPixelShares.h | 72 +++ ...onLZA_AutoStoryPlatformArrivalDetector.cpp | 204 +++++++ ...emonLZA_AutoStoryPlatformArrivalDetector.h | 83 +++ ...PokemonLZA_AutoStorySkipPromptDetector.cpp | 53 ++ .../PokemonLZA_AutoStorySkipPromptDetector.h | 43 ++ .../ReferenceData/plza-platform.json | 70 +++ SerialPrograms/cmake/SourceFiles.cmake | 18 + 22 files changed, 1911 insertions(+) create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.cpp create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.h create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.cpp create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.h create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.cpp create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.h create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.cpp create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.h create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.cpp create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.h create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.cpp create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.h create mode 100644 SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/ReferenceData/plza-platform.json diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index d135188804..f23153a8f7 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -228,6 +228,8 @@ add_library(SerialProgramsLib STATIC ${CORE_LIBRARY_SOURCES} ${LIBRARY_SOURCES}) # Data files built into the program (see cmake/EmbedTextFiles.cmake). include(cmake/EmbedTextFiles.cmake) +pa_embed_text_files(SerialProgramsLib Source/VideoGames/PokemonLZA/Programs/AutoStory/ReferenceData + VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h PokemonAutomation::NintendoSwitch::PokemonLZA autostory_reference_json) # Create binary target diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp index ead2ea2d67..cea137acd2 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp @@ -19,6 +19,7 @@ #include "Inference/Map/PokemonLZA_MapIconDetector.h" #include "Inference/PokemonLZA_OverworldPartySelectionDetector.h" #include "Inference/PokemonLZA_SelectionArrowDetector.h" +#include "Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h" namespace PokemonAutomation{ namespace NintendoSwitch{ @@ -41,6 +42,7 @@ void add_tests(UnitTestDatabase& database){ add_tests_MapIconDetector(database); add_tests_OverworldPartySelectionDetector(database); add_tests_SelectionArrowDetector(database); + add_tests_AutoStoryDetectors(database); } diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp new file mode 100644 index 0000000000..933f849543 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp @@ -0,0 +1,217 @@ +/* PokemonLZA AutoStory - Advance arrow detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStoryAdvanceArrowDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +// A 4-connected blob of a mask: its bounds and its pixel count. +struct Blob{ + size_t min_x; + size_t min_y; + size_t max_x; + size_t max_y; + size_t count = 0; +}; + +// An area's mask split into blobs: each pixel's blob label (0 outside the +// mask; blob i has label i + 1). +struct BlobImage{ + size_t width = 0; + size_t height = 0; + std::vector labels; + std::vector blobs; + + uint32_t label(size_t x, size_t y) const{ + return labels[y * width + x]; + } +}; + +// A mask pixel no blob has taken yet. +constexpr uint32_t UNLABELLED = UINT32_MAX; + +// Labels the blob of mask pixels 4-connected to (x, y). +void flood_blob(size_t x, size_t y, BlobImage& image){ + const uint32_t label = (uint32_t)image.blobs.size() + 1; + Blob blob{x, y, x, y, 0}; + std::vector> stack{{x, y}}; + image.labels[y * image.width + x] = label; + const auto take = [&](size_t nx, size_t ny){ + uint32_t& here = image.labels[ny * image.width + nx]; + if (here == UNLABELLED){ + here = label; + stack.emplace_back(nx, ny); + } + }; + while (!stack.empty()){ + const auto [cx, cy] = stack.back(); + stack.pop_back(); + blob.count++; + blob.min_x = std::min(blob.min_x, cx); + blob.max_x = std::max(blob.max_x, cx); + blob.min_y = std::min(blob.min_y, cy); + blob.max_y = std::max(blob.max_y, cy); + if (cx > 0){ + take(cx - 1, cy); + } + if (cx + 1 < image.width){ + take(cx + 1, cy); + } + if (cy > 0){ + take(cx, cy - 1); + } + if (cy + 1 < image.height){ + take(cx, cy + 1); + } + } + image.blobs.push_back(blob); +} + +// The pixels of `area` that `in`, split into blobs. +template +BlobImage find_blobs(const ImageViewRGB32& area, In&& in){ + BlobImage image; + image.width = area.width(); + image.height = area.height(); + image.labels.assign(image.width * image.height, 0); + for (size_t y = 0; y < image.height; y++){ + for (size_t x = 0; x < image.width; x++){ + if (in(Color(area.pixel(x, y)))){ + image.labels[y * image.width + x] = UNLABELLED; + } + } + } + for (size_t y = 0; y < image.height; y++){ + for (size_t x = 0; x < image.width; x++){ + if (image.label(x, y) == UNLABELLED){ + flood_blob(x, y, image); + } + } + } + return image; +} + +// How well blob `label` fills a V across its bounds: its overlap with the +// V over their union. Each arm runs from 10% in at the top to the middle +// at the bottom, 22% of the width across. +double v_match(const BlobImage& image, uint32_t label){ + const Blob& blob = image.blobs[label - 1]; + const double width = double(blob.max_x - blob.min_x + 1); + const double height = double(blob.max_y - blob.min_y + 1); + size_t both = 0; + size_t either = 0; + for (size_t y = blob.min_y; y <= blob.max_y; y++){ + const double v = (double(y - blob.min_y) + 0.5) / height; + for (size_t x = blob.min_x; x <= blob.max_x; x++){ + const double u = (double(x - blob.min_x) + 0.5) / width; + const bool in_v = std::abs(std::abs(u - 0.5) - (0.40 * (1 - v) + 0.03)) <= 0.11; + const bool in_blob = image.label(x, y) == label; + if (in_v && in_blob){ + both++; + } + if (in_v || in_blob){ + either++; + } + } + } + return either == 0 ? 0 : double(both) / double(either); +} + +// The mask pixels of other blobs within `margin` of blob `label`'s +// bounds, per pixel of the blob. +double stray_share(const BlobImage& image, uint32_t label, size_t margin){ + const Blob& blob = image.blobs[label - 1]; + const size_t min_x = blob.min_x > margin ? blob.min_x - margin : 0; + const size_t min_y = blob.min_y > margin ? blob.min_y - margin : 0; + const size_t max_x = std::min(blob.max_x + margin, image.width - 1); + const size_t max_y = std::min(blob.max_y + margin, image.height - 1); + size_t stray = 0; + for (size_t y = min_y; y <= max_y; y++){ + for (size_t x = min_x; x <= max_x; x++){ + const uint32_t here = image.label(x, y); + if (here != 0 && here != label){ + stray++; + } + } + } + return double(stray) / double(blob.count); +} + +// Whether blob `label` is the chevron, on a screen `scale` times 1080p: +// 32x19 measured, 30-33 x 18-20 on every frame with it. +bool chevron_blob(const BlobImage& image, uint32_t label, double scale){ + const Blob& blob = image.blobs[label - 1]; + const double width = double(blob.max_x - blob.min_x + 1) / scale; + const double height = double(blob.max_y - blob.min_y + 1) / scale; + if (width < 26 || width > 40 || height < 15 || height > 24){ + return false; + } + const double aspect = width / height; + if (aspect < 1.4 || aspect > 2.1){ + return false; + } + return stray_share(image, label, size_t(4 * scale + 0.5)) <= 0.05 && v_match(image, label) >= 0.45; +} + +// Whether a blob of the pixels of `area` that `in` is the chevron. +template +bool chevron_in(const ImageViewRGB32& area, double scale, In&& in){ + const BlobImage image = find_blobs(area, in); + for (uint32_t label = 1; label <= image.blobs.size(); label++){ + if (chevron_blob(image, label, scale)){ + return true; + } + } + return false; +} + +} // namespace + + +AdvanceArrowDetector::AdvanceArrowDetector(Color color) + : m_color(color) + , m_area(1370.0 / 1920, 930.0 / 1080, 110.0 / 1920, 120.0 / 1080) +{} + +void AdvanceArrowDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_area); +} + +// Measured on 1920x1080 (console B, 2026-11-10; 01.01's runs 1 and 2, the +// lab's and the user's opening frames, 459 frames): the chevron's 35 +// frames (the ad's subtitles, white boxes, the opening's blue box) fill +// the V 0.53-0.70 with at most 1% stray pixels. Of the rest, the closest +// in size and shape is run 1's 0033 (a dark flyover blade, V 0.49 but +// 16% stray); the closest with nothing around it fill the V 0.43 or less +// and are out of the chevron's size. +bool AdvanceArrowDetector::detect(const ImageViewRGB32& screen){ + const double scale = double(screen.height()) / 1080; + const ImageViewRGB32 area = extract_box_reference(screen, m_area); + const bool white = chevron_in(area, scale, [](Color pixel){ + return pixel.red() >= 235 && pixel.green() >= 235 && pixel.blue() >= 235; + }); + return white || chevron_in(area, scale, [](Color pixel){ + return pixel.red() <= 110 && pixel.green() <= 110 && pixel.blue() <= 110; + }); +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h new file mode 100644 index 0000000000..8debae4975 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h @@ -0,0 +1,46 @@ +/* PokemonLZA AutoStory - Advance arrow detector + * + * From: https://github.com/PokemonAutomation/ + * + * The "press A to go on" chevron at the bottom right of a dialogue box or + * of the boxless subtitles of 01.01's ad. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryAdvanceArrowDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryAdvanceArrowDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The "press A to go on" chevron (a V about 32x19 on 1920x1080) at the +// right end of a dialogue box and of boxless subtitles: dark on a white +// box, white on a blue box and on the ad's subtitles. Found by shape in +// x 1370-1480, y 930-1050: a blob of white (every channel 235+) or dark +// (every channel 110 or under) pixels the size of the chevron, filling a +// V, with no other pixel of its colour around it. Text is not read, so +// any language matches. +class AdvanceArrowDetector : public StaticScreenDetector{ +public: + explicit AdvanceArrowDetector(Color color = COLOR_GREEN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_area; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.cpp new file mode 100644 index 0000000000..e76391ea53 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.cpp @@ -0,0 +1,83 @@ +/* PokemonLZA AutoStory - Autosave detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStoryAutosaveDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +// The icon, and the strips around it (left, above, below) clear of white +// and cyan, `dx` pixels (of 1920) right of where the Chinese badge has +// them. +bool autosave_icon_at( + const ImageViewRGB32& screen, const ImageFloatBox& icon, + std::initializer_list around, double dx +){ + const ImageFloatBox icon_here = shifted(icon, dx); + const double white = white_share(screen, icon_here); + if (white < 0.08 || white > 0.40 || cyan_share(screen, icon_here) < 0.25){ + return false; + } + for (const ImageFloatBox& box : around){ + const ImageFloatBox here = shifted(box, dx); + if (white_share(screen, here) >= 0.05 || cyan_share(screen, here) >= 0.05){ + return false; + } + } + return true; +} + +} // namespace + + +AutosaveDetector::AutosaveDetector(Color color) + : m_color(color) + , m_icon(1603.0 / 1920, 73.0 / 1080, 27.0 / 1920, 31.0 / 1080) + , m_left(1570.0 / 1920, 73.0 / 1080, 28.0 / 1920, 31.0 / 1080) + , m_above(1603.0 / 1920, 56.0 / 1080, 27.0 / 1920, 14.0 / 1080) + , m_below(1603.0 / 1920, 108.0 / 1080, 27.0 / 1920, 14.0 / 1080) +{} + +void AutosaveDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_icon); + items.add(m_color, m_left); + items.add(m_color, m_above); + items.add(m_color, m_below); +} + +// Measured on the user's run (console B, 2026-11-10, frames 1.5 s apart): +// the badge's one frame (rec1_u132) has the icon 19% white and 48% cyan, +// nothing left of it. The only other frames over the icon bounds (u103, +// u104: a bright cutscene flash) are 70%+ white and 39%+ cyan left of it +// too; none of the opening's 56 lab frames come near. +// The badge is seen in Chinese only; with a shorter text (English +// "Saving...") a right-aligned badge moves the icon right, so the icon is +// looked for from where it is in Chinese to 240 px right of it. +// The 01.01 run (console B, 2026-11-10 13:00) took two flyover frames for +// the badge: 0033, a glowing blade at dx 40, and 0038, the sky at dx +// 152-172. Both glow on past the icon: the strips 14 px above and below it +// are 22%+ white or cyan there, 0% on u132; they must stay under 5%. +bool AutosaveDetector::detect(const ImageViewRGB32& screen){ + for (double dx = 0; dx <= 240; dx += 4){ + if (autosave_icon_at(screen, m_icon, {m_left, m_above, m_below}, dx)){ + return true; + } + } + return false; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.h new file mode 100644 index 0000000000..e18115915c --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.h @@ -0,0 +1,46 @@ +/* PokemonLZA AutoStory - Autosave detector + * + * From: https://github.com/PokemonAutomation/ + * + * The autosave badge, a pencil and "Saving...", which shows for 1-3 s + * after the game autosaves. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryAutosaveDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryAutosaveDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The autosave badge at the top right: a white pencil with a cyan glow at +// x 1603-1630, y 73-104 on 1920x1080, and nothing white or cyan just left +// of it, above it or below it. Only the icon is read, so any language +// matches. +class AutosaveDetector : public StaticScreenDetector{ +public: + explicit AutosaveDetector(Color color = COLOR_YELLOW); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_icon; + ImageFloatBox m_left; + ImageFloatBox m_above; + ImageFloatBox m_below; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.cpp new file mode 100644 index 0000000000..d1b545c486 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.cpp @@ -0,0 +1,58 @@ +/* PokemonLZA AutoStory - Controls hint detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStoryControlsHintDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Measured on 1920x1080 (console B, 2026-11-10): the card at x 946-971, +// y 868-901 with its spine at x 974-976; the "?" at x 953-965, y 876-894. +ControlsHintDetector::ControlsHintDetector(Color color) + : m_color(color) + , m_card(947.0 / 1920, 872.0 / 1080, 24.0 / 1920, 28.0 / 1080) + , m_glyph(955.0 / 1920, 879.0 / 1080, 8.0 / 1920, 7.0 / 1080) + , m_around{ + {928.0 / 1920, 872.0 / 1080, 14.0 / 1920, 28.0 / 1080}, + {980.0 / 1920, 872.0 / 1080, 14.0 / 1920, 28.0 / 1080}, + {945.0 / 1920, 856.0 / 1080, 30.0 / 1920, 8.0 / 1080}, + {945.0 / 1920, 906.0 / 1080, 30.0 / 1920, 8.0 / 1080}, + } +{} + +void ControlsHintDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_card); + for (const ImageFloatBox& box : m_around){ + items.add(m_color, box); + } +} + +// The hint's 13 frames (01.01 run 0001-0006, the user's u132-u138): the +// card 88%+ sky blue, the "?" at most 48%, nothing around it. Of the 376 +// frames without it, the closest are the 01.01 run's 0056 (the sky +// behind the girl: card 76%, "?" 100%, around 54%), 0060 (card 35%, +// around 27%) and the user's u126 (card 22%, around 32%). +bool ControlsHintDetector::detect(const ImageViewRGB32& screen){ + if (sky_blue_share(screen, m_card) < 0.80 || sky_blue_share(screen, m_glyph) > 0.70){ + return false; + } + for (const ImageFloatBox& box : m_around){ + if (sky_blue_share(screen, box) > 0.10){ + return false; + } + } + return true; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.h new file mode 100644 index 0000000000..5dfe675d07 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.h @@ -0,0 +1,47 @@ +/* PokemonLZA AutoStory - Controls hint detector + * + * From: https://github.com/PokemonAutomation/ + * + * The tutorial's controls hint under the player, a "?" card over + * "(L) Move" and "(R) Control Camera", which shows for ~10 s when the + * player gets control. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryControlsHintDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryControlsHintDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The controls hint the tutorial shows under the player when they get +// control: its "?" card, sky blue with a dark "?", at x 947-971, y 872-900 +// on 1920x1080, centred, with no sky blue around it. Only the card is +// read: the rows under it ("(L) Move", "(R) Control Camera") are centred +// too, so their icons move with the text's length. +class ControlsHintDetector : public StaticScreenDetector{ +public: + explicit ControlsHintDetector(Color color = COLOR_BLUE); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_card; + ImageFloatBox m_glyph; + ImageFloatBox m_around[4]; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp new file mode 100644 index 0000000000..9a74171631 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp @@ -0,0 +1,542 @@ +/* PokemonLZA AutoStory - Detectors Tests + * + * From: https://github.com/PokemonAutomation/ + * + * Synthetic frames always. The opening's frames are not in + * CommandLineTests yet; each set runs when its folder is in the + * environment and is skipped otherwise (console B, 1920x1080): + * - PLZA_OPENING_FRAMES: platform_ref.png and the lab's cutscene, + * 0001-*.png ... 0056-*.png; + * - PLZA_USER_OPENING_FRAMES: the user's run, rec1_u001.png ... + * rec1_u146.png, one per ~1.5 s, with the autosave badge (in Chinese); + * - PLZA_0101_RUN1_FRAMES: 01.01's first run (2026-11-10 13:00, in + * Chinese), 0001-*.png ... 0089-*.png: the platform with the controls + * hint, the walk to the plaza and the arrival scenes; + * - PLZA_0101_RUN2_FRAMES: 01.01's second run (2026-11-10 13:28, in + * Chinese), 0001-*.png ... 0115-*.png: the same, on to the ad's + * subtitles (0090-0115), where the run stopped; + * - PLZA_0101_RUN3_FRAMES: 01.01's third run (2026-11-10 13:49, in + * Chinese), 0001-*.png ... 0181-*.png: the same, through the ad and + * Pancham taking the bag, to the "MAIN MISSION 01" banner (0181); + * - PLZA_0101_RUN4_FRAMES: 01.01's fourth run (2026-11-10, in Chinese), + * 0001-*.png ... 0221-*.png, the "MAIN MISSION 01" banner on + * 0181-0189; + * - PLZA_0103_RUN2_FRAMES: 01.03's second run (2026-11-10, in Chinese), + * 0001-*.png ... 0139-*.png and mission01-complete-fullres.png: the + * "MAIN MISSION 01 COMPLETE" card (0045, 0046, the full-res frame), + * the "MAIN MISSION 02" banner (0133-0139). + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "PokemonLZA_AutoStoryAdvanceArrowDetector.h" +#include "PokemonLZA_AutoStoryAutosaveDetector.h" +#include "PokemonLZA_AutoStoryControlsHintDetector.h" +#include "PokemonLZA_AutoStoryDetectors_Tests.h" +#include "PokemonLZA_AutoStoryMainMissionBannerDetector.h" +#include "PokemonLZA_AutoStoryMissionCompleteDetector.h" +#include "PokemonLZA_AutoStoryPlatformArrivalDetector.h" +#include "PokemonLZA_AutoStorySkipPromptDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + +namespace{ + +constexpr size_t WIDTH = 1920; +constexpr size_t HEIGHT = 1080; + +void fill_box(size_t min_x, size_t min_y, size_t max_x, size_t max_y, uint32_t color, ImageRGB32& image){ + for (size_t y = min_y; y < max_y; y++){ + for (size_t x = min_x; x < max_x; x++){ + image.pixel(x, y) = color; + } + } +} + +// The skip badge drawn on black: the white icon, white text strokes (one +// column in four), the dark strip under them. +ImageRGB32 skip_badge_frame(){ + ImageRGB32 image(WIDTH, HEIGHT); + image.fill(0xff000000); + fill_box(1500, 100, 1920, 175, 0xff2c2c30, image); + fill_box(1564, 128, 1586, 150, 0xffffffff, image); + for (size_t x = 1630; x < 1875; x += 4){ + fill_box(x, 124, x + 1, 156, 0xffffffff, image); + } + return image; +} + +// The controls hint's card drawn on black: sky blue with a dark block for +// the top of its "?". +ImageRGB32 controls_hint_frame(){ + ImageRGB32 image(WIDTH, HEIGHT); + image.fill(0xff000000); + fill_box(946, 868, 972, 902, 0xff7dd2fa, image); + fill_box(956, 879, 962, 886, 0xff1c7db0, image); + return image; +} + +UnitTestResult test_synthetic_platform(Logger& logger){ + const ImageRGB32 reference = platform_reference_image(WIDTH, HEIGHT); + const PlatformDistances same = platform_distances(reference); + logger.log(std::format("reference: {}", to_string(same))); + if (!platform_matches(same) || same.top > 1 || same.left > 1 || same.right > 1){ + return std::format("Expected the reference's own cells to match at about 0: {}", to_string(same)); + } + ImageRGB32 gray(WIDTH, HEIGHT); + gray.fill(0xff808080); + const PlatformDistances other = platform_distances(gray); + if (platform_matches(other)){ + return std::format("Expected a gray frame not to be the platform: {}", to_string(other)); + } + // The player's area (x 780-1200, y 330-1080) is in no region. + ImageRGB32 with_player = reference.copy(); + fill_box(780, 330, 1200, HEIGHT, 0xffffffff, with_player); + if (!platform_matches(platform_distances(with_player))){ + return "Expected the player's area not to count."; + } + return true; +} + +// The advance chevron drawn in `ink` on `background`: a V 32x19 at +// x 1400, y 1004, each arm 18% of the width across. +ImageRGB32 chevron_frame(uint32_t background, uint32_t ink){ + ImageRGB32 image(WIDTH, HEIGHT); + image.fill(background); + for (size_t y = 0; y < 19; y++){ + const double v = (double(y) + 0.5) / 19; + for (size_t x = 0; x < 32; x++){ + const double u = (double(x) + 0.5) / 32; + if (std::abs(std::abs(u - 0.5) - (0.40 * (1 - v) + 0.03)) <= 0.09){ + image.pixel(1400 + x, 1004 + y) = ink; + } + } + } + return image; +} + +// The chevron white on the ad's light blue and dark on a white box; not a +// plain white or black frame, nor a solid block the chevron's size. +UnitTestResult test_synthetic_arrow(Logger&){ + AdvanceArrowDetector arrow; + if (!arrow.detect(chevron_frame(0xffc1ecf0, 0xffffffff)) || !arrow.detect(chevron_frame(0xfffcfcf7, 0xff323648))){ + return "Expected the drawn chevrons to be the advance arrow."; + } + ImageRGB32 block(WIDTH, HEIGHT); + block.fill(0xffc1ecf0); + fill_box(1400, 1004, 1432, 1023, 0xffffffff, block); + ImageRGB32 white(WIDTH, HEIGHT); + white.fill(0xffffffff); + ImageRGB32 black(WIDTH, HEIGHT); + black.fill(0xff000000); + if (arrow.detect(block) || arrow.detect(white) || arrow.detect(black)){ + return "Expected a white block, a white or a black frame not to be the advance arrow."; + } + return true; +} + +UnitTestResult test_synthetic_badges(Logger&){ + SkipPromptDetector detector; + if (!detector.detect(skip_badge_frame())){ + return "Expected the drawn badge to be the skip prompt."; + } + ImageRGB32 white(WIDTH, HEIGHT); + white.fill(0xffffffff); + ImageRGB32 black(WIDTH, HEIGHT); + black.fill(0xff000000); + if (detector.detect(white) || detector.detect(black)){ + return "Expected a white or black frame not to be the skip prompt."; + } + AutosaveDetector autosave; + if (autosave.detect(white) || autosave.detect(black) || autosave.detect(skip_badge_frame())){ + return "Expected a white, black or skip-badge frame not to be the autosave badge."; + } + ControlsHintDetector hint; + if (!hint.detect(controls_hint_frame()) || hint.detect(white) || hint.detect(black)){ + return "Expected the drawn card, and no white or black frame, to be the controls hint."; + } + ImageRGB32 sky(WIDTH, HEIGHT); + sky.fill(0xff7dd2fa); + if (hint.detect(sky)){ + return "Expected a frame all in the card's blue (a sky) not to be the controls hint."; + } + return true; +} + +// The mission banner drawn on a grey scene: the yellow bar, and the +// icon's white hexagon (as a block) with the orange stroke, `dx` right of +// run 3's; with `icon` false, the bar alone. +ImageRGB32 mission_banner_frame(size_t dx, bool icon){ + ImageRGB32 image(WIDTH, HEIGHT); + image.fill(0xff606060); + fill_box(345, 410, 1590, 600, 0xfffad228, image); + if (icon){ + fill_box(700 + dx, 327, 738 + dx, 370, 0xffffffff, image); + fill_box(716 + dx, 336, 720 + dx, 357, 0xfff07020, image); + } + return image; +} + +// The banner at run 3's place and 40 px right; not the bar without the +// icon, a frame all in the bar's yellow, or a white frame. +UnitTestResult test_synthetic_mission_banner(Logger&){ + MainMissionBannerDetector banner; + if (!banner.detect(mission_banner_frame(0, true)) || !banner.detect(mission_banner_frame(40, true))){ + return "Expected the drawn banners to be the mission banner."; + } + ImageRGB32 yellow(WIDTH, HEIGHT); + yellow.fill(0xfffad228); + ImageRGB32 white(WIDTH, HEIGHT); + white.fill(0xffffffff); + if (banner.detect(mission_banner_frame(0, false)) || banner.detect(yellow) || banner.detect(white)){ + return "Expected a bar without the icon, a yellow or a white frame not to be the mission banner."; + } + return true; +} + +// The mission complete card drawn on a grey scene: the yellow bar, and +// "COMPLETE" as green strokes (three columns in four) above it. +ImageRGB32 mission_complete_frame(){ + ImageRGB32 image(WIDTH, HEIGHT); + image.fill(0xff606060); + fill_box(345, 390, 1590, 585, 0xfffad228, image); + for (size_t x = 615; x < 1300; x += 4){ + fill_box(x, 275, x + 3, 345, 0xffa5f07d, image); + } + return image; +} + +// The drawn card, not the banner; the drawn banner, a frame all in the +// card's green or the bar's yellow, not the card. +UnitTestResult test_synthetic_mission_complete(Logger&){ + MissionCompleteDetector complete; + MainMissionBannerDetector banner; + const ImageRGB32 card = mission_complete_frame(); + if (!complete.detect(card)){ + return "Expected the drawn card to be the mission complete card."; + } + if (banner.detect(card)){ + return "Expected the drawn card not to be the mission banner."; + } + ImageRGB32 green(WIDTH, HEIGHT); + green.fill(0xffa5f07d); + ImageRGB32 yellow(WIDTH, HEIGHT); + yellow.fill(0xfffad228); + if (complete.detect(mission_banner_frame(0, true)) || complete.detect(green) || complete.detect(yellow)){ + return "Expected the banner, a green or a yellow frame not to be the mission complete card."; + } + return true; +} + +// What a frame should show. +struct FrameExpectation{ + bool platform; + bool skip_prompt; + bool autosave; + bool controls_hint; + bool advance_arrow; + bool mission_banner = false; + bool mission_complete = false; +}; + +// The lab's opening: platform_ref.png is the platform, no cutscene frame +// 0001-0056 is; only 0050 shows the skip prompt. The platform reference +// was taken after the controls hint had gone. +std::optional lab_opening(const std::string& name){ + return FrameExpectation{ + .platform = name == "platform_ref.png", .skip_prompt = name.starts_with("0050-"), + .autosave = false, .controls_hint = false, .advance_arrow = false, + }; +} + +// The user's run rec1_u001-u146: the platform settles at u131 (u124-u130 +// are the fade in and the camera moving in; u128 already passes, top 77 +// under 90, as the platform it shows), the autosave badge shows on +// u132 only, the controls hint on u132-u138 (~10 s), no skip prompt. The +// train's blue box has its advance chevron on u116-u122. +// Later frames of the folder are not in the set. +std::optional user_opening(const std::string& name){ + const int index = std::stoi(name.substr(6, 3)); + if (index > 146){ + return std::nullopt; + } + return FrameExpectation{ + .platform = index == 128 || index >= 131, .skip_prompt = false, + .autosave = index == 132, .controls_hint = index >= 132 && index <= 138, + .advance_arrow = index >= 116 && index <= 122, + }; +} + +// 01.01's first run: the platform (0001-0003, as the run read it) and the +// walk with the controls hint (0001-0006; it is gone by 0007), then the +// white fade, the title card, the text-free flyover, the starters and the +// girl, the tourist choice (0065) and the ad (0080-0089): no free control. +// No autosave: the run took flyover frames 0033 and 0038 for one. The +// advance chevron on the one box with its text done, 0040. +std::optional run1_arrival(const std::string& name){ + const int index = std::stoi(name.substr(0, 4)); + return FrameExpectation{ + .platform = name.find("+platform") != std::string::npos, .skip_prompt = false, + .autosave = false, .controls_hint = index <= 6, .advance_arrow = index == 40, + }; +} + +// 01.01's second run: as the first, with the hint on 0001-0006, on to the +// ad (the logo 0088-0089, the president's subtitles 0090-0115). The +// advance chevron on the boxes with their text done (0044, 0049, 0053, +// 0078) and on the ad's subtitles once typed (0093-0115). +std::optional run2_arrival(const std::string& name){ + const int index = std::stoi(name.substr(0, 4)); + const bool box_arrow = index == 44 || index == 49 || index == 53 || index == 78; + return FrameExpectation{ + .platform = name.find("+platform") != std::string::npos, .skip_prompt = false, + .autosave = false, .controls_hint = index <= 6, + .advance_arrow = box_arrow || (index >= 93 && index <= 115), + }; +} + +// 01.01's third run: as the second, on through the ad and the bag scene. +// The platform, the controls hint and the advance chevron as the run read +// them (in each frame's name; checked by eye on the hint and the chevrons +// through 0115 in run 2); the mission banner on 0181 only, the last frame. +std::optional run3_arrival(const std::string& name){ + const int index = std::stoi(name.substr(0, 4)); + return FrameExpectation{ + .platform = name.find("+platform") != std::string::npos, .skip_prompt = false, + .autosave = false, .controls_hint = name.find("+controls_hint") != std::string::npos, + .advance_arrow = name.find("+advance_arrow") != std::string::npos, + .mission_banner = index == 181, + }; +} + +// A frame as a run read it, by its name ("0133-nothing+mission_banner.png"). +FrameExpectation as_named(const std::string& name){ + const auto has = [&name](std::string_view reading){ + return name.find(reading) != std::string::npos; + }; + return FrameExpectation{ + .platform = has("+platform"), .skip_prompt = has("+skip_prompt"), + .autosave = has("+autosave"), .controls_hint = has("+controls_hint"), + .advance_arrow = has("+advance_arrow"), .mission_banner = has("+mission_banner"), + }; +} + +// 01.01's fourth run: every detector as the run read it; the banner on +// 0181-0189 (checked by eye), no complete card. +std::optional run4_arrival(const std::string& name){ + return as_named(name); +} + +// 01.03's second run: as the run read it, and the complete card on 0045, +// 0046 (named "nothing": the run had no detector for it) and the full-res +// frame; the "MAIN MISSION 02" banner on 0133-0139 is not the card. +std::optional run_0103_2(const std::string& name){ + FrameExpectation want = as_named(name); + want.mission_complete = name.starts_with("0045-") || name.starts_with("0046-") || + name == "mission01-complete-fullres.png"; + return want; +} + +// The opening's detectors, one of each. +struct OpeningDetectors{ + PlatformArrivalDetector platform; + SkipPromptDetector skip; + AutosaveDetector autosave; + ControlsHintDetector hint; + AdvanceArrowDetector arrow; + MainMissionBannerDetector banner; + MissionCompleteDetector complete; + + FrameExpectation read(const ImageRGB32& frame){ + return FrameExpectation{ + .platform = platform.detect(frame), .skip_prompt = skip.detect(frame), + .autosave = autosave.detect(frame), .controls_hint = hint.detect(frame), + .advance_arrow = arrow.detect(frame), .mission_banner = banner.detect(frame), + .mission_complete = complete.detect(frame), + }; + } +}; + +bool same_reads(const FrameExpectation& got, const FrameExpectation& want){ + return got.platform == want.platform && got.skip_prompt == want.skip_prompt && + got.autosave == want.autosave && got.controls_hint == want.controls_hint && + got.advance_arrow == want.advance_arrow && got.mission_banner == want.mission_banner && + got.mission_complete == want.mission_complete; +} + +// The frames `prefix`*.png in `folder` that `expect` has an expectation +// for, against it; logs each one. +UnitTestResult check_frames( + Logger& logger, const std::string& folder, const std::string& prefix, + size_t count, const std::function(const std::string&)>& expect +){ + OpeningDetectors detectors; + size_t frames = 0; + std::string failures; + for (const Filesystem::Path& path : Filesystem::list_directory(Filesystem::Path(folder))){ + const std::string name = path.filename().string(); + const std::optional want = + name.starts_with(prefix) && name.ends_with(".png") ? expect(name) : std::nullopt; + if (!want){ + continue; + } + const ImageRGB32 frame(path.string()); + const FrameExpectation got = detectors.read(frame); + logger.log(std::format("{}: {}{}{}{}{}{}{}{}", name, to_string(detectors.platform.last_distances()), + got.platform ? ", platform" : "", got.skip_prompt ? ", skip prompt" : "", + got.autosave ? ", autosave" : "", got.controls_hint ? ", controls hint" : "", + got.advance_arrow ? ", advance arrow" : "", got.mission_banner ? ", mission banner" : "", + got.mission_complete ? ", mission complete" : "")); + if (!same_reads(got, *want)){ + failures += std::format(" {}", name); + } + frames++; + } + if (frames != count){ + return std::format("Expected {} frames in {}, found {}.", count, folder, frames); + } + if (!failures.empty()){ + return std::format("Wrong on:{}", failures); + } + return true; +} + +// The folder in the environment variable `variable`; none when unset. +std::string frames_folder(std::string_view variable){ + const auto* folder = std::getenv(std::string(variable).c_str()); + if (folder == nullptr || !Filesystem::is_directory(Filesystem::Path(folder))){ + return ""; + } + return folder; +} + +UnitTestResult test_lab_opening_frames(Logger& logger){ + const std::string folder = frames_folder("PLZA_OPENING_FRAMES"); + if (folder.empty()){ + return UnitTestResult(UnitTestResult::SKIPPED, "PLZA_OPENING_FRAMES is not a folder."); + } + // 0001-0056 and platform_ref.png. + return check_frames(logger, folder, "", 57, lab_opening); +} + +UnitTestResult test_user_opening_frames(Logger& logger){ + const std::string folder = frames_folder("PLZA_USER_OPENING_FRAMES"); + if (folder.empty()){ + return UnitTestResult(UnitTestResult::SKIPPED, "PLZA_USER_OPENING_FRAMES is not a folder."); + } + return check_frames(logger, folder, "rec1_u", 146, user_opening); +} + +UnitTestResult test_run1_arrival_frames(Logger& logger){ + const std::string folder = frames_folder("PLZA_0101_RUN1_FRAMES"); + if (folder.empty()){ + return UnitTestResult(UnitTestResult::SKIPPED, "PLZA_0101_RUN1_FRAMES is not a folder."); + } + return check_frames(logger, folder, "00", 89, run1_arrival); +} + +UnitTestResult test_run2_arrival_frames(Logger& logger){ + const std::string folder = frames_folder("PLZA_0101_RUN2_FRAMES"); + if (folder.empty()){ + return UnitTestResult(UnitTestResult::SKIPPED, "PLZA_0101_RUN2_FRAMES is not a folder."); + } + return check_frames(logger, folder, "0", 115, run2_arrival); +} + +UnitTestResult test_run3_arrival_frames(Logger& logger){ + const std::string folder = frames_folder("PLZA_0101_RUN3_FRAMES"); + if (folder.empty()){ + return UnitTestResult(UnitTestResult::SKIPPED, "PLZA_0101_RUN3_FRAMES is not a folder."); + } + return check_frames(logger, folder, "0", 181, run3_arrival); +} + +UnitTestResult test_run4_arrival_frames(Logger& logger){ + const std::string folder = frames_folder("PLZA_0101_RUN4_FRAMES"); + if (folder.empty()){ + return UnitTestResult(UnitTestResult::SKIPPED, "PLZA_0101_RUN4_FRAMES is not a folder."); + } + return check_frames(logger, folder, "0", 221, run4_arrival); +} + +UnitTestResult test_0103_run2_frames(Logger& logger){ + const std::string folder = frames_folder("PLZA_0103_RUN2_FRAMES"); + if (folder.empty()){ + return UnitTestResult(UnitTestResult::SKIPPED, "PLZA_0103_RUN2_FRAMES is not a folder."); + } + // 0001-0139 and mission01-complete-fullres.png. + return check_frames(logger, folder, "", 140, run_0103_2); +} + + +struct OpeningTestInput{ + std::string name; + std::function body; +}; + +struct OpeningTestExpectation{ +}; + +struct OpeningTestCase{ + OpeningTestInput input; + OpeningTestExpectation expectation; +}; + +class Test_AutoStoryOpeningDetectors : public UnitTest{ +public: + explicit Test_AutoStoryOpeningDetectors(OpeningTestCase test_case) + : UnitTest(std::format("PokemonLZA::AutoStoryDetectors - {}", test_case.input.name)) + , m_case(std::move(test_case)) + {} + + virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ + return m_case.input.body(logger); + } + +private: + OpeningTestCase m_case; +}; + +} // namespace + + +const OpeningTestCase OPENING_TEST_CASES[] = { + {.input = {"platform: synthetic frames", test_synthetic_platform}, .expectation = {}}, + {.input = {"skip prompt, autosave and controls hint: synthetic frames", test_synthetic_badges}, .expectation = {}}, + {.input = {"advance arrow: synthetic frames", test_synthetic_arrow}, .expectation = {}}, + {.input = {"mission banner: synthetic frames", test_synthetic_mission_banner}, .expectation = {}}, + {.input = {"mission complete card: synthetic frames", test_synthetic_mission_complete}, .expectation = {}}, + {.input = {"the lab's opening frames (PLZA_OPENING_FRAMES)", test_lab_opening_frames}, .expectation = {}}, + {.input = {"the user's opening frames (PLZA_USER_OPENING_FRAMES)", test_user_opening_frames}, .expectation = {}}, + {.input = {"01.01 run 1's arrival frames (PLZA_0101_RUN1_FRAMES)", test_run1_arrival_frames}, .expectation = {}}, + {.input = {"01.01 run 2's arrival frames to the ad (PLZA_0101_RUN2_FRAMES)", test_run2_arrival_frames}, .expectation = {}}, + {.input = {"01.01 run 3's arrival frames to the mission banner (PLZA_0101_RUN3_FRAMES)", test_run3_arrival_frames}, .expectation = {}}, + {.input = {"01.01 run 4's arrival frames to the mission banner (PLZA_0101_RUN4_FRAMES)", test_run4_arrival_frames}, .expectation = {}}, + {.input = {"01.03 run 2's frames: the mission complete card and the MM02 banner (PLZA_0103_RUN2_FRAMES)", test_0103_run2_frames}, .expectation = {}}, +}; + +void add_tests_AutoStoryDetectors(UnitTestDatabase& database){ + for (const OpeningTestCase& item : OPENING_TEST_CASES){ + database.add(item); + } +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h new file mode 100644 index 0000000000..3bae55b920 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h @@ -0,0 +1,28 @@ +/* PokemonLZA AutoStory - Detectors Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryDetectors_Tests_H +#define PokemonAutomation_PokemonLZA_AutoStoryDetectors_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Registers the tests of the AutoStory detectors (Detectors/): synthetic +// frames always; the opening's frames from the folders in +// PLZA_OPENING_FRAMES and PLZA_USER_OPENING_FRAMES, and the runs' frames +// from PLZA_0101_RUN1_FRAMES and the others, each skipped when unset. +void add_tests_AutoStoryDetectors(UnitTestDatabase& database); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.cpp new file mode 100644 index 0000000000..d855d6fe50 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.cpp @@ -0,0 +1,77 @@ +/* PokemonLZA AutoStory - Main mission banner detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStoryMainMissionBannerDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Measured on 1920x1080 (console B, 2026-11-10, 01.01 run 3's 0181): the +// bar's yellow fills x 430-820 and x 1100-1500 at y 425-585 (left and +// right of the name), none of it 15 px above or below the bar; the icon's +// hexagon is white at x 700-710 and 724-734, y 337-358, with the "!"'s +// orange stroke at x 716-719 between them. +MainMissionBannerDetector::MainMissionBannerDetector(Color color) + : m_color(color) + , m_bar_left(430.0 / 1920, 425.0 / 1080, 390.0 / 1920, 160.0 / 1080) + , m_bar_right(1100.0 / 1920, 425.0 / 1080, 400.0 / 1920, 160.0 / 1080) + , m_above(430.0 / 1920, 375.0 / 1080, 1070.0 / 1920, 25.0 / 1080) + , m_below(430.0 / 1920, 615.0 / 1080, 1070.0 / 1920, 25.0 / 1080) + , m_icon_left(700.0 / 1920, 337.0 / 1080, 10.0 / 1920, 21.0 / 1080) + , m_icon_right(724.0 / 1920, 337.0 / 1080, 10.0 / 1920, 21.0 / 1080) + , m_icon_stroke(716.0 / 1920, 337.0 / 1080, 3.0 / 1920, 11.0 / 1080) +{} + +void MainMissionBannerDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_bar_left); + items.add(m_color, m_bar_right); + items.add(m_color, m_above); + items.add(m_color, m_below); + items.add(m_color, m_icon_left); + items.add(m_color, m_icon_right); + items.add(m_color, m_icon_stroke); +} + +// Measured on 01.01's runs 1-3, the lab's and the user's opening frames +// and the other PLZA lab captures (746 frames, console B, 2026-11-10): +// the banner's one frame (run 3's 0181) has both bar boxes 100% yellow, +// the strips around it 0-1%, and the icon's three boxes 100% at dx 0. The +// closest others: the bar boxes 8% at most, the icon's weakest box 27% at +// most (run 2's 0032, the flyover). The icon is looked for 60 px either side, as a longer +// or shorter header moves it. +bool MainMissionBannerDetector::detect(const ImageViewRGB32& screen){ + if (banner_yellow_share(screen, m_bar_left) < 0.6 || banner_yellow_share(screen, m_bar_right) < 0.6){ + return false; + } + if (banner_yellow_share(screen, m_above) > 0.3 || banner_yellow_share(screen, m_below) > 0.3){ + return false; + } + const auto bright = [](Color pixel){ + return pixel.red() > 225 && pixel.green() > 225 && pixel.blue() > 225; + }; + const auto orange = [](Color pixel){ + return pixel.red() > 200 && pixel.blue() < 150; + }; + for (double dx = -60; dx <= 60; dx += 2){ + if (share_of(screen, shifted(m_icon_left, dx), bright) >= 0.7 && + share_of(screen, shifted(m_icon_right, dx), bright) >= 0.7 && + share_of(screen, shifted(m_icon_stroke, dx), orange) >= 0.7 + ){ + return true; + } + } + return false; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.h new file mode 100644 index 0000000000..17bc5d634c --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.h @@ -0,0 +1,52 @@ +/* PokemonLZA AutoStory - Main mission banner detector + * + * From: https://github.com/PokemonAutomation/ + * + * The "MAIN MISSION 01" banner across the middle when a main mission + * starts. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryMainMissionBannerDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryMainMissionBannerDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The banner a main mission starts with: a see-through yellow bar at +// x 345-1590, y 410-600 on 1920x1080 with the mission's name in the +// middle, and above it, left of "MAIN MISSION 01", a white hexagon with +// an orange "!" at about x 700-738, y 327-370. The bar is read left and +// right of the name, with no yellow just above or below it, and the icon +// is looked for along the header (the header's text is centred). Text is +// not read, so any language matches. +class MainMissionBannerDetector : public StaticScreenDetector{ +public: + explicit MainMissionBannerDetector(Color color = COLOR_ORANGE); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_bar_left; + ImageFloatBox m_bar_right; + ImageFloatBox m_above; + ImageFloatBox m_below; + ImageFloatBox m_icon_left; + ImageFloatBox m_icon_right; + ImageFloatBox m_icon_stroke; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.cpp new file mode 100644 index 0000000000..85d58c7d82 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.cpp @@ -0,0 +1,58 @@ +/* PokemonLZA AutoStory - Mission complete detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStoryMissionCompleteDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Measured on 1920x1080 (console B, 2026-11-10, 01.03 run 2's +// mission01-complete-fullres.png): the word's green fills x 615-1300, +// y 271-349, none of it 20 px above or below; the bar's yellow fills +// y 390-585, read left and right of the name as for the banner. +MissionCompleteDetector::MissionCompleteDetector(Color color) + : m_color(color) + , m_word(640.0 / 1920, 285.0 / 1080, 640.0 / 1920, 55.0 / 1080) + , m_above(620.0 / 1920, 245.0 / 1080, 680.0 / 1920, 20.0 / 1080) + , m_below(620.0 / 1920, 358.0 / 1080, 680.0 / 1920, 20.0 / 1080) + , m_bar_left(430.0 / 1920, 440.0 / 1080, 390.0 / 1920, 140.0 / 1080) + , m_bar_right(1100.0 / 1920, 440.0 / 1080, 400.0 / 1920, 140.0 / 1080) +{} + +void MissionCompleteDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_word); + items.add(m_color, m_above); + items.add(m_color, m_below); + items.add(m_color, m_bar_left); + items.add(m_color, m_bar_right); +} + +// Measured on every PLZA lab capture (1109 frames, console B, 2026-11-10): +// the card's three frames (01.03 run 2's 0045, 0046 and the full-res +// one) have the word 70% green, nothing green around it and both bar +// boxes 100% yellow. The closest others: the word 11% green at most +// (plza-platform's boxes.png, no bar), and with the bar, the mission +// banners (01.01 run 3's 0181, run 4's 0182-0189, 01.03 run 2's +// 0133-0139) at 2% at most. +bool MissionCompleteDetector::detect(const ImageViewRGB32& screen){ + if (complete_green_share(screen, m_word) < 0.40){ + return false; + } + if (complete_green_share(screen, m_above) > 0.10 || complete_green_share(screen, m_below) > 0.10){ + return false; + } + return banner_yellow_share(screen, m_bar_left) >= 0.6 && banner_yellow_share(screen, m_bar_right) >= 0.6; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.h new file mode 100644 index 0000000000..e337d5ff41 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.h @@ -0,0 +1,49 @@ +/* PokemonLZA AutoStory - Mission complete detector + * + * From: https://github.com/PokemonAutomation/ + * + * The "COMPLETE" card when a main mission ends, which waits for A. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryMissionCompleteDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryMissionCompleteDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The card a main mission ends with, which waits for A: a big green +// "COMPLETE" at x 615-1300, y 271-349 on 1920x1080 over the mission's +// yellow bar (x 345-1590, y 390-585, "MAIN MISSION 01" and the mission's +// name), in a yellow hexagon with the mission's item. The word is read +// with nothing green just above or below it, and the bar left and right +// of the name. The word is the same in every language seen (the header +// stays English in Chinese), so its text is not read. +class MissionCompleteDetector : public StaticScreenDetector{ +public: + explicit MissionCompleteDetector(Color color = COLOR_GREEN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_word; + ImageFloatBox m_above; + ImageFloatBox m_below; + ImageFloatBox m_bar_left; + ImageFloatBox m_bar_right; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.cpp new file mode 100644 index 0000000000..38d41b598f --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.cpp @@ -0,0 +1,61 @@ +/* PokemonLZA AutoStory - Pixel shares + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "PokemonLZA_AutoStoryPixelShares.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +double cyan_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.blue() > pixel.red() + 50 && pixel.green() > pixel.red() + 30 && pixel.blue() > 120; + }); +} + +// The share of `box`'s pixels whose every channel is above 200. +double white_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.red() > 200 && pixel.green() > 200 && pixel.blue() > 200; + }); +} + +// The share of `box`'s pixels in the controls hint card's sky blue (about +// 125, 210, 250): blue over 200, green over 160, blue over red by 60. +double sky_blue_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.blue() > 200 && pixel.green() > 160 && pixel.blue() > pixel.red() + 60; + }); +} + +// The share of `box`'s pixels in the mission banner's yellow (about 250, +// 210, 40 over a scene): red over 200, green over 140, blue under 130, +// red over blue by 110. +double banner_yellow_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.red() > 200 && pixel.green() > 140 && pixel.blue() < 130 && + pixel.red() > pixel.blue() + 110; + }); +} + +// The share of `box`'s pixels in the mission complete card's green (about +// 165, 240, 125): green over 180, over red by 40 and over blue by 70. +double complete_green_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.green() > 180 && pixel.green() > pixel.red() + 40 && pixel.green() > pixel.blue() + 70; + }); +} + +ImageFloatBox shifted(ImageFloatBox box, double dx){ + box.x += dx / 1920; + return box; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.h new file mode 100644 index 0000000000..d31b670875 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.h @@ -0,0 +1,72 @@ +/* PokemonLZA AutoStory - Pixel shares + * + * From: https://github.com/PokemonAutomation/ + * + * How much of a box's pixels are of a color: what the badge detectors + * (Detectors/) read. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryPixelShares_H +#define PokemonAutomation_PokemonLZA_AutoStoryPixelShares_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The share of `box`'s pixels that `matches` (a Color -> bool). +template +double share_of(const ImageViewRGB32& screen, const ImageFloatBox& box, Matches&& matches); + +// The share of `box`'s pixels that glow cyan: blue over red by 50, green +// over red by 30, blue over 120. +double cyan_share(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// The share of `box`'s pixels whose every channel is above 200. +double white_share(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// The share of `box`'s pixels in the controls hint card's sky blue (about +// 125, 210, 250): blue over 200, green over 160, blue over red by 60. +double sky_blue_share(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// The share of `box`'s pixels in the mission banner's yellow (about 250, +// 210, 40 over a scene): red over 200, green over 140, blue under 130, +// red over blue by 110. +double banner_yellow_share(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// The share of `box`'s pixels in the mission complete card's green (about +// 165, 240, 125): green over 180, over red by 40 and over blue by 70. +double complete_green_share(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// `box` moved `dx` pixels (of 1920) right. +ImageFloatBox shifted(ImageFloatBox box, double dx); + + +template +double share_of(const ImageViewRGB32& screen, const ImageFloatBox& box, Matches&& matches){ + const ImageViewRGB32 area = extract_box_reference(screen, box); + const size_t total = area.width() * area.height(); + if (total == 0){ + return 0; + } + size_t count = 0; + for (size_t y = 0; y < area.height(); y++){ + for (size_t x = 0; x < area.width(); x++){ + if (matches(Color(area.pixel(x, y)))){ + count++; + } + } + } + return (double)count / (double)total; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp new file mode 100644 index 0000000000..e89ae9892d --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp @@ -0,0 +1,204 @@ +/* PokemonLZA AutoStory - Opening detectors + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Json/JsonArray.h" +#include "Common/Cpp/Json/JsonObject.h" +#include "Common/Cpp/Json/JsonValue.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPlatformArrivalDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +// A region of the frame and the cells it is averaged into. On 1920x1080 +// (the user's design, 2026-11-10): the top band y 0-300 across; the left +// side x 0-640 and the right side x 1280-1920, both y 300-1080. The player +// (x 780-1200, y 330-1080) is in none of them. Each cell is 30x30 (top) or +// 40x39 pixels (sides). +struct PlatformRegion{ + std::string_view name; + ImageFloatBox box; + size_t cols; + size_t rows; +}; + +const PlatformRegion PLATFORM_REGIONS[] = { + {"top", {0.0, 0.0, 1.0, 300.0 / 1080}, 64, 10}, + {"left", {0.0, 300.0 / 1080, 640.0 / 1920, 780.0 / 1080}, 16, 20}, + {"right", {1280.0 / 1920, 300.0 / 1080, 640.0 / 1920, 780.0 / 1080}, 16, 20}, +}; + +// The bounds of platform_matches(), from the opening's frames (console B, +// 1920x1080: the platform reference and the 56 cutscene frames before +// it, measured with this code's cell averages): +// - the closest cutscene frame is 0056, the train's interior just before +// the platform: top 112.8, left 82.6, right 71.9; the next are the +// city flyovers 0022-0024 at top 134.7+, left 95.6+; +// - the reference against itself moved 16 px: top 42.2, left 20.8, +// right 21.2; 10% darker: top 24.6, left 19.9, right 16.4. +// - the platform after loading its autosave (camera turned a little, +// 2026-11-10): top 67.9-68.1, left 25.8-41.7, right 16.5-16.9. +// So the bounds sit between the loaded view and 0056: top 90 (23 below +// 0056), left 55 (27 below). The right side holds a walking NPC and the companion, so it only +// has to be roughly the platform (120: a menu or another place is 140+). +constexpr double PLATFORM_TOP_MAX = 90.0; +constexpr double PLATFORM_LEFT_MAX = 55.0; +constexpr double PLATFORM_RIGHT_MAX = 120.0; + +using Cells = std::vector; + +// `region` of `screen`, averaged into its cells, row by row. +Cells region_cells(const ImageViewRGB32& screen, const PlatformRegion& region){ + Cells cells; + cells.reserve(region.cols * region.rows); + const double cell_width = region.box.width / (double)region.cols; + const double cell_height = region.box.height / (double)region.rows; + for (size_t row = 0; row < region.rows; row++){ + for (size_t col = 0; col < region.cols; col++){ + const ImageFloatBox cell( + region.box.x + cell_width * (double)col, region.box.y + cell_height * (double)row, + cell_width, cell_height + ); + cells.push_back(image_stats(extract_box_reference(screen, cell)).average); + } + } + return cells; +} + +// "rrggbb" to its color. +FloatPixel hex_pixel(const std::string& hex, size_t offset){ + const auto channel = [&](size_t index){ + return (double)std::stoi(hex.substr(offset + index * 2, 2), nullptr, 16); + }; + return FloatPixel(channel(0), channel(1), channel(2)); +} + +// The reference's cells of `region` from ReferenceData/plza-platform.json: +// "regions": {: {"cols": n, "rows": ["rrggbb...", ...]}}. +Cells reference_cells(const JsonObject& regions, const PlatformRegion& region){ + const std::string source = "plza-platform.json"; + const JsonObject& entry = regions.get_object_throw(std::string(region.name), source); + const JsonArray& rows = entry.get_array_throw("rows", source); + if ((size_t)entry.get_integer_throw("cols", source) != region.cols || rows.size() != region.rows){ + throw ParseException(std::format("{}: region {} is not {}x{} cells.", source, region.name, region.cols, region.rows)); + } + Cells cells; + for (const JsonValue& row : rows){ + const std::string& hex = row.to_string_throw(source); + if (hex.size() != region.cols * 6){ + throw ParseException(std::format("{}: a row of region {} is not {} colors.", source, region.name, region.cols)); + } + for (size_t col = 0; col < region.cols; col++){ + cells.push_back(hex_pixel(hex, col * 6)); + } + } + return cells; +} + +// The reference's cells, per PLATFORM_REGIONS entry. +const std::vector& platform_reference(){ + static const std::vector reference = []{ + for (const auto& [name, text] : autostory_reference_json()){ + if (name != "plza-platform"){ + continue; + } + const JsonValue json = parse_json(text); + const JsonObject& regions = json.to_object_throw("plza-platform.json").get_object_throw("regions"); + std::vector cells; + for (const PlatformRegion& region : PLATFORM_REGIONS){ + cells.push_back(reference_cells(regions, region)); + } + return cells; + } + throw FileException(nullptr, PA_CURRENT_FUNCTION, "The platform reference is not built in.", "plza-platform.json"); + }(); + return reference; +} + +double cells_rmsd(const Cells& reference, const Cells& cells){ + double sum = 0; + for (size_t index = 0; index < cells.size(); index++){ + const double r = reference[index].r - cells[index].r; + const double g = reference[index].g - cells[index].g; + const double b = reference[index].b - cells[index].b; + sum += r * r + g * g + b * b; + } + return std::sqrt(sum / (double)cells.size()); +} + +} // namespace + + +PlatformDistances platform_distances(const ImageViewRGB32& screen){ + const std::vector& reference = platform_reference(); + double distance[3] = {}; + for (size_t index = 0; index < 3; index++){ + distance[index] = cells_rmsd(reference[index], region_cells(screen, PLATFORM_REGIONS[index])); + } + return PlatformDistances{.top = distance[0], .left = distance[1], .right = distance[2]}; +} + +bool platform_matches(const PlatformDistances& distances){ + return distances.top <= PLATFORM_TOP_MAX + && distances.left <= PLATFORM_LEFT_MAX + && distances.right <= PLATFORM_RIGHT_MAX; +} + +ImageRGB32 platform_reference_image(size_t width, size_t height){ + ImageRGB32 image(width, height); + image.fill(0xff000000); + const std::vector& reference = platform_reference(); + for (size_t index = 0; index < 3; index++){ + const PlatformRegion& region = PLATFORM_REGIONS[index]; + const ImagePixelBox box = floatbox_to_pixelbox(width, height, region.box); + for (size_t y = box.min_y; y < box.max_y; y++){ + for (size_t x = box.min_x; x < box.max_x; x++){ + const size_t col = (x - box.min_x) * region.cols / (box.max_x - box.min_x); + const size_t row = (y - box.min_y) * region.rows / (box.max_y - box.min_y); + const FloatPixel& cell = reference[index][row * region.cols + col]; + image.pixel(x, y) = (uint32_t)Color((uint8_t)std::lround(cell.r), (uint8_t)std::lround(cell.g), (uint8_t)std::lround(cell.b)); + } + } + } + return image; +} + +std::string to_string(const PlatformDistances& distances){ + return std::format("top {:.1f}, left {:.1f}, right {:.1f}", distances.top, distances.left, distances.right); +} + + +PlatformArrivalDetector::PlatformArrivalDetector(Color color) + : m_color(color) +{} + +void PlatformArrivalDetector::make_overlays(VideoOverlaySet& items) const{ + for (const PlatformRegion& region : PLATFORM_REGIONS){ + items.add(m_color, region.box); + } +} + +bool PlatformArrivalDetector::detect(const ImageViewRGB32& screen){ + m_last = platform_distances(screen); + return platform_matches(m_last); +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h new file mode 100644 index 0000000000..d8635ccc5e --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h @@ -0,0 +1,83 @@ +/* PokemonLZA AutoStory - Platform arrival detector + * + * From: https://github.com/PokemonAutomation/ + * + * The station platform once the new game's opening cutscene ends and the + * player has control (PlatformArrivalDetector): the frame's top and sides + * against a reference of that first frame, with the player cropped out. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryPlatformArrivalDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryPlatformArrivalDetector_H + +#include +#include +#include +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The checked-in reference files, ReferenceData/*.json, built into the +// program (cmake/EmbedTextFiles.cmake): (name, text), by name. +// plza-platform holds the platform reference's cell colors per region. +const std::vector>& autostory_reference_json(); + + +// How far a frame's regions are from the platform reference: each region +// is averaged into cells, and the distance is the RMSD over its cells of +// the summed squared channel differences (0 = the same, at most 441). +struct PlatformDistances{ + double top; + double left; + double right; +}; + +// The regions of a frame against the platform reference. +PlatformDistances platform_distances(const ImageViewRGB32& screen); + +// Whether `distances` are the platform: top and left within their bounds, +// and the right side within a looser one (the companion and the NPCs there +// move). The bounds and the measurements behind them are in the .cpp. +bool platform_matches(const PlatformDistances& distances); + +// The reference as a `width` x `height` frame: each cell painted its +// color, the player's area black. A frame the detector sees as the +// platform, for tests. +ImageRGB32 platform_reference_image(size_t width, size_t height); + +// "top 12.3, left 4.5, right 30.1", for the log. +std::string to_string(const PlatformDistances& distances); + + +// The station platform with free control, right after the opening +// cutscene, at the default camera. +class PlatformArrivalDetector : public StaticScreenDetector{ +public: + explicit PlatformArrivalDetector(Color color = COLOR_CYAN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + + // The distances of the last detect(). + const PlatformDistances& last_distances() const{ + return m_last; + } + +private: + Color m_color; + PlatformDistances m_last{.top = 0, .left = 0, .right = 0}; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.cpp new file mode 100644 index 0000000000..ac1ded93af --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.cpp @@ -0,0 +1,53 @@ +/* PokemonLZA AutoStory - Skip prompt detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStorySkipPromptDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The badge on 1920x1080 (frame 0050 of the opening): the white (+) icon +// at x 1564-1586, y 128-150; the text x 1630-1875, y 122-158; the strip +// under the text x 1640-1880, y 161-169. +SkipPromptDetector::SkipPromptDetector(Color color) + : m_color(color) + , m_icon(1564.0 / 1920, 128.0 / 1080, 22.0 / 1920, 22.0 / 1080) + , m_text(1630.0 / 1920, 122.0 / 1080, 245.0 / 1920, 36.0 / 1080) + , m_strip(1640.0 / 1920, 161.0 / 1080, 240.0 / 1920, 8.0 / 1080) +{} + +void SkipPromptDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_icon); + items.add(m_color, m_text); + items.add(m_color, m_strip); +} + +// On frame 0050 the icon is 70% white, the text 24%, and the strip is +// dark and even (average 44, deviation 2). No other opening frame has the +// icon over 40% white with a dark strip: the brightest (0044, a white +// flash) has the icon at 38% and the strip at 220. +bool SkipPromptDetector::detect(const ImageViewRGB32& screen){ + if (white_share(screen, m_icon) < 0.40){ + return false; + } + const double text = white_share(screen, m_text); + if (text < 0.10 || text > 0.45){ + return false; + } + const ImageStats strip = image_stats(extract_box_reference(screen, m_strip)); + return strip.average.sum() < 3 * 100.0 && strip.stddev.sum() < 3 * 15.0; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.h new file mode 100644 index 0000000000..78079b4a31 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.h @@ -0,0 +1,43 @@ +/* PokemonLZA AutoStory - Skip prompt detector + * + * From: https://github.com/PokemonAutomation/ + * + * The opening cutscene's "(+) Hold button to skip" badge. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStorySkipPromptDetector_H +#define PokemonAutomation_PokemonLZA_AutoStorySkipPromptDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The cutscene's "(+) Hold button to skip" badge at the top right: the +// white (+) icon, white text on the badge, and the badge's dark, even +// strip under the text. +class SkipPromptDetector : public StaticScreenDetector{ +public: + explicit SkipPromptDetector(Color color = COLOR_MAGENTA); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_icon; + ImageFloatBox m_text; + ImageFloatBox m_strip; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/ReferenceData/plza-platform.json b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/ReferenceData/plza-platform.json new file mode 100644 index 0000000000..1e673d8cca --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/ReferenceData/plza-platform.json @@ -0,0 +1,70 @@ +{ + "source": "platform_ref.png: console B, the station platform right after the opening cutscene (default camera)", + "regions": { + "top": { + "cols": 64, + "rows": [ + "96dbf56fa2a7364b27577a55a2e8ffa8edffadf1ffb2f4ffb6f6ffb3f7ffb0f6ff9ee2f090d3e6a6edfca8f1ffa6f0ffa3eeff73a7bb234156476f8774afc846698241657d21415a243a4047658321436244668048687e4a6b8248698047687e4869804c6e854b6d8548697f45647a45667c39586c2c4557213a4c26466726475f1f3b554756665e65736d70797679817c7d856c70786e71796d7381777d8f74798773767e6f727963676f6c717c61656d5b606b61646b676a72666a7254585e", + "719ba7374b263245245576645c816a85b6a297cec6c0faffc4fdffc1fcffbdfbffb9faffadeef780b7ca86bfd09cdbe9aff3ff74a8b924455b466a7da7eeff6797ab2b475b25486022384139566e36587441608046677d3c5b6f3753673450632e495b2d485a2d485a2f4a5d355163355164294456243d4e1f3a51315d8b3a65911e3d5554616f6c7078797b8274767e797b826569716b6f79747a8c6e7586767b8b70747e6d707873757d6a6e775e626a5b606955596065697362677152565c", + "2d3849343e4c39474636442f4057323f553396c6b9c8feffc9ffffc9ffffc8ffffc2fafca5dbe37eabb76e97a587bac8afedf67fb3bf25475b486e7fb4f5ff82bbce385e782f546e223843324e63365a7745617f253f54314d6233516838576e33557030516b30526b33516738566b2942562840531f394b2e567e4c79ae5f769121435e54606d73767f75787f6c707a686c756c6f767479875d636d4f58666870816b707a77798183848b737781676a725d626c6669706f727a686b73545961", + "364f742a36482c3848313d4e3743504b5e5dacd7dcccfeffd1ffffd3ffffd3ffffcdfdfdb1dfe391bac3b4e6eaccffffb9edf27daab7224555456876c2fbffc0f9fdb0e9f16596a9223d472945542e516d3a5875213c513350663456723d607c46687f4a6c84496a813e627a2a4a62274255223a4c2546624671a4596d865e70885563735b626e6367706b6f78777f93777d8d676b746e73814a50583f464d4d545e6a6d7773757d84848b787c85676b746569737d7e8371757d60646c555a62", + "7da9f775a6f85b86ca4566943648622f3b4b313e524e5e6d83a0a9bce2e5d6ffffc9f1f350677192b5bbd8ffffdaffffd7ffff96c2c922414e43626ecdfeffd2ffffd1ffffc1f4f629455026404b29496237516326405726455c2e4a5b2f4b5c2c47592b47592c47582d48592e475927455a223b4f39536e4860773e5a72405b736d7179676c756c6f78767a856067766f76886c717c646a7442494e3d434a3c434b70747e7878807a7c837a7d8773757d666a7475777e6e7077696d744f545d", + "9cb8e59ebceba0c0f49bbffb8cb8fb7aa6e75b6281434559313c4b313f505366748da9b1aed1d594b2b6dfffffe1ffffdfffffa1c7cc2543523e5b67b1d9e3bdebf4b3efff97e2fe45768f28485a284b622c455229475f2c475d38576d2d4d6432526934546b3150662d4b61233b4d243f54304d664353626268725f68715c656e6a6f796a6e776e7179717581484e56565d6a757b89565c6341474ea4a7a9c7c9cbd9dadcd9dadcd9dadbdadbddd8d8da6e727b71747b7879806d70774d525d", + "9eb9e19fbae3a2bde6aac5ebacc8eeaecaf39fafd36b56606e5d705f67845470963b4d66323e4f47576494acb2a9c3c8defefea4c9d025414d39566290deff74c3ee70bae478cdf761a3c433556a3d6784243e4a35546a233d5235556c34546b305168315269325168284255254256374b5e546578495764767881787a8275777f6f727b6c70796b6f78676c7841484d4c535e757b8b545a62434950cfd0d2d2d3d6d2d3d7cbced2d5d7dbc6c8ccf7f7f76f737c7f808787878c63676e4f555f", + "9ab5e0a3bee4a6c0e7a2bde6a5c0e7a7c2e8aabfe46b54516c56546d5a5ea6c4f59fc3fa8aaee56483b24b5e783341544661734f7678264550344f5b70b8d876c0e774bee56cadd44c6f863e657966afd72a48562d49582742562b495f355870365a743456702f4e6528455a3f55673a4f5d75777f676b747a7c838281887d7e8674777f72767e676b746167723f4e5c4a5b70696f7c535963464c56d0d1d4c7c9ced7dcdeced1d5dadde0e3e5e6efeff06468707e808786868c64686f4c515c", + "9bb4dd9eb8df9fbbe1a3bce3a9c2e6a4bfe4a9c1e66a53506b54516a53519db6dd7e8a9e79889da6c3f18eb6f18ab5f57595c75a759425424c304952507c6c5ea09e7fd6ff7dd5ff5077903c61727ad1fc487c972b47532643532d4e645490b565aed65a98be47799a364f62495c69626a73787a8070737c7e7e8585848b626e7c707580787b8371747c606672223f5c3e50664e5c6c525762474d59b0b2b7d1d3d5ced3d8dfe0e1d2d5d7dcdddedadbdc575c6462666c696c735f636b494f5c", + "6b81a37088aa758db087a0c3a1b9dd9fb9df9ab2d54c4d5e524d5a584f5793abd28086927e756b9bb8e68cacd980a7df96b3e08397b3233b442b404a6882a84f66834a6e8466aacf4f77903b5e6e69add94a799540687c2946553c647e7fb6ca8ecbdf7ab9d472c2eb4c62726e747c7f80866f737b6f737b7e7f8586858b7d838c6f77846d707b7679815d66715870824c5a6758616c515761454c574147524b515c4a566c585b694c535e6f727b6d70795459626c6f77686c7460656d494f5c" + ] + }, + "left": { + "cols": 16, + "rows": [ + "6c7e92596a7d49618167604c6c685a474c443b413f3c4959808c9a7c92b17b95bc7ea2d38da8d54b5a64263a418caee2", + "6b809a4d5d6b486488645d48655e473841442d353f31384067604e6b7b8553667e799cc9708296314a5c273b41667993", + "768aa24856644a668a605a465f5a44323b402b343a2c353e645f4f5f6c7252667f7496c16f849a3a596c244251576c82", + "7692b5596d834f6e945b564452504329343c28323a353d455151496f776f5469817292b9626d8036536720373d4a5f77", + "6b748351535d506d915651414d4c433638432c313b3038454649465c5b624d63806f8eb16d7f9a4b68741f3d4e5c7a9a", + "58555f50505e4f6c8f524e414d4c42282a3222262e3d3c444b59697188a17594b86a8db780a1cb40576b1d2c30547293", + "5455605a5d6c4863895e666b626c762e37442b34403c3a425a759680a0c87e9fc8678bb77b9dc843647d1e425341596a", + "667e9d6f8aad81a0c683a2c97a9ac02e3a4827333d2a3240708fb57a9ac57698c457759a4c65802d434f1f323634485d", + "84a5ca81a3c880a2c77c9ec57899c15c7a9e486e9f252e396486b1567499455e77485f73475d722d425220313931455f", + "7fa1c87c9fc77a9dc6779ac47395c05f7ca3547cb8456896425b74485e72495f74405670364c6f354c70364c714b5c7a", + "7a9cc5789ac47698c37295c05c7ea6435d7b43596e495f74495f73445a71395071344a6d344a6e344b6f526280828898", + "7497c17094bf5a79a0445d7942586c475d704a6075485e743c526f344a6e344c70344b70344b6f58668284899a838999", + "506e903f5871445a6e4a6075495f73495e72405671354c6e334a6d344c71344b70354c705f6c85828898868b9c838898", + "44596d495f734a6174495f73435970384f70334a6e32496d334b6f344c72364d72647086828797868c9d858b9c70798e", + "465c70465c70455a6f3b516e344b6f344b6f32496c344c71334a6f394f716e778c848998848a9983889778809370798d", + "465c6f3f556e33496b334b6f334b6f32496d32486c31486b3a4d6d747c8e848a988187948186947d83926b74887f8492", + "3247642e43632f456531476a31466930466830476a40526f777d8c8388938489957f84907e8390677083797f8d808593", + "2c405d2b405d2d42612f45663046683046674958747d828f878b95858994858a958387926f7685737a888085907f838e", + "2a3c572a3d592c405e2e42612f45654b5a747f8490858992848890858891848892747986666e7f7e838d7f838d7f848e", + "293b55293b55293b562e415e545f7381858e878a92868a9283868f8488917a7e89626a7a797d8882868e7f838c7d828b" + ] + }, + "right": { + "cols": 16, + "rows": [ + "666c785d636e62666f474d59545963595e68464c573e4450485364494e60636872797a81656a7387868c6e7178505663", + "686e796b6e78676b74464c57555a64585e67454b553c424d4c4e5d4b4048686b74797b8154596271747c6a6e754d535f", + "5352526c6f785f646e464c58555a64565c65464c563b414c5559644e332c4d4a4d70737a5a5f676b6f76656a724b515f", + "5b5e57656a73555c674248565e626c585d664349533f42496d6b6b83623e747367676b73575c646c6e75646871484e5c", + "515b63656a75545d6d404653565d6b4e54603f4651473c3e958b63af98357e7235616671484d5565686e686c74454c59", + "4050694c5769485367383f4c545b694c525d3b434d403a3fb4a25bb0972e9e8a325b626d4b515c595f68585e65424958", + "3f51713f52714051714052724c597240495a3440522c2c3281765081712c836e2e6469714f555f60666e555b66434a59", + "3f52733f52734052744052734153744053744d5d80515c772f333e12192052464860656e5056605b616a505761434a58", + "3e52733e52744053753f52743f5273405374415475435473283344121b264d545f4f56604a515d5259624e545f404855", + "3e52743e52744054773f52744154773f53744154764255772a374c151e293f48543d444b424d5f4b5462474d5a3b4251", + "3e53773d52753e52743f53774054774256793f5375415476303b50171e278a74407e735f474748435678415371415271", + "3d52753f54783f54783e53763f53764054774054765061825e505f6b5a61615e59ad83334747423f4e6845587a425475", + "3e54793c52753c52753e54783e54773e53763e53764055794155784255784f5a6f71644f8b7243445571425577415374", + "3e53783d52763c51753b51753d53763e53773e53763f54763e537641557942567943577a40547643577a425678415475", + "384e713c51753c51743c50733c50723f53773e53773e52753e52753e51744054784154784053764054764052743f5071", + "3a4e6f374c6d394d6f3b4f723c50733b4e703c4f713d50723c50723b4f703c4f703e51734053763e50713c4e6f3e4f6f", + "4f5d76374b6c374a6c374a6c374a6b374a6a394c6c3b4e703a4d6e3a4d6d3a4d6d3b4e6e3d4e6f3c4e6d394b69384966", + "6d75863d4e6c364869374969364767364867374968384a6a384a6a3a4c6c3b4d6d3c4d6d3a4c6a3b4c6b384865374763", + "7b808c576279344563354766354766344665354766394a69354765374967374967394b6938496637486436466135455e", + "7b7f8b707683404f6835466535476635476535466435476533456136476536476435466234445f32425b32425b324158" + ] + } + } +} diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 6a52c9c381..b3296d7f35 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -2324,6 +2324,24 @@ file(GLOB LIBRARY_SOURCES Source/VideoGames/PokemonLZA/PokemonLZA_Settings.h Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp Source/VideoGames/PokemonLZA/PokemonLZA_Tests.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.cpp + Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.h Source/VideoGames/PokemonLZA/Programs/Farming/PokemonLZA_DonutMaker.cpp Source/VideoGames/PokemonLZA/Programs/Farming/PokemonLZA_DonutMaker.h Source/VideoGames/PokemonLZA/Programs/Farming/PokemonLZA_FriendshipFarmer.cpp From 38272ef8814a42e86e34b4e9269db77d3114f18c Mon Sep 17 00:00:00 2001 From: dayfine Date: Wed, 11 Nov 2026 22:50:15 -0800 Subject: [PATCH 2/5] AutoStory: forward-declare Pokemon::ScreenEvents in Pokemon_StoryBackend.h --- .../Source/Pokemon/AutoStory/Engine/Pokemon_StoryBackend.cpp | 1 + .../Source/Pokemon/AutoStory/Engine/Pokemon_StoryBackend.h | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryBackend.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryBackend.cpp index 17141560df..4011f9d1bc 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryBackend.cpp +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryBackend.cpp @@ -5,6 +5,7 @@ */ #include +#include "Pokemon/AutoStory/Pokemon_ScreenEventTypes.h" #include "Pokemon_StoryBackend.h" namespace PokemonAutomation{ diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryBackend.h b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryBackend.h index 7c8d17fc67..3909cda363 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryBackend.h +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryBackend.h @@ -19,7 +19,6 @@ #include "Common/Cpp/Time.h" #include "CommonTools/ScreenMachine/ScreenObservation.h" #include "NintendoSwitch/Controllers/NintendoSwitch_ControllerButtons.h" -#include "Pokemon/AutoStory/Pokemon_ScreenEventTypes.h" #include "Pokemon/AutoStory/Pokemon_StoryOutcome.h" #include "Pokemon/AutoStory/Types/Pokemon_StoryActionArg.h" #include "Pokemon/Navigation/Pokemon_Grid.h" @@ -29,6 +28,8 @@ namespace PokemonAutomation{ namespace Pokemon{ +class ScreenEvents; + // How fast a held direction moves the player. enum class StoryPace{ From 44951e4ccc1e17b72cef129136d60aea99aba89d Mon Sep 17 00:00:00 2001 From: dayfine Date: Fri, 13 Nov 2026 22:24:00 -0800 Subject: [PATCH 3/5] PokemonLZA AutoStory detectors: test tables as name, input, expectation; guard empty frames The detector tests become two tables of {name, Input, Expectation}: synthetic frames with what each detector must read on them, and recorded frame folders with their frame count and per-frame reads, one runner each. A synthetic case checks only the detectors it names, so the cases assert what the old bodies did. Adds an empty image, on which every detector reads no. PlatformArrivalDetector returns no on an empty frame instead of reaching NaN distances; the reference's hex colors parse with from_chars and throw ParseException on a bad digit; region loops use std::size. Frame file names that are not numbered are skipped instead of throwing from std::stoi, and the frames folder lookup returns an optional instead of "". Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- ...kemonLZA_AutoStoryAdvanceArrowDetector.cpp | 8 +- .../PokemonLZA_AutoStoryDetectors_Tests.cpp | 728 ++++++++++-------- ...onLZA_AutoStoryPlatformArrivalDetector.cpp | 33 +- 3 files changed, 442 insertions(+), 327 deletions(-) diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp index 933f849543..f7210122fe 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp @@ -23,10 +23,10 @@ namespace{ // A 4-connected blob of a mask: its bounds and its pixel count. struct Blob{ - size_t min_x; - size_t min_y; - size_t max_x; - size_t max_y; + size_t min_x = 0; + size_t min_y = 0; + size_t max_x = 0; + size_t max_y = 0; size_t count = 0; }; diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp index 9a74171631..ec11dad6ba 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp @@ -28,6 +28,8 @@ * */ + +#include #include #include #include @@ -35,8 +37,6 @@ #include #include #include -#include -#include #include "Common/Cpp/Filesystem/Filesystem.h" #include "Common/Cpp/Logging/AbstractLogger.h" #include "Common/Cpp/TestRunners/UnitTest.h" @@ -54,11 +54,94 @@ namespace PokemonAutomation{ namespace NintendoSwitch{ namespace PokemonLZA{ + namespace{ constexpr size_t WIDTH = 1920; constexpr size_t HEIGHT = 1080; + +// What each detector reads on a frame: true or false, or none when a test +// does not check that detector. +struct DetectorReads{ + std::optional platform = {}; + std::optional skip_prompt = {}; + std::optional autosave = {}; + std::optional controls_hint = {}; + std::optional advance_arrow = {}; + std::optional mission_banner = {}; + std::optional mission_complete = {}; +}; + +struct DetectorField{ + std::string_view name; + std::optional DetectorReads::* read; +}; +const DetectorField DETECTOR_FIELDS[] = { + {"platform", &DetectorReads::platform}, + {"skip prompt", &DetectorReads::skip_prompt}, + {"autosave", &DetectorReads::autosave}, + {"controls hint", &DetectorReads::controls_hint}, + {"advance arrow", &DetectorReads::advance_arrow}, + {"mission banner", &DetectorReads::mission_banner}, + {"mission complete", &DetectorReads::mission_complete}, +}; + +// `reads` with every detector checked: one it leaves unset must not fire. +DetectorReads every_detector(DetectorReads reads){ + for (const DetectorField& field : DETECTOR_FIELDS){ + reads.*field.read = (reads.*field.read).value_or(false); + } + return reads; +} + +// The detectors that fire in `got`, for the log: ", platform, autosave". +std::string firing(const DetectorReads& got){ + std::string text; + for (const DetectorField& field : DETECTOR_FIELDS){ + if ((got.*field.read).value_or(false)){ + text += std::format(", {}", field.name); + } + } + return text; +} + +// The detectors `want` checks that `got` reads otherwise: " platform +// (expected yes)"; empty when all agree. +std::string mismatches(const DetectorReads& got, const DetectorReads& want){ + std::string text; + for (const DetectorField& field : DETECTOR_FIELDS){ + const std::optional& expected = want.*field.read; + if (expected && got.*field.read != expected){ + text += std::format(" {} (expected {})", field.name, *expected ? "yes" : "no"); + } + } + return text; +} + +// Every detector of this module, one of each. +struct StoryDetectors{ + PlatformArrivalDetector platform; + SkipPromptDetector skip; + AutosaveDetector autosave; + ControlsHintDetector hint; + AdvanceArrowDetector arrow; + MainMissionBannerDetector banner; + MissionCompleteDetector complete; + + DetectorReads read(const ImageViewRGB32& frame){ + return DetectorReads{ + .platform = platform.detect(frame), .skip_prompt = skip.detect(frame), + .autosave = autosave.detect(frame), .controls_hint = hint.detect(frame), + .advance_arrow = arrow.detect(frame), .mission_banner = banner.detect(frame), + .mission_complete = complete.detect(frame), + }; + } +}; + + +// Synthetic frames. + void fill_box(size_t min_x, size_t min_y, size_t max_x, size_t max_y, uint32_t color, ImageRGB32& image){ for (size_t y = min_y; y < max_y; y++){ for (size_t x = min_x; x < max_x; x++){ @@ -67,11 +150,16 @@ void fill_box(size_t min_x, size_t min_y, size_t max_x, size_t max_y, uint32_t c } } +ImageRGB32 solid_frame(uint32_t color){ + ImageRGB32 image(WIDTH, HEIGHT); + image.fill(color); + return image; +} + // The skip badge drawn on black: the white icon, white text strokes (one // column in four), the dark strip under them. ImageRGB32 skip_badge_frame(){ - ImageRGB32 image(WIDTH, HEIGHT); - image.fill(0xff000000); + ImageRGB32 image = solid_frame(0xff000000); fill_box(1500, 100, 1920, 175, 0xff2c2c30, image); fill_box(1564, 128, 1586, 150, 0xffffffff, image); for (size_t x = 1630; x < 1875; x += 4){ @@ -83,40 +171,24 @@ ImageRGB32 skip_badge_frame(){ // The controls hint's card drawn on black: sky blue with a dark block for // the top of its "?". ImageRGB32 controls_hint_frame(){ - ImageRGB32 image(WIDTH, HEIGHT); - image.fill(0xff000000); + ImageRGB32 image = solid_frame(0xff000000); fill_box(946, 868, 972, 902, 0xff7dd2fa, image); fill_box(956, 879, 962, 886, 0xff1c7db0, image); return image; } -UnitTestResult test_synthetic_platform(Logger& logger){ - const ImageRGB32 reference = platform_reference_image(WIDTH, HEIGHT); - const PlatformDistances same = platform_distances(reference); - logger.log(std::format("reference: {}", to_string(same))); - if (!platform_matches(same) || same.top > 1 || same.left > 1 || same.right > 1){ - return std::format("Expected the reference's own cells to match at about 0: {}", to_string(same)); - } - ImageRGB32 gray(WIDTH, HEIGHT); - gray.fill(0xff808080); - const PlatformDistances other = platform_distances(gray); - if (platform_matches(other)){ - return std::format("Expected a gray frame not to be the platform: {}", to_string(other)); - } - // The player's area (x 780-1200, y 330-1080) is in no region. - ImageRGB32 with_player = reference.copy(); - fill_box(780, 330, 1200, HEIGHT, 0xffffffff, with_player); - if (!platform_matches(platform_distances(with_player))){ - return "Expected the player's area not to count."; - } - return true; +// The platform reference with the player's area (x 780-1200, y 330-1080), +// which is in no region, painted white. +ImageRGB32 platform_with_player_frame(){ + ImageRGB32 image = platform_reference_image(WIDTH, HEIGHT); + fill_box(780, 330, 1200, HEIGHT, 0xffffffff, image); + return image; } // The advance chevron drawn in `ink` on `background`: a V 32x19 at // x 1400, y 1004, each arm 18% of the width across. ImageRGB32 chevron_frame(uint32_t background, uint32_t ink){ - ImageRGB32 image(WIDTH, HEIGHT); - image.fill(background); + ImageRGB32 image = solid_frame(background); for (size_t y = 0; y < 19; y++){ const double v = (double(y) + 0.5) / 19; for (size_t x = 0; x < 32; x++){ @@ -129,90 +201,36 @@ ImageRGB32 chevron_frame(uint32_t background, uint32_t ink){ return image; } -// The chevron white on the ad's light blue and dark on a white box; not a -// plain white or black frame, nor a solid block the chevron's size. -UnitTestResult test_synthetic_arrow(Logger&){ - AdvanceArrowDetector arrow; - if (!arrow.detect(chevron_frame(0xffc1ecf0, 0xffffffff)) || !arrow.detect(chevron_frame(0xfffcfcf7, 0xff323648))){ - return "Expected the drawn chevrons to be the advance arrow."; - } - ImageRGB32 block(WIDTH, HEIGHT); - block.fill(0xffc1ecf0); - fill_box(1400, 1004, 1432, 1023, 0xffffffff, block); - ImageRGB32 white(WIDTH, HEIGHT); - white.fill(0xffffffff); - ImageRGB32 black(WIDTH, HEIGHT); - black.fill(0xff000000); - if (arrow.detect(block) || arrow.detect(white) || arrow.detect(black)){ - return "Expected a white block, a white or a black frame not to be the advance arrow."; - } - return true; +// A solid white block the chevron's size on the ad's light blue. +ImageRGB32 chevron_sized_block_frame(){ + ImageRGB32 image = solid_frame(0xffc1ecf0); + fill_box(1400, 1004, 1432, 1023, 0xffffffff, image); + return image; } -UnitTestResult test_synthetic_badges(Logger&){ - SkipPromptDetector detector; - if (!detector.detect(skip_badge_frame())){ - return "Expected the drawn badge to be the skip prompt."; - } - ImageRGB32 white(WIDTH, HEIGHT); - white.fill(0xffffffff); - ImageRGB32 black(WIDTH, HEIGHT); - black.fill(0xff000000); - if (detector.detect(white) || detector.detect(black)){ - return "Expected a white or black frame not to be the skip prompt."; - } - AutosaveDetector autosave; - if (autosave.detect(white) || autosave.detect(black) || autosave.detect(skip_badge_frame())){ - return "Expected a white, black or skip-badge frame not to be the autosave badge."; - } - ControlsHintDetector hint; - if (!hint.detect(controls_hint_frame()) || hint.detect(white) || hint.detect(black)){ - return "Expected the drawn card, and no white or black frame, to be the controls hint."; - } - ImageRGB32 sky(WIDTH, HEIGHT); - sky.fill(0xff7dd2fa); - if (hint.detect(sky)){ - return "Expected a frame all in the card's blue (a sky) not to be the controls hint."; - } - return true; -} +// How mission_banner_frame() draws the icon. +enum class BannerIcon{ + NONE, + DRAWN, +}; -// The mission banner drawn on a grey scene: the yellow bar, and the -// icon's white hexagon (as a block) with the orange stroke, `dx` right of -// run 3's; with `icon` false, the bar alone. -ImageRGB32 mission_banner_frame(size_t dx, bool icon){ - ImageRGB32 image(WIDTH, HEIGHT); - image.fill(0xff606060); +// The mission banner drawn on a grey scene: the yellow bar, and with +// BannerIcon::DRAWN the icon's white hexagon (as a block) with the orange +// stroke, `dx` right of where the 01.01 banner frame has it. +ImageRGB32 mission_banner_frame(size_t dx, BannerIcon icon){ + ImageRGB32 image = solid_frame(0xff606060); fill_box(345, 410, 1590, 600, 0xfffad228, image); - if (icon){ + if (icon == BannerIcon::DRAWN){ fill_box(700 + dx, 327, 738 + dx, 370, 0xffffffff, image); fill_box(716 + dx, 336, 720 + dx, 357, 0xfff07020, image); } return image; } -// The banner at run 3's place and 40 px right; not the bar without the -// icon, a frame all in the bar's yellow, or a white frame. -UnitTestResult test_synthetic_mission_banner(Logger&){ - MainMissionBannerDetector banner; - if (!banner.detect(mission_banner_frame(0, true)) || !banner.detect(mission_banner_frame(40, true))){ - return "Expected the drawn banners to be the mission banner."; - } - ImageRGB32 yellow(WIDTH, HEIGHT); - yellow.fill(0xfffad228); - ImageRGB32 white(WIDTH, HEIGHT); - white.fill(0xffffffff); - if (banner.detect(mission_banner_frame(0, false)) || banner.detect(yellow) || banner.detect(white)){ - return "Expected a bar without the icon, a yellow or a white frame not to be the mission banner."; - } - return true; -} - // The mission complete card drawn on a grey scene: the yellow bar, and // "COMPLETE" as green strokes (three columns in four) above it. ImageRGB32 mission_complete_frame(){ - ImageRGB32 image(WIDTH, HEIGHT); - image.fill(0xff606060); + ImageRGB32 image = solid_frame(0xff606060); fill_box(345, 390, 1590, 585, 0xfffad228, image); for (size_t x = 615; x < 1300; x += 4){ fill_box(x, 275, x + 3, 345, 0xffa5f07d, image); @@ -220,47 +238,169 @@ ImageRGB32 mission_complete_frame(){ return image; } -// The drawn card, not the banner; the drawn banner, a frame all in the -// card's green or the bar's yellow, not the card. -UnitTestResult test_synthetic_mission_complete(Logger&){ - MissionCompleteDetector complete; - MainMissionBannerDetector banner; - const ImageRGB32 card = mission_complete_frame(); - if (!complete.detect(card)){ - return "Expected the drawn card to be the mission complete card."; +constexpr uint32_t WHITE = 0xffffffff; +constexpr uint32_t BLACK = 0xff000000; +constexpr uint32_t GRAY = 0xff808080; +constexpr uint32_t AD_LIGHT_BLUE = 0xffc1ecf0; +constexpr uint32_t BOX_WHITE = 0xfffcfcf7; +constexpr uint32_t BOX_TEXT_DARK = 0xff323648; +constexpr uint32_t HINT_SKY_BLUE = 0xff7dd2fa; +constexpr uint32_t BANNER_YELLOW = 0xfffad228; +constexpr uint32_t COMPLETE_GREEN = 0xffa5f07d; + +struct SyntheticFrameCase{ + std::string name; + struct Input{ + std::function frame; + }; + Input input; + struct Expectation{ + DetectorReads reads = {}; + }; + Expectation expectation; +}; + +const SyntheticFrameCase SYNTHETIC_FRAME_CASES[] = { + { + .name = "an empty image", + .input = {.frame = []{ return ImageRGB32(); }}, + .expectation = {.reads = every_detector({})}, + }, + { + .name = "a black frame", + .input = {.frame = []{ return solid_frame(BLACK); }}, + .expectation = {.reads = {.skip_prompt = false, .autosave = false, .controls_hint = false, .advance_arrow = false}}, + }, + { + .name = "a white frame", + .input = {.frame = []{ return solid_frame(WHITE); }}, + .expectation = {.reads = { + .skip_prompt = false, .autosave = false, .controls_hint = false, .advance_arrow = false, .mission_banner = false, + }}, + }, + { + .name = "a gray frame", + .input = {.frame = []{ return solid_frame(GRAY); }}, + .expectation = {.reads = {.platform = false}}, + }, + { + .name = "the platform reference", + .input = {.frame = []{ return platform_reference_image(WIDTH, HEIGHT); }}, + .expectation = {.reads = {.platform = true}}, + }, + { + .name = "the platform reference with the player's area painted", + .input = {.frame = platform_with_player_frame}, + .expectation = {.reads = {.platform = true}}, + }, + { + .name = "the skip badge", + .input = {.frame = skip_badge_frame}, + .expectation = {.reads = {.skip_prompt = true, .autosave = false}}, + }, + { + .name = "the controls hint card", + .input = {.frame = controls_hint_frame}, + .expectation = {.reads = {.controls_hint = true}}, + }, + { + .name = "a frame all in the hint card's blue (a sky)", + .input = {.frame = []{ return solid_frame(HINT_SKY_BLUE); }}, + .expectation = {.reads = {.controls_hint = false}}, + }, + { + .name = "a white chevron on the ad's light blue", + .input = {.frame = []{ return chevron_frame(AD_LIGHT_BLUE, WHITE); }}, + .expectation = {.reads = {.advance_arrow = true}}, + }, + { + .name = "a dark chevron on a white box", + .input = {.frame = []{ return chevron_frame(BOX_WHITE, BOX_TEXT_DARK); }}, + .expectation = {.reads = {.advance_arrow = true}}, + }, + { + .name = "a solid block the chevron's size", + .input = {.frame = chevron_sized_block_frame}, + .expectation = {.reads = {.advance_arrow = false}}, + }, + { + .name = "the mission banner", + .input = {.frame = []{ return mission_banner_frame(0, BannerIcon::DRAWN); }}, + .expectation = {.reads = {.mission_banner = true, .mission_complete = false}}, + }, + { + .name = "the mission banner with its icon 40 px right", + .input = {.frame = []{ return mission_banner_frame(40, BannerIcon::DRAWN); }}, + .expectation = {.reads = {.mission_banner = true}}, + }, + { + .name = "the mission banner's bar without the icon", + .input = {.frame = []{ return mission_banner_frame(0, BannerIcon::NONE); }}, + .expectation = {.reads = {.mission_banner = false}}, + }, + { + .name = "a frame all in the bar's yellow", + .input = {.frame = []{ return solid_frame(BANNER_YELLOW); }}, + .expectation = {.reads = {.mission_banner = false, .mission_complete = false}}, + }, + { + .name = "the mission complete card", + .input = {.frame = mission_complete_frame}, + .expectation = {.reads = {.mission_banner = false, .mission_complete = true}}, + }, + { + .name = "a frame all in the card's green", + .input = {.frame = []{ return solid_frame(COMPLETE_GREEN); }}, + .expectation = {.reads = {.mission_complete = false}}, + }, +}; + +UnitTestResult check_synthetic_frame(const SyntheticFrameCase& test, Logger& logger){ + StoryDetectors detectors; + const DetectorReads got = detectors.read(test.input.frame()); + logger.log(std::format("{}: {}{}", test.name, to_string(detectors.platform.last_distances()), firing(got))); + const std::string wrong = mismatches(got, test.expectation.reads); + if (!wrong.empty()){ + return std::format("Wrong on {}:{}", test.name, wrong); } - if (banner.detect(card)){ - return "Expected the drawn card not to be the mission banner."; + return true; +} + + +// Recorded frames, from folders named by environment variables. + +// The number of `digits` digits at `offset` in a frame's file name; none +// if they are not all digits. +std::optional frame_number(std::string_view name, size_t offset, size_t digits){ + if (name.size() < offset + digits){ + return std::nullopt; } - ImageRGB32 green(WIDTH, HEIGHT); - green.fill(0xffa5f07d); - ImageRGB32 yellow(WIDTH, HEIGHT); - yellow.fill(0xfffad228); - if (complete.detect(mission_banner_frame(0, true)) || complete.detect(green) || complete.detect(yellow)){ - return "Expected the banner, a green or a yellow frame not to be the mission complete card."; + const std::string_view text = name.substr(offset, digits); + int number = 0; + const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), number); + if (error != std::errc() || end != text.data() + text.size()){ + return std::nullopt; } - return true; + return number; } -// What a frame should show. -struct FrameExpectation{ - bool platform; - bool skip_prompt; - bool autosave; - bool controls_hint; - bool advance_arrow; - bool mission_banner = false; - bool mission_complete = false; -}; +// A frame as a run read it, by its name ("0133-nothing+mission_banner.png"). +DetectorReads as_named(std::string_view name){ + const auto has = [name](std::string_view reading){ + return name.find(reading) != std::string_view::npos; + }; + return DetectorReads{ + .platform = has("+platform"), .skip_prompt = has("+skip_prompt"), + .autosave = has("+autosave"), .controls_hint = has("+controls_hint"), + .advance_arrow = has("+advance_arrow"), .mission_banner = has("+mission_banner"), + }; +} // The lab's opening: platform_ref.png is the platform, no cutscene frame // 0001-0056 is; only 0050 shows the skip prompt. The platform reference // was taken after the controls hint had gone. -std::optional lab_opening(const std::string& name){ - return FrameExpectation{ - .platform = name == "platform_ref.png", .skip_prompt = name.starts_with("0050-"), - .autosave = false, .controls_hint = false, .advance_arrow = false, - }; +std::optional lab_opening(std::string_view name){ + return DetectorReads{.platform = name == "platform_ref.png", .skip_prompt = name.starts_with("0050-")}; } // The user's run rec1_u001-u146: the platform settles at u131 (u124-u130 @@ -269,15 +409,15 @@ std::optional lab_opening(const std::string& name){ // u132 only, the controls hint on u132-u138 (~10 s), no skip prompt. The // train's blue box has its advance chevron on u116-u122. // Later frames of the folder are not in the set. -std::optional user_opening(const std::string& name){ - const int index = std::stoi(name.substr(6, 3)); - if (index > 146){ +std::optional user_opening(std::string_view name){ + const std::optional index = frame_number(name, 6, 3); + if (!index || *index > 146){ return std::nullopt; } - return FrameExpectation{ - .platform = index == 128 || index >= 131, .skip_prompt = false, - .autosave = index == 132, .controls_hint = index >= 132 && index <= 138, - .advance_arrow = index >= 116 && index <= 122, + return DetectorReads{ + .platform = *index == 128 || *index >= 131, + .autosave = *index == 132, .controls_hint = *index >= 132 && *index <= 138, + .advance_arrow = *index >= 116 && *index <= 122, }; } @@ -287,11 +427,14 @@ std::optional user_opening(const std::string& name){ // girl, the tourist choice (0065) and the ad (0080-0089): no free control. // No autosave: the run took flyover frames 0033 and 0038 for one. The // advance chevron on the one box with its text done, 0040. -std::optional run1_arrival(const std::string& name){ - const int index = std::stoi(name.substr(0, 4)); - return FrameExpectation{ - .platform = name.find("+platform") != std::string::npos, .skip_prompt = false, - .autosave = false, .controls_hint = index <= 6, .advance_arrow = index == 40, +std::optional run1_arrival(std::string_view name){ + const std::optional index = frame_number(name, 0, 4); + if (!index){ + return std::nullopt; + } + return DetectorReads{ + .platform = name.find("+platform") != std::string_view::npos, + .controls_hint = *index <= 6, .advance_arrow = *index == 40, }; } @@ -299,13 +442,16 @@ std::optional run1_arrival(const std::string& name){ // ad (the logo 0088-0089, the president's subtitles 0090-0115). The // advance chevron on the boxes with their text done (0044, 0049, 0053, // 0078) and on the ad's subtitles once typed (0093-0115). -std::optional run2_arrival(const std::string& name){ - const int index = std::stoi(name.substr(0, 4)); - const bool box_arrow = index == 44 || index == 49 || index == 53 || index == 78; - return FrameExpectation{ - .platform = name.find("+platform") != std::string::npos, .skip_prompt = false, - .autosave = false, .controls_hint = index <= 6, - .advance_arrow = box_arrow || (index >= 93 && index <= 115), +std::optional run2_arrival(std::string_view name){ + const std::optional index = frame_number(name, 0, 4); + if (!index){ + return std::nullopt; + } + const bool box_arrow = *index == 44 || *index == 49 || *index == 53 || *index == 78; + return DetectorReads{ + .platform = name.find("+platform") != std::string_view::npos, + .controls_hint = *index <= 6, + .advance_arrow = box_arrow || (*index >= 93 && *index <= 115), }; } @@ -313,101 +459,132 @@ std::optional run2_arrival(const std::string& name){ // The platform, the controls hint and the advance chevron as the run read // them (in each frame's name; checked by eye on the hint and the chevrons // through 0115 in run 2); the mission banner on 0181 only, the last frame. -std::optional run3_arrival(const std::string& name){ - const int index = std::stoi(name.substr(0, 4)); - return FrameExpectation{ - .platform = name.find("+platform") != std::string::npos, .skip_prompt = false, - .autosave = false, .controls_hint = name.find("+controls_hint") != std::string::npos, - .advance_arrow = name.find("+advance_arrow") != std::string::npos, - .mission_banner = index == 181, - }; -} - -// A frame as a run read it, by its name ("0133-nothing+mission_banner.png"). -FrameExpectation as_named(const std::string& name){ - const auto has = [&name](std::string_view reading){ - return name.find(reading) != std::string::npos; - }; - return FrameExpectation{ - .platform = has("+platform"), .skip_prompt = has("+skip_prompt"), - .autosave = has("+autosave"), .controls_hint = has("+controls_hint"), - .advance_arrow = has("+advance_arrow"), .mission_banner = has("+mission_banner"), +std::optional run3_arrival(std::string_view name){ + const std::optional index = frame_number(name, 0, 4); + if (!index){ + return std::nullopt; + } + return DetectorReads{ + .platform = name.find("+platform") != std::string_view::npos, + .controls_hint = name.find("+controls_hint") != std::string_view::npos, + .advance_arrow = name.find("+advance_arrow") != std::string_view::npos, + .mission_banner = *index == 181, }; } // 01.01's fourth run: every detector as the run read it; the banner on // 0181-0189 (checked by eye), no complete card. -std::optional run4_arrival(const std::string& name){ +std::optional run4_arrival(std::string_view name){ return as_named(name); } // 01.03's second run: as the run read it, and the complete card on 0045, // 0046 (named "nothing": the run had no detector for it) and the full-res // frame; the "MAIN MISSION 02" banner on 0133-0139 is not the card. -std::optional run_0103_2(const std::string& name){ - FrameExpectation want = as_named(name); +std::optional run_0103_2(std::string_view name){ + DetectorReads want = as_named(name); want.mission_complete = name.starts_with("0045-") || name.starts_with("0046-") || name == "mission01-complete-fullres.png"; return want; } -// The opening's detectors, one of each. -struct OpeningDetectors{ - PlatformArrivalDetector platform; - SkipPromptDetector skip; - AutosaveDetector autosave; - ControlsHintDetector hint; - AdvanceArrowDetector arrow; - MainMissionBannerDetector banner; - MissionCompleteDetector complete; +struct FrameFolderCase{ + std::string name; + struct Input{ + std::string_view folder_variable; // The environment variable naming the folder. + std::string_view prefix; // The file names to read: *.png. + }; + Input input; + struct Expectation{ + size_t frames = 0; // How many files of the folder `reads` has an expectation for. + // Each frame's reads, by file name; none for a file not in the set. + // A detector it leaves unset must not fire. + std::function(std::string_view name)> reads = {}; + }; + Expectation expectation; +}; - FrameExpectation read(const ImageRGB32& frame){ - return FrameExpectation{ - .platform = platform.detect(frame), .skip_prompt = skip.detect(frame), - .autosave = autosave.detect(frame), .controls_hint = hint.detect(frame), - .advance_arrow = arrow.detect(frame), .mission_banner = banner.detect(frame), - .mission_complete = complete.detect(frame), - }; - } +const FrameFolderCase FRAME_FOLDER_CASES[] = { + { + .name = "the lab's opening frames", + .input = {.folder_variable = "PLZA_OPENING_FRAMES", .prefix = ""}, + // 0001-0056 and platform_ref.png. + .expectation = {.frames = 57, .reads = lab_opening}, + }, + { + .name = "the user's opening frames", + .input = {.folder_variable = "PLZA_USER_OPENING_FRAMES", .prefix = "rec1_u"}, + .expectation = {.frames = 146, .reads = user_opening}, + }, + { + .name = "01.01 run 1's arrival frames", + .input = {.folder_variable = "PLZA_0101_RUN1_FRAMES", .prefix = "00"}, + .expectation = {.frames = 89, .reads = run1_arrival}, + }, + { + .name = "01.01 run 2's arrival frames to the ad", + .input = {.folder_variable = "PLZA_0101_RUN2_FRAMES", .prefix = "0"}, + .expectation = {.frames = 115, .reads = run2_arrival}, + }, + { + .name = "01.01 run 3's arrival frames to the mission banner", + .input = {.folder_variable = "PLZA_0101_RUN3_FRAMES", .prefix = "0"}, + .expectation = {.frames = 181, .reads = run3_arrival}, + }, + { + .name = "01.01 run 4's arrival frames to the mission banner", + .input = {.folder_variable = "PLZA_0101_RUN4_FRAMES", .prefix = "0"}, + .expectation = {.frames = 221, .reads = run4_arrival}, + }, + { + .name = "01.03 run 2's frames: the mission complete card and the MM02 banner", + .input = {.folder_variable = "PLZA_0103_RUN2_FRAMES", .prefix = ""}, + // 0001-0139 and mission01-complete-fullres.png. + .expectation = {.frames = 140, .reads = run_0103_2}, + }, }; -bool same_reads(const FrameExpectation& got, const FrameExpectation& want){ - return got.platform == want.platform && got.skip_prompt == want.skip_prompt && - got.autosave == want.autosave && got.controls_hint == want.controls_hint && - got.advance_arrow == want.advance_arrow && got.mission_banner == want.mission_banner && - got.mission_complete == want.mission_complete; +// The folder in the environment variable `variable`; none when it is unset +// or not a folder. +std::optional frames_folder(std::string_view variable){ + const auto* value = std::getenv(std::string(variable).c_str()); + if (value == nullptr){ + return std::nullopt; + } + std::string folder(value); + if (!Filesystem::is_directory(Filesystem::Path(folder))){ + return std::nullopt; + } + return folder; } -// The frames `prefix`*.png in `folder` that `expect` has an expectation -// for, against it; logs each one. -UnitTestResult check_frames( - Logger& logger, const std::string& folder, const std::string& prefix, - size_t count, const std::function(const std::string&)>& expect -){ - OpeningDetectors detectors; +UnitTestResult check_frame_folder(const FrameFolderCase& test, Logger& logger){ + const std::optional folder = frames_folder(test.input.folder_variable); + if (!folder){ + return UnitTestResult(UnitTestResult::SKIPPED, std::format("{} is not a folder.", test.input.folder_variable)); + } + StoryDetectors detectors; size_t frames = 0; std::string failures; - for (const Filesystem::Path& path : Filesystem::list_directory(Filesystem::Path(folder))){ + for (const Filesystem::Path& path : Filesystem::list_directory(Filesystem::Path(*folder))){ const std::string name = path.filename().string(); - const std::optional want = - name.starts_with(prefix) && name.ends_with(".png") ? expect(name) : std::nullopt; + if (!name.starts_with(test.input.prefix) || !name.ends_with(".png")){ + continue; + } + const std::optional want = test.expectation.reads(name); if (!want){ continue; } - const ImageRGB32 frame(path.string()); - const FrameExpectation got = detectors.read(frame); - logger.log(std::format("{}: {}{}{}{}{}{}{}{}", name, to_string(detectors.platform.last_distances()), - got.platform ? ", platform" : "", got.skip_prompt ? ", skip prompt" : "", - got.autosave ? ", autosave" : "", got.controls_hint ? ", controls hint" : "", - got.advance_arrow ? ", advance arrow" : "", got.mission_banner ? ", mission banner" : "", - got.mission_complete ? ", mission complete" : "")); - if (!same_reads(got, *want)){ - failures += std::format(" {}", name); + const DetectorReads got = detectors.read(ImageRGB32(path.string())); + logger.log(std::format("{}: {}{}", name, to_string(detectors.platform.last_distances()), firing(got))); + const std::string wrong = mismatches(got, every_detector(*want)); + if (!wrong.empty()){ + failures += std::format(" {}:{};", name, wrong); } frames++; } - if (frames != count){ - return std::format("Expected {} frames in {}, found {}.", count, folder, frames); + if (frames != test.expectation.frames){ + return std::format("Expected {} frames in {}, found {}.", test.expectation.frames, *folder, frames); } if (!failures.empty()){ return std::format("Wrong on:{}", failures); @@ -415,123 +592,44 @@ UnitTestResult check_frames( return true; } -// The folder in the environment variable `variable`; none when unset. -std::string frames_folder(std::string_view variable){ - const auto* folder = std::getenv(std::string(variable).c_str()); - if (folder == nullptr || !Filesystem::is_directory(Filesystem::Path(folder))){ - return ""; - } - return folder; -} - -UnitTestResult test_lab_opening_frames(Logger& logger){ - const std::string folder = frames_folder("PLZA_OPENING_FRAMES"); - if (folder.empty()){ - return UnitTestResult(UnitTestResult::SKIPPED, "PLZA_OPENING_FRAMES is not a folder."); - } - // 0001-0056 and platform_ref.png. - return check_frames(logger, folder, "", 57, lab_opening); -} - -UnitTestResult test_user_opening_frames(Logger& logger){ - const std::string folder = frames_folder("PLZA_USER_OPENING_FRAMES"); - if (folder.empty()){ - return UnitTestResult(UnitTestResult::SKIPPED, "PLZA_USER_OPENING_FRAMES is not a folder."); - } - return check_frames(logger, folder, "rec1_u", 146, user_opening); -} - -UnitTestResult test_run1_arrival_frames(Logger& logger){ - const std::string folder = frames_folder("PLZA_0101_RUN1_FRAMES"); - if (folder.empty()){ - return UnitTestResult(UnitTestResult::SKIPPED, "PLZA_0101_RUN1_FRAMES is not a folder."); - } - return check_frames(logger, folder, "00", 89, run1_arrival); -} - -UnitTestResult test_run2_arrival_frames(Logger& logger){ - const std::string folder = frames_folder("PLZA_0101_RUN2_FRAMES"); - if (folder.empty()){ - return UnitTestResult(UnitTestResult::SKIPPED, "PLZA_0101_RUN2_FRAMES is not a folder."); - } - return check_frames(logger, folder, "0", 115, run2_arrival); -} -UnitTestResult test_run3_arrival_frames(Logger& logger){ - const std::string folder = frames_folder("PLZA_0101_RUN3_FRAMES"); - if (folder.empty()){ - return UnitTestResult(UnitTestResult::SKIPPED, "PLZA_0101_RUN3_FRAMES is not a folder."); - } - return check_frames(logger, folder, "0", 181, run3_arrival); -} - -UnitTestResult test_run4_arrival_frames(Logger& logger){ - const std::string folder = frames_folder("PLZA_0101_RUN4_FRAMES"); - if (folder.empty()){ - return UnitTestResult(UnitTestResult::SKIPPED, "PLZA_0101_RUN4_FRAMES is not a folder."); - } - return check_frames(logger, folder, "0", 221, run4_arrival); -} - -UnitTestResult test_0103_run2_frames(Logger& logger){ - const std::string folder = frames_folder("PLZA_0103_RUN2_FRAMES"); - if (folder.empty()){ - return UnitTestResult(UnitTestResult::SKIPPED, "PLZA_0103_RUN2_FRAMES is not a folder."); +class Test_SyntheticFrame : public UnitTest{ +public: + explicit Test_SyntheticFrame(const SyntheticFrameCase& test) + : UnitTest(std::format("PokemonLZA::AutoStoryDetectors - synthetic frame: {}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ + return check_synthetic_frame(m_test, logger); } - // 0001-0139 and mission01-complete-fullres.png. - return check_frames(logger, folder, "", 140, run_0103_2); -} - -struct OpeningTestInput{ - std::string name; - std::function body; -}; - -struct OpeningTestExpectation{ -}; - -struct OpeningTestCase{ - OpeningTestInput input; - OpeningTestExpectation expectation; +private: + const SyntheticFrameCase& m_test; }; -class Test_AutoStoryOpeningDetectors : public UnitTest{ +class Test_FrameFolder : public UnitTest{ public: - explicit Test_AutoStoryOpeningDetectors(OpeningTestCase test_case) - : UnitTest(std::format("PokemonLZA::AutoStoryDetectors - {}", test_case.input.name)) - , m_case(std::move(test_case)) + explicit Test_FrameFolder(const FrameFolderCase& test) + : UnitTest(std::format("PokemonLZA::AutoStoryDetectors - {} ({})", test.name, test.input.folder_variable)) + , m_test(test) {} - virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ - return m_case.input.body(logger); + return check_frame_folder(m_test, logger); } private: - OpeningTestCase m_case; + const FrameFolderCase& m_test; }; } // namespace -const OpeningTestCase OPENING_TEST_CASES[] = { - {.input = {"platform: synthetic frames", test_synthetic_platform}, .expectation = {}}, - {.input = {"skip prompt, autosave and controls hint: synthetic frames", test_synthetic_badges}, .expectation = {}}, - {.input = {"advance arrow: synthetic frames", test_synthetic_arrow}, .expectation = {}}, - {.input = {"mission banner: synthetic frames", test_synthetic_mission_banner}, .expectation = {}}, - {.input = {"mission complete card: synthetic frames", test_synthetic_mission_complete}, .expectation = {}}, - {.input = {"the lab's opening frames (PLZA_OPENING_FRAMES)", test_lab_opening_frames}, .expectation = {}}, - {.input = {"the user's opening frames (PLZA_USER_OPENING_FRAMES)", test_user_opening_frames}, .expectation = {}}, - {.input = {"01.01 run 1's arrival frames (PLZA_0101_RUN1_FRAMES)", test_run1_arrival_frames}, .expectation = {}}, - {.input = {"01.01 run 2's arrival frames to the ad (PLZA_0101_RUN2_FRAMES)", test_run2_arrival_frames}, .expectation = {}}, - {.input = {"01.01 run 3's arrival frames to the mission banner (PLZA_0101_RUN3_FRAMES)", test_run3_arrival_frames}, .expectation = {}}, - {.input = {"01.01 run 4's arrival frames to the mission banner (PLZA_0101_RUN4_FRAMES)", test_run4_arrival_frames}, .expectation = {}}, - {.input = {"01.03 run 2's frames: the mission complete card and the MM02 banner (PLZA_0103_RUN2_FRAMES)", test_0103_run2_frames}, .expectation = {}}, -}; - void add_tests_AutoStoryDetectors(UnitTestDatabase& database){ - for (const OpeningTestCase& item : OPENING_TEST_CASES){ - database.add(item); + for (const SyntheticFrameCase& test : SYNTHETIC_FRAME_CASES){ + database.add(test); + } + for (const FrameFolderCase& test : FRAME_FOLDER_CASES){ + database.add(test); } } diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp index e89ae9892d..18f8f1fa52 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp @@ -4,7 +4,9 @@ * */ +#include #include +#include #include #include #include @@ -25,8 +27,8 @@ namespace PokemonLZA{ namespace{ -// A region of the frame and the cells it is averaged into. On 1920x1080 -// (the user's design, 2026-11-10): the top band y 0-300 across; the left +// A region of the frame and the cells it is averaged into. On 1920x1080: +// the top band y 0-300 across; the left // side x 0-640 and the right side x 1280-1920, both y 300-1080. The player // (x 780-1200, y 330-1080) is in none of them. Each cell is 30x30 (top) or // 40x39 pixels (sides). @@ -60,6 +62,9 @@ constexpr double PLATFORM_TOP_MAX = 90.0; constexpr double PLATFORM_LEFT_MAX = 55.0; constexpr double PLATFORM_RIGHT_MAX = 120.0; +// The largest distance: every channel of every cell 255 apart, sqrt(3) * 255. +constexpr double MAX_PLATFORM_DISTANCE = 441.7; + using Cells = std::vector; // `region` of `screen`, averaged into its cells, row by row. @@ -80,10 +85,17 @@ Cells region_cells(const ImageViewRGB32& screen, const PlatformRegion& region){ return cells; } -// "rrggbb" to its color. -FloatPixel hex_pixel(const std::string& hex, size_t offset){ +// The "rrggbb" at `offset` of `hex` as a color. Throws ParseException for +// a character that is not a hex digit. +FloatPixel hex_pixel(std::string_view hex, size_t offset){ const auto channel = [&](size_t index){ - return (double)std::stoi(hex.substr(offset + index * 2, 2), nullptr, 16); + const std::string_view digits = hex.substr(offset + index * 2, 2); + int value = 0; + const auto [end, error] = std::from_chars(digits.data(), digits.data() + digits.size(), value, 16); + if (error != std::errc() || end != digits.data() + digits.size()){ + throw ParseException(std::format("plza-platform.json: \"{}\" is not a hex color channel.", digits)); + } + return (double)value; }; return FloatPixel(channel(0), channel(1), channel(2)); } @@ -146,8 +158,8 @@ double cells_rmsd(const Cells& reference, const Cells& cells){ PlatformDistances platform_distances(const ImageViewRGB32& screen){ const std::vector& reference = platform_reference(); - double distance[3] = {}; - for (size_t index = 0; index < 3; index++){ + double distance[std::size(PLATFORM_REGIONS)] = {}; + for (size_t index = 0; index < std::size(PLATFORM_REGIONS); index++){ distance[index] = cells_rmsd(reference[index], region_cells(screen, PLATFORM_REGIONS[index])); } return PlatformDistances{.top = distance[0], .left = distance[1], .right = distance[2]}; @@ -163,7 +175,7 @@ ImageRGB32 platform_reference_image(size_t width, size_t height){ ImageRGB32 image(width, height); image.fill(0xff000000); const std::vector& reference = platform_reference(); - for (size_t index = 0; index < 3; index++){ + for (size_t index = 0; index < std::size(PLATFORM_REGIONS); index++){ const PlatformRegion& region = PLATFORM_REGIONS[index]; const ImagePixelBox box = floatbox_to_pixelbox(width, height, region.box); for (size_t y = box.min_y; y < box.max_y; y++){ @@ -194,6 +206,11 @@ void PlatformArrivalDetector::make_overlays(VideoOverlaySet& items) const{ } bool PlatformArrivalDetector::detect(const ImageViewRGB32& screen){ + if (screen.width() == 0 || screen.height() == 0){ + // No frame: as far from the platform as a frame can be. + m_last = PlatformDistances{.top = MAX_PLATFORM_DISTANCE, .left = MAX_PLATFORM_DISTANCE, .right = MAX_PLATFORM_DISTANCE}; + return false; + } m_last = platform_distances(screen); return platform_matches(m_last); } From 508bafadc721fc4dd067fc9cb3a0eb5260e5645f Mon Sep 17 00:00:00 2001 From: dayfine Date: Fri, 13 Nov 2026 23:16:13 -0800 Subject: [PATCH 4/5] PokemonLZA AutoStory detectors: frames from CommandLineTests; comments say what each detector checks The console-frame tests read environment-variable folders that only existed on the lab machine, so CI always skipped them. They now read CommandLineTests/PokemonLZA/AutoStoryDetectors// (1920x1080 JPEG) and fail when the folder is missing. Fork CI checks out dayfine/CommandLineTests, which has these frames. The folder of 1.5 s frames with the autosave badge is gone, so the autosave detector keeps only its synthetic tests. The detector comments state what each one checks and the margins it keeps, without naming the runs, frames or dates they were measured on. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../workflows/cpp-ci-serial-programs-base.yml | 4 +- ...kemonLZA_AutoStoryAdvanceArrowDetector.cpp | 12 +- ...PokemonLZA_AutoStoryAdvanceArrowDetector.h | 2 +- .../PokemonLZA_AutoStoryAutosaveDetector.cpp | 18 +- ...kemonLZA_AutoStoryControlsHintDetector.cpp | 10 +- .../PokemonLZA_AutoStoryDetectors_Tests.cpp | 183 ++++++------------ .../PokemonLZA_AutoStoryDetectors_Tests.h | 4 +- ...LZA_AutoStoryMainMissionBannerDetector.cpp | 21 +- ...onLZA_AutoStoryMissionCompleteDetector.cpp | 13 +- ...onLZA_AutoStoryPlatformArrivalDetector.cpp | 24 +-- ...PokemonLZA_AutoStorySkipPromptDetector.cpp | 10 +- .../ReferenceData/plza-platform.json | 2 +- 12 files changed, 111 insertions(+), 192 deletions(-) diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index efb3c80a04..4edef34dd0 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -245,7 +245,9 @@ jobs: uses: actions/checkout@v7 if: inputs.run-tests with: - repository: 'PokemonAutomation/CommandLineTests' + # The fork's fixtures: upstream's plus the PokemonLZA AutoStory detector frames. + repository: 'dayfine/CommandLineTests' + ref: '5de4ff774f9141d904c4b5c557fa7b9dc85eaef1' path: 'CommandLineTests' # The unit tests assume the downloadable resources they need are already downloaded diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp index f7210122fe..8576fb0938 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp @@ -193,13 +193,11 @@ void AdvanceArrowDetector::make_overlays(VideoOverlaySet& items) const{ items.add(m_color, m_area); } -// Measured on 1920x1080 (console B, 2026-11-10; 01.01's runs 1 and 2, the -// lab's and the user's opening frames, 459 frames): the chevron's 35 -// frames (the ad's subtitles, white boxes, the opening's blue box) fill -// the V 0.53-0.70 with at most 1% stray pixels. Of the rest, the closest -// in size and shape is run 1's 0033 (a dark flyover blade, V 0.49 but -// 16% stray); the closest with nothing around it fill the V 0.43 or less -// and are out of the chevron's size. +// On 1920x1080 the chevron (on the ad's subtitles, on white boxes, on the +// opening's blue box) fills the V 0.53-0.70 with at most 1% stray pixels. +// The closest other shape in size and shape, a dark blade of scenery, +// fills the V 0.49 but has 16% stray pixels; shapes with nothing around +// them fill the V 0.43 or less and are out of the chevron's size. bool AdvanceArrowDetector::detect(const ImageViewRGB32& screen){ const double scale = double(screen.height()) / 1080; const ImageViewRGB32 area = extract_box_reference(screen, m_area); diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h index 8debae4975..0e5f34b804 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h @@ -3,7 +3,7 @@ * From: https://github.com/PokemonAutomation/ * * The "press A to go on" chevron at the bottom right of a dialogue box or - * of the boxless subtitles of 01.01's ad. + * of the boxless subtitles of the ad early in the story. * */ diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.cpp index e76391ea53..7aeadb757e 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.cpp @@ -56,18 +56,16 @@ void AutosaveDetector::make_overlays(VideoOverlaySet& items) const{ items.add(m_color, m_below); } -// Measured on the user's run (console B, 2026-11-10, frames 1.5 s apart): -// the badge's one frame (rec1_u132) has the icon 19% white and 48% cyan, -// nothing left of it. The only other frames over the icon bounds (u103, -// u104: a bright cutscene flash) are 70%+ white and 39%+ cyan left of it -// too; none of the opening's 56 lab frames come near. -// The badge is seen in Chinese only; with a shorter text (English +// On the badge the icon is 19% white and 48% cyan, with nothing bright +// left of it. A bright cutscene flash passes the icon's bounds too, but is +// 70%+ white and 39%+ cyan left of the icon as well. +// Only the Chinese badge is checked; with a shorter text (English // "Saving...") a right-aligned badge moves the icon right, so the icon is // looked for from where it is in Chinese to 240 px right of it. -// The 01.01 run (console B, 2026-11-10 13:00) took two flyover frames for -// the badge: 0033, a glowing blade at dx 40, and 0038, the sky at dx -// 152-172. Both glow on past the icon: the strips 14 px above and below it -// are 22%+ white or cyan there, 0% on u132; they must stay under 5%. +// Glowing scenery (a lit blade, a bright sky) can match the icon somewhere +// in that span, but glows on past it: the strips 14 px above and below the +// icon are 22%+ white or cyan there and empty around the badge; they must +// stay under 5%. bool AutosaveDetector::detect(const ImageViewRGB32& screen){ for (double dx = 0; dx <= 240; dx += 4){ if (autosave_icon_at(screen, m_icon, {m_left, m_above, m_below}, dx)){ diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.cpp index d1b545c486..5d5ad4b33d 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.cpp @@ -14,7 +14,7 @@ namespace NintendoSwitch{ namespace PokemonLZA{ -// Measured on 1920x1080 (console B, 2026-11-10): the card at x 946-971, +// On 1920x1080: the card at x 946-971, // y 868-901 with its spine at x 974-976; the "?" at x 953-965, y 876-894. ControlsHintDetector::ControlsHintDetector(Color color) : m_color(color) @@ -35,11 +35,9 @@ void ControlsHintDetector::make_overlays(VideoOverlaySet& items) const{ } } -// The hint's 13 frames (01.01 run 0001-0006, the user's u132-u138): the -// card 88%+ sky blue, the "?" at most 48%, nothing around it. Of the 376 -// frames without it, the closest are the 01.01 run's 0056 (the sky -// behind the girl: card 76%, "?" 100%, around 54%), 0060 (card 35%, -// around 27%) and the user's u126 (card 22%, around 32%). +// With the hint up the card is 88%+ sky blue, the "?" at most 48%, and +// nothing around it is. A sky behind the card comes closest (card 76%), +// but then the "?" and the area around it are blue too (100%, 54%). bool ControlsHintDetector::detect(const ImageViewRGB32& screen){ if (sky_blue_share(screen, m_card) < 0.80 || sky_blue_share(screen, m_glyph) > 0.70){ return false; diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp index ec11dad6ba..20c2e41354 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp @@ -2,36 +2,15 @@ * * From: https://github.com/PokemonAutomation/ * - * Synthetic frames always. The opening's frames are not in - * CommandLineTests yet; each set runs when its folder is in the - * environment and is skipped otherwise (console B, 1920x1080): - * - PLZA_OPENING_FRAMES: platform_ref.png and the lab's cutscene, - * 0001-*.png ... 0056-*.png; - * - PLZA_USER_OPENING_FRAMES: the user's run, rec1_u001.png ... - * rec1_u146.png, one per ~1.5 s, with the autosave badge (in Chinese); - * - PLZA_0101_RUN1_FRAMES: 01.01's first run (2026-11-10 13:00, in - * Chinese), 0001-*.png ... 0089-*.png: the platform with the controls - * hint, the walk to the plaza and the arrival scenes; - * - PLZA_0101_RUN2_FRAMES: 01.01's second run (2026-11-10 13:28, in - * Chinese), 0001-*.png ... 0115-*.png: the same, on to the ad's - * subtitles (0090-0115), where the run stopped; - * - PLZA_0101_RUN3_FRAMES: 01.01's third run (2026-11-10 13:49, in - * Chinese), 0001-*.png ... 0181-*.png: the same, through the ad and - * Pancham taking the bag, to the "MAIN MISSION 01" banner (0181); - * - PLZA_0101_RUN4_FRAMES: 01.01's fourth run (2026-11-10, in Chinese), - * 0001-*.png ... 0221-*.png, the "MAIN MISSION 01" banner on - * 0181-0189; - * - PLZA_0103_RUN2_FRAMES: 01.03's second run (2026-11-10, in Chinese), - * 0001-*.png ... 0139-*.png and mission01-complete-fullres.png: the - * "MAIN MISSION 01 COMPLETE" card (0045, 0046, the full-res frame), - * the "MAIN MISSION 02" banner (0133-0139). + * Synthetic frames, and console frames (1920x1080 JPEG) from + * CommandLineTests/PokemonLZA/AutoStoryDetectors/, one folder per scene + * sequence. * */ #include #include -#include #include #include #include @@ -40,6 +19,7 @@ #include "Common/Cpp/Filesystem/Filesystem.h" #include "Common/Cpp/Logging/AbstractLogger.h" #include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "PokemonLZA_AutoStoryAdvanceArrowDetector.h" #include "PokemonLZA_AutoStoryAutosaveDetector.h" @@ -216,7 +196,7 @@ enum class BannerIcon{ // The mission banner drawn on a grey scene: the yellow bar, and with // BannerIcon::DRAWN the icon's white hexagon (as a block) with the orange -// stroke, `dx` right of where the 01.01 banner frame has it. +// stroke, `dx` right of where the detector expects it. ImageRGB32 mission_banner_frame(size_t dx, BannerIcon icon){ ImageRGB32 image = solid_frame(0xff606060); fill_box(345, 410, 1590, 600, 0xfffad228, image); @@ -367,7 +347,7 @@ UnitTestResult check_synthetic_frame(const SyntheticFrameCase& test, Logger& log } -// Recorded frames, from folders named by environment variables. +// Console frames, from folders in CommandLineTests. // The number of `digits` digits at `offset` in a frame's file name; none // if they are not all digits. @@ -384,7 +364,8 @@ std::optional frame_number(std::string_view name, size_t offset, size_t dig return number; } -// A frame as a run read it, by its name ("0133-nothing+mission_banner.png"). +// What a frame shows, from the readings in its name +// ("0133-nothing+mission_banner.jpg"). DetectorReads as_named(std::string_view name){ const auto has = [name](std::string_view reading){ return name.find(reading) != std::string_view::npos; @@ -396,38 +377,17 @@ DetectorReads as_named(std::string_view name){ }; } -// The lab's opening: platform_ref.png is the platform, no cutscene frame -// 0001-0056 is; only 0050 shows the skip prompt. The platform reference -// was taken after the controls hint had gone. -std::optional lab_opening(std::string_view name){ - return DetectorReads{.platform = name == "platform_ref.png", .skip_prompt = name.starts_with("0050-")}; +// The opening cutscene up to the platform: only 0050 shows the skip +// prompt, and none is the platform. +std::optional opening(std::string_view name){ + return DetectorReads{.skip_prompt = name.starts_with("0050-")}; } -// The user's run rec1_u001-u146: the platform settles at u131 (u124-u130 -// are the fade in and the camera moving in; u128 already passes, top 77 -// under 90, as the platform it shows), the autosave badge shows on -// u132 only, the controls hint on u132-u138 (~10 s), no skip prompt. The -// train's blue box has its advance chevron on u116-u122. -// Later frames of the folder are not in the set. -std::optional user_opening(std::string_view name){ - const std::optional index = frame_number(name, 6, 3); - if (!index || *index > 146){ - return std::nullopt; - } - return DetectorReads{ - .platform = *index == 128 || *index >= 131, - .autosave = *index == 132, .controls_hint = *index >= 132 && *index <= 138, - .advance_arrow = *index >= 116 && *index <= 122, - }; -} - -// 01.01's first run: the platform (0001-0003, as the run read it) and the -// walk with the controls hint (0001-0006; it is gone by 0007), then the -// white fade, the title card, the text-free flyover, the starters and the -// girl, the tourist choice (0065) and the ad (0080-0089): no free control. -// No autosave: the run took flyover frames 0033 and 0038 for one. The -// advance chevron on the one box with its text done, 0040. -std::optional run1_arrival(std::string_view name){ +// The platform (as named) with the controls hint (0001-0006), the walk, +// the white fade, the title card, the text-free flyover, the starters, the +// tourist choice and the start of the ad: no autosave badge, the advance +// chevron only on the one box with its text done (0040). +std::optional arrival_to_ad(std::string_view name){ const std::optional index = frame_number(name, 0, 4); if (!index){ return std::nullopt; @@ -438,11 +398,10 @@ std::optional run1_arrival(std::string_view name){ }; } -// 01.01's second run: as the first, with the hint on 0001-0006, on to the -// ad (the logo 0088-0089, the president's subtitles 0090-0115). The -// advance chevron on the boxes with their text done (0044, 0049, 0053, -// 0078) and on the ad's subtitles once typed (0093-0115). -std::optional run2_arrival(std::string_view name){ +// As arrival_to_ad(), on to the ad's subtitles (0090-0115). The advance +// chevron on the boxes with their text done (0044, 0049, 0053, 0078) and +// on the subtitles once typed (0093-0115). +std::optional arrival_to_ad_subtitles(std::string_view name){ const std::optional index = frame_number(name, 0, 4); if (!index){ return std::nullopt; @@ -455,11 +414,10 @@ std::optional run2_arrival(std::string_view name){ }; } -// 01.01's third run: as the second, on through the ad and the bag scene. -// The platform, the controls hint and the advance chevron as the run read -// them (in each frame's name; checked by eye on the hint and the chevrons -// through 0115 in run 2); the mission banner on 0181 only, the last frame. -std::optional run3_arrival(std::string_view name){ +// As arrival_to_ad_subtitles(), on through the ad and the bag scene. The +// platform, the controls hint and the advance chevron as named; the +// mission banner on 0181 only, the last frame. +std::optional arrival_to_mission_banner(std::string_view name){ const std::optional index = frame_number(name, 0, 4); if (!index){ return std::nullopt; @@ -472,31 +430,27 @@ std::optional run3_arrival(std::string_view name){ }; } -// 01.01's fourth run: every detector as the run read it; the banner on -// 0181-0189 (checked by eye), no complete card. -std::optional run4_arrival(std::string_view name){ +// Every detector as named; the banner on 0181-0189, no complete card. +std::optional arrival_past_mission_banner(std::string_view name){ return as_named(name); } -// 01.03's second run: as the run read it, and the complete card on 0045, -// 0046 (named "nothing": the run had no detector for it) and the full-res -// frame; the "MAIN MISSION 02" banner on 0133-0139 is not the card. -std::optional run_0103_2(std::string_view name){ +// As named, and the complete card on 0045 and 0046 (named "nothing"); the +// next mission's banner (0133-0139) is not the card. +std::optional mission_complete_to_next_banner(std::string_view name){ DetectorReads want = as_named(name); - want.mission_complete = name.starts_with("0045-") || name.starts_with("0046-") || - name == "mission01-complete-fullres.png"; + want.mission_complete = name.starts_with("0045-") || name.starts_with("0046-"); return want; } struct FrameFolderCase{ std::string name; struct Input{ - std::string_view folder_variable; // The environment variable naming the folder. - std::string_view prefix; // The file names to read: *.png. + std::string_view folder; // In CommandLineTests/PokemonLZA/AutoStoryDetectors/. }; Input input; struct Expectation{ - size_t frames = 0; // How many files of the folder `reads` has an expectation for. + size_t frames = 0; // How many frames of the folder `reads` has an expectation for. // Each frame's reads, by file name; none for a file not in the set. // A detector it leaves unset must not fire. std::function(std::string_view name)> reads = {}; @@ -506,69 +460,48 @@ struct FrameFolderCase{ const FrameFolderCase FRAME_FOLDER_CASES[] = { { - .name = "the lab's opening frames", - .input = {.folder_variable = "PLZA_OPENING_FRAMES", .prefix = ""}, - // 0001-0056 and platform_ref.png. - .expectation = {.frames = 57, .reads = lab_opening}, - }, - { - .name = "the user's opening frames", - .input = {.folder_variable = "PLZA_USER_OPENING_FRAMES", .prefix = "rec1_u"}, - .expectation = {.frames = 146, .reads = user_opening}, + .name = "the opening cutscene", + .input = {.folder = "Opening"}, + .expectation = {.frames = 56, .reads = opening}, }, { - .name = "01.01 run 1's arrival frames", - .input = {.folder_variable = "PLZA_0101_RUN1_FRAMES", .prefix = "00"}, - .expectation = {.frames = 89, .reads = run1_arrival}, + .name = "the arrival to the ad", + .input = {.folder = "ArrivalToAd"}, + .expectation = {.frames = 89, .reads = arrival_to_ad}, }, { - .name = "01.01 run 2's arrival frames to the ad", - .input = {.folder_variable = "PLZA_0101_RUN2_FRAMES", .prefix = "0"}, - .expectation = {.frames = 115, .reads = run2_arrival}, + .name = "the arrival to the ad's subtitles", + .input = {.folder = "ArrivalToAdSubtitles"}, + .expectation = {.frames = 115, .reads = arrival_to_ad_subtitles}, }, { - .name = "01.01 run 3's arrival frames to the mission banner", - .input = {.folder_variable = "PLZA_0101_RUN3_FRAMES", .prefix = "0"}, - .expectation = {.frames = 181, .reads = run3_arrival}, + .name = "the arrival to the mission banner", + .input = {.folder = "ArrivalToMissionBanner"}, + .expectation = {.frames = 181, .reads = arrival_to_mission_banner}, }, { - .name = "01.01 run 4's arrival frames to the mission banner", - .input = {.folder_variable = "PLZA_0101_RUN4_FRAMES", .prefix = "0"}, - .expectation = {.frames = 221, .reads = run4_arrival}, + .name = "the arrival past the mission banner", + .input = {.folder = "ArrivalPastMissionBanner"}, + .expectation = {.frames = 221, .reads = arrival_past_mission_banner}, }, { - .name = "01.03 run 2's frames: the mission complete card and the MM02 banner", - .input = {.folder_variable = "PLZA_0103_RUN2_FRAMES", .prefix = ""}, - // 0001-0139 and mission01-complete-fullres.png. - .expectation = {.frames = 140, .reads = run_0103_2}, + .name = "the mission complete card to the next mission's banner", + .input = {.folder = "MissionCompleteToNextBanner"}, + .expectation = {.frames = 139, .reads = mission_complete_to_next_banner}, }, }; -// The folder in the environment variable `variable`; none when it is unset -// or not a folder. -std::optional frames_folder(std::string_view variable){ - const auto* value = std::getenv(std::string(variable).c_str()); - if (value == nullptr){ - return std::nullopt; - } - std::string folder(value); - if (!Filesystem::is_directory(Filesystem::Path(folder))){ - return std::nullopt; - } - return folder; -} - UnitTestResult check_frame_folder(const FrameFolderCase& test, Logger& logger){ - const std::optional folder = frames_folder(test.input.folder_variable); - if (!folder){ - return UnitTestResult(UnitTestResult::SKIPPED, std::format("{} is not a folder.", test.input.folder_variable)); + const Filesystem::Path folder(std::format("{}PokemonLZA/AutoStoryDetectors/{}/", UNIT_TEST_RESOURCE_PATH(), test.input.folder)); + if (!Filesystem::is_directory(folder)){ + return std::format("{} is not a folder.", folder.string()); } StoryDetectors detectors; size_t frames = 0; std::string failures; - for (const Filesystem::Path& path : Filesystem::list_directory(Filesystem::Path(*folder))){ + for (const Filesystem::Path& path : Filesystem::list_directory(folder)){ const std::string name = path.filename().string(); - if (!name.starts_with(test.input.prefix) || !name.ends_with(".png")){ + if (!name.ends_with(".jpg")){ continue; } const std::optional want = test.expectation.reads(name); @@ -584,7 +517,7 @@ UnitTestResult check_frame_folder(const FrameFolderCase& test, Logger& logger){ frames++; } if (frames != test.expectation.frames){ - return std::format("Expected {} frames in {}, found {}.", test.expectation.frames, *folder, frames); + return std::format("Expected {} frames in {}, found {}.", test.expectation.frames, folder.string(), frames); } if (!failures.empty()){ return std::format("Wrong on:{}", failures); @@ -610,7 +543,7 @@ class Test_SyntheticFrame : public UnitTest{ class Test_FrameFolder : public UnitTest{ public: explicit Test_FrameFolder(const FrameFolderCase& test) - : UnitTest(std::format("PokemonLZA::AutoStoryDetectors - {} ({})", test.name, test.input.folder_variable)) + : UnitTest(std::format("PokemonLZA::AutoStoryDetectors - console frames: {}", test.name)) , m_test(test) {} virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h index 3bae55b920..bda582d5d1 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h @@ -15,9 +15,7 @@ namespace PokemonLZA{ // Registers the tests of the AutoStory detectors (Detectors/): synthetic -// frames always; the opening's frames from the folders in -// PLZA_OPENING_FRAMES and PLZA_USER_OPENING_FRAMES, and the runs' frames -// from PLZA_0101_RUN1_FRAMES and the others, each skipped when unset. +// frames, and console frames from CommandLineTests. void add_tests_AutoStoryDetectors(UnitTestDatabase& database); diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.cpp index d855d6fe50..0ea79661f1 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.cpp @@ -14,11 +14,10 @@ namespace NintendoSwitch{ namespace PokemonLZA{ -// Measured on 1920x1080 (console B, 2026-11-10, 01.01 run 3's 0181): the -// bar's yellow fills x 430-820 and x 1100-1500 at y 425-585 (left and -// right of the name), none of it 15 px above or below the bar; the icon's -// hexagon is white at x 700-710 and 724-734, y 337-358, with the "!"'s -// orange stroke at x 716-719 between them. +// On 1920x1080: the bar's yellow fills x 430-820 and x 1100-1500 at +// y 425-585 (left and right of the name), none of it 15 px above or +// below the bar; the icon's hexagon is white at x 700-710 and 724-734, +// y 337-358, with the "!"'s orange stroke at x 716-719 between them. MainMissionBannerDetector::MainMissionBannerDetector(Color color) : m_color(color) , m_bar_left(430.0 / 1920, 425.0 / 1080, 390.0 / 1920, 160.0 / 1080) @@ -40,13 +39,11 @@ void MainMissionBannerDetector::make_overlays(VideoOverlaySet& items) const{ items.add(m_color, m_icon_stroke); } -// Measured on 01.01's runs 1-3, the lab's and the user's opening frames -// and the other PLZA lab captures (746 frames, console B, 2026-11-10): -// the banner's one frame (run 3's 0181) has both bar boxes 100% yellow, -// the strips around it 0-1%, and the icon's three boxes 100% at dx 0. The -// closest others: the bar boxes 8% at most, the icon's weakest box 27% at -// most (run 2's 0032, the flyover). The icon is looked for 60 px either side, as a longer -// or shorter header moves it. +// On the banner both bar boxes are 100% yellow, the strips around the bar +// 0-1%, and the icon's three boxes 100% at dx 0. Other screens have the +// bar boxes 8% yellow at most and the icon's weakest box 27% at most (a +// city flyover). The icon is looked for 60 px either side, as a longer or +// shorter header moves it. bool MainMissionBannerDetector::detect(const ImageViewRGB32& screen){ if (banner_yellow_share(screen, m_bar_left) < 0.6 || banner_yellow_share(screen, m_bar_right) < 0.6){ return false; diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.cpp index 85d58c7d82..5485103435 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.cpp @@ -14,8 +14,7 @@ namespace NintendoSwitch{ namespace PokemonLZA{ -// Measured on 1920x1080 (console B, 2026-11-10, 01.03 run 2's -// mission01-complete-fullres.png): the word's green fills x 615-1300, +// On 1920x1080: the word's green fills x 615-1300, // y 271-349, none of it 20 px above or below; the bar's yellow fills // y 390-585, read left and right of the name as for the banner. MissionCompleteDetector::MissionCompleteDetector(Color color) @@ -35,13 +34,9 @@ void MissionCompleteDetector::make_overlays(VideoOverlaySet& items) const{ items.add(m_color, m_bar_right); } -// Measured on every PLZA lab capture (1109 frames, console B, 2026-11-10): -// the card's three frames (01.03 run 2's 0045, 0046 and the full-res -// one) have the word 70% green, nothing green around it and both bar -// boxes 100% yellow. The closest others: the word 11% green at most -// (plza-platform's boxes.png, no bar), and with the bar, the mission -// banners (01.01 run 3's 0181, run 4's 0182-0189, 01.03 run 2's -// 0133-0139) at 2% at most. +// On the card the word is 70% green, nothing around it is green and both +// bar boxes are 100% yellow. Other screens have the word 11% green at +// most; the main mission banners, which have the same bar, 2% at most. bool MissionCompleteDetector::detect(const ImageViewRGB32& screen){ if (complete_green_share(screen, m_word) < 0.40){ return false; diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp index 18f8f1fa52..9f4a95bbf4 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp @@ -45,19 +45,19 @@ const PlatformRegion PLATFORM_REGIONS[] = { {"right", {1280.0 / 1920, 300.0 / 1080, 640.0 / 1920, 780.0 / 1080}, 16, 20}, }; -// The bounds of platform_matches(), from the opening's frames (console B, -// 1920x1080: the platform reference and the 56 cutscene frames before -// it, measured with this code's cell averages): -// - the closest cutscene frame is 0056, the train's interior just before -// the platform: top 112.8, left 82.6, right 71.9; the next are the -// city flyovers 0022-0024 at top 134.7+, left 95.6+; +// The bounds of platform_matches(), from this code's cell averages on +// 1920x1080, against the platform reference: +// - the closest cutscene scene, the train's interior just before the +// platform: top 112.8, left 82.6, right 71.9; the next, city flyovers: +// top 134.7+, left 95.6+; // - the reference against itself moved 16 px: top 42.2, left 20.8, -// right 21.2; 10% darker: top 24.6, left 19.9, right 16.4. -// - the platform after loading its autosave (camera turned a little, -// 2026-11-10): top 67.9-68.1, left 25.8-41.7, right 16.5-16.9. -// So the bounds sit between the loaded view and 0056: top 90 (23 below -// 0056), left 55 (27 below). The right side holds a walking NPC and the companion, so it only -// has to be roughly the platform (120: a menu or another place is 140+). +// right 21.2; 10% darker: top 24.6, left 19.9, right 16.4; +// - the platform after loading its autosave (camera turned a little): +// top 67.9-68.1, left 25.8-41.7, right 16.5-16.9. +// So the bounds sit between the loaded view and the train's interior: +// top 90 (23 below it), left 55 (27 below). The right side holds a walking +// NPC and the companion, so it only has to be roughly the platform (120: +// a menu or another place is 140+). constexpr double PLATFORM_TOP_MAX = 90.0; constexpr double PLATFORM_LEFT_MAX = 55.0; constexpr double PLATFORM_RIGHT_MAX = 120.0; diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.cpp index ac1ded93af..2a3a013043 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStorySkipPromptDetector.cpp @@ -15,7 +15,7 @@ namespace NintendoSwitch{ namespace PokemonLZA{ -// The badge on 1920x1080 (frame 0050 of the opening): the white (+) icon +// The badge on 1920x1080: the white (+) icon // at x 1564-1586, y 128-150; the text x 1630-1875, y 122-158; the strip // under the text x 1640-1880, y 161-169. SkipPromptDetector::SkipPromptDetector(Color color) @@ -31,10 +31,10 @@ void SkipPromptDetector::make_overlays(VideoOverlaySet& items) const{ items.add(m_color, m_strip); } -// On frame 0050 the icon is 70% white, the text 24%, and the strip is -// dark and even (average 44, deviation 2). No other opening frame has the -// icon over 40% white with a dark strip: the brightest (0044, a white -// flash) has the icon at 38% and the strip at 220. +// On the badge the icon is 70% white, the text 24%, and the strip is dark +// and even (average 44, deviation 2). No other scene of the opening has +// the icon over 40% white with a dark strip: the brightest, a white flash, +// has the icon at 38% and the strip at 220. bool SkipPromptDetector::detect(const ImageViewRGB32& screen){ if (white_share(screen, m_icon) < 0.40){ return false; diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/ReferenceData/plza-platform.json b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/ReferenceData/plza-platform.json index 1e673d8cca..1d59ee8577 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/ReferenceData/plza-platform.json +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/ReferenceData/plza-platform.json @@ -1,5 +1,5 @@ { - "source": "platform_ref.png: console B, the station platform right after the opening cutscene (default camera)", + "source": "the station platform right after the opening cutscene, default camera, 1920x1080", "regions": { "top": { "cols": 64, From 29a3db7b8a28fffeb195600cd242063d1740653a Mon Sep 17 00:00:00 2001 From: dayfine Date: Sat, 14 Nov 2026 00:18:52 -0800 Subject: [PATCH 5/5] PLZA AutoStory: advance arrow reads the JPEG test frames The console frames moved to CommandLineTests as quality-80 JPEG. JPEG keeps colour at half resolution, so the white chevron over the player's orange shorts (ArrivalPastMissionBanner/0138) lost blue below 235 at its edges and split into pieces. White now asks blue for 220 (190-230 all work), and stray pixels are allowed up to 9%: on JPEG the chevron has at most 6%, the closest other shape 13%. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- ...okemonLZA_AutoStoryAdvanceArrowDetector.cpp | 18 +++++++++++------- .../PokemonLZA_AutoStoryAdvanceArrowDetector.h | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp index 8576fb0938..08d79f17ff 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp @@ -166,7 +166,7 @@ bool chevron_blob(const BlobImage& image, uint32_t label, double scale){ if (aspect < 1.4 || aspect > 2.1){ return false; } - return stray_share(image, label, size_t(4 * scale + 0.5)) <= 0.05 && v_match(image, label) >= 0.45; + return stray_share(image, label, size_t(4 * scale + 0.5)) <= 0.09 && v_match(image, label) >= 0.45; } // Whether a blob of the pixels of `area` that `in` is the chevron. @@ -193,16 +193,20 @@ void AdvanceArrowDetector::make_overlays(VideoOverlaySet& items) const{ items.add(m_color, m_area); } -// On 1920x1080 the chevron (on the ad's subtitles, on white boxes, on the -// opening's blue box) fills the V 0.53-0.70 with at most 1% stray pixels. -// The closest other shape in size and shape, a dark blade of scenery, -// fills the V 0.49 but has 16% stray pixels; shapes with nothing around -// them fill the V 0.43 or less and are out of the chevron's size. +// On 1920x1080 frames saved as JPEG (quality 80) the chevron (on the ad's +// subtitles, on white boxes, on the opening's blue box) fills the V +// 0.51-0.71 with at most 6% stray pixels. The closest other shape in size +// and shape, a dark blade of scenery, fills the V 0.48 but has 13% stray +// pixels, so stray pixels are allowed up to 9%; shapes with nothing +// around them fill the V 0.43 or less and are out of the chevron's size. +// JPEG keeps colour at half resolution, so a white chevron over orange +// (the player's shorts) loses blue below 235 at its edges: white asks +// for 220 in blue (any floor from 190 to 230 reads every frame right). bool AdvanceArrowDetector::detect(const ImageViewRGB32& screen){ const double scale = double(screen.height()) / 1080; const ImageViewRGB32 area = extract_box_reference(screen, m_area); const bool white = chevron_in(area, scale, [](Color pixel){ - return pixel.red() >= 235 && pixel.green() >= 235 && pixel.blue() >= 235; + return pixel.red() >= 235 && pixel.green() >= 235 && pixel.blue() >= 220; }); return white || chevron_in(area, scale, [](Color pixel){ return pixel.red() <= 110 && pixel.green() <= 110 && pixel.blue() <= 110; diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h index 0e5f34b804..fb27998a3b 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h @@ -22,7 +22,7 @@ namespace PokemonLZA{ // The "press A to go on" chevron (a V about 32x19 on 1920x1080) at the // right end of a dialogue box and of boxless subtitles: dark on a white // box, white on a blue box and on the ad's subtitles. Found by shape in -// x 1370-1480, y 930-1050: a blob of white (every channel 235+) or dark +// x 1370-1480, y 930-1050: a blob of white (red and green 235+, blue 220+) or dark // (every channel 110 or under) pixels the size of the chevron, filling a // V, with no other pixel of its colour around it. Text is not read, so // any language matches.