Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
9a4f754
PokemonLZA: AutoStory: detectors and their tests
dayfine Nov 12, 2026
fc8ed83
Merge branch 'autostory/engine' into plza/story-detectors
dayfine Nov 12, 2026
38272ef
AutoStory: forward-declare Pokemon::ScreenEvents in Pokemon_StoryBack…
dayfine Nov 12, 2026
5c54fb8
Merge branch 'autostory/engine' into plza/story-detectors
dayfine Nov 12, 2026
cd22907
Merge branch 'autostory/engine' into plza/story-detectors
dayfine Nov 12, 2026
5f76667
Merge remote-tracking branch 'origin/autostory/engine' into plza/stor…
dayfine Nov 12, 2026
d586e97
Merge remote-tracking branch 'origin/autostory/engine' into plza/stor…
dayfine Nov 12, 2026
3df2e38
Merge origin/autostory/engine into plza/story-detectors
dayfine Nov 12, 2026
4b30095
Merge remote-tracking branch 'origin/autostory/engine' into plza/stor…
dayfine Nov 13, 2026
504204c
Merge branch 'autostory/engine' into plza/story-detectors
dayfine Nov 13, 2026
d7a2b54
Merge branch 'autostory/engine' into plza/story-detectors
dayfine Nov 13, 2026
1f45fd1
Merge remote-tracking branch 'origin/autostory/engine' into HEAD
dayfine Nov 13, 2026
cfd16ca
Merge origin/autostory/engine into plza/story-detectors: #45 review f…
dayfine Nov 13, 2026
75f57f6
Merge origin/autostory/engine into plza/story-detectors: #45 review f…
dayfine Nov 13, 2026
ed3e1d0
Merge origin/autostory/engine into plza/story-detectors: shared check…
dayfine Nov 13, 2026
9afe706
Merge origin/autostory/story-breakpoints into plza/story-detectors
dayfine Nov 13, 2026
8d9d791
Merge origin/autostory/story-breakpoints into plza/story-detectors
dayfine Nov 13, 2026
733d702
Merge origin/autostory/story-breakpoints into plza/story-detectors
dayfine Nov 13, 2026
e02f4ea
Merge remote-tracking branch 'origin/autostory/story-breakpoints' int…
dayfine Nov 14, 2026
53c97a6
Merge autostory/story-breakpoints into plza/story-detectors
dayfine Nov 14, 2026
44951e4
PokemonLZA AutoStory detectors: test tables as name, input, expectati…
dayfine Nov 14, 2026
508bafa
PokemonLZA AutoStory detectors: frames from CommandLineTests; comment…
dayfine Nov 14, 2026
29a3db7
PLZA AutoStory: advance arrow reads the JPEG test frames
dayfine Nov 14, 2026
cba62bf
Merge autostory/story-breakpoints into plza/story-detectors
dayfine Nov 14, 2026
bfea2d5
Merge origin/autostory/story-breakpoints into plza/story-detectors
dayfine Nov 14, 2026
1e4cd69
Merge origin/autostory/story-breakpoints into plza/story-detectors
dayfine Nov 15, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions SerialPrograms/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,8 @@ add_library(SerialProgramsLib STATIC ${CORE_LIBRARY_SOURCES} ${LIBRARY_SOURCES})
include(cmake/EmbedTextFiles.cmake)
pa_embed_text_files(SerialProgramsLib Source/Pokemon/Navigation/TestMaps
Pokemon/Navigation/Pokemon_NavigationTestMaps.h PokemonAutomation::Pokemon navigation_test_map_json)
pa_embed_text_files(SerialProgramsLib Source/VideoGames/PokemonLZA/Programs/AutoStory/ReferenceData
VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h PokemonAutomation::NintendoSwitch::PokemonLZA autostory_reference_json)


# Create binary target
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#include "Inference/Map/PokemonLZA_MapIconDetector.h"
#include "Inference/PokemonLZA_OverworldPartySelectionDetector.h"
#include "Inference/PokemonLZA_SelectionArrowDetector.h"
#include "Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h"

namespace PokemonAutomation{
namespace NintendoSwitch{
Expand All @@ -41,6 +42,7 @@ void add_tests(UnitTestDatabase& database){
add_tests_MapIconDetector(database);
add_tests_OverworldPartySelectionDetector(database);
add_tests_SelectionArrowDetector(database);
add_tests_AutoStoryDetectors(database);

}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
/* PokemonLZA AutoStory - Advance arrow detector
*
* From: https://github.com/PokemonAutomation/
*
*/

#include <algorithm>
#include <cmath>
#include <stdint.h>
#include <utility>
#include <vector>
#include "CommonFramework/ImageTypes/ImageViewRGB32.h"
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
#include "PokemonLZA_AutoStoryPixelShares.h"
#include "PokemonLZA_AutoStoryAdvanceArrowDetector.h"

namespace PokemonAutomation{
namespace NintendoSwitch{
namespace PokemonLZA{


namespace{

// A 4-connected blob of a mask: its bounds and its pixel count.
struct Blob{
size_t min_x = 0;
size_t min_y = 0;
size_t max_x = 0;
size_t max_y = 0;
size_t count = 0;
};

// An area's mask split into blobs: each pixel's blob label (0 outside the
// mask; blob i has label i + 1).
struct BlobImage{
size_t width = 0;
size_t height = 0;
std::vector<uint32_t> labels;
std::vector<Blob> blobs;

uint32_t label(size_t x, size_t y) const{
return labels[y * width + x];
}
};

// A mask pixel no blob has taken yet.
constexpr uint32_t UNLABELLED = UINT32_MAX;

// Labels the blob of mask pixels 4-connected to (x, y).
void flood_blob(size_t x, size_t y, BlobImage& image){

Check warning on line 50 in SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

PokemonAutomation::NintendoSwitch::PokemonLZA::flood_blob( size_t x , size_t y , BlobImage & image): function_lines=35 (advisory above 25, limit 50)
const uint32_t label = (uint32_t)image.blobs.size() + 1;
Blob blob{x, y, x, y, 0};
std::vector<std::pair<size_t, size_t>> stack{{x, y}};
image.labels[y * image.width + x] = label;
const auto take = [&](size_t nx, size_t ny){
uint32_t& here = image.labels[ny * image.width + nx];
if (here == UNLABELLED){
here = label;
stack.emplace_back(nx, ny);
}
};
while (!stack.empty()){
const auto [cx, cy] = stack.back();
stack.pop_back();
blob.count++;
blob.min_x = std::min(blob.min_x, cx);
blob.max_x = std::max(blob.max_x, cx);
blob.min_y = std::min(blob.min_y, cy);
blob.max_y = std::max(blob.max_y, cy);
if (cx > 0){
take(cx - 1, cy);
}
if (cx + 1 < image.width){
take(cx + 1, cy);
}
if (cy > 0){
take(cx, cy - 1);
}
if (cy + 1 < image.height){
take(cx, cy + 1);
}
}
image.blobs.push_back(blob);
}

// The pixels of `area` that `in`, split into blobs.
template <typename In>
BlobImage find_blobs(const ImageViewRGB32& area, In&& in){
BlobImage image;
image.width = area.width();
image.height = area.height();
image.labels.assign(image.width * image.height, 0);
for (size_t y = 0; y < image.height; y++){
for (size_t x = 0; x < image.width; x++){
if (in(Color(area.pixel(x, y)))){
image.labels[y * image.width + x] = UNLABELLED;
}
}
}
for (size_t y = 0; y < image.height; y++){
for (size_t x = 0; x < image.width; x++){
if (image.label(x, y) == UNLABELLED){
flood_blob(x, y, image);
}
}
}
return image;
}

// How well blob `label` fills a V across its bounds: its overlap with the
// V over their union. Each arm runs from 10% in at the top to the middle
// at the bottom, 22% of the width across.
double v_match(const BlobImage& image, uint32_t label){
const Blob& blob = image.blobs[label - 1];
const double width = double(blob.max_x - blob.min_x + 1);
const double height = double(blob.max_y - blob.min_y + 1);
size_t both = 0;
size_t either = 0;
for (size_t y = blob.min_y; y <= blob.max_y; y++){
const double v = (double(y - blob.min_y) + 0.5) / height;
for (size_t x = blob.min_x; x <= blob.max_x; x++){
const double u = (double(x - blob.min_x) + 0.5) / width;
const bool in_v = std::abs(std::abs(u - 0.5) - (0.40 * (1 - v) + 0.03)) <= 0.11;
const bool in_blob = image.label(x, y) == label;
if (in_v && in_blob){
both++;
}
if (in_v || in_blob){
either++;
}
}
}
return either == 0 ? 0 : double(both) / double(either);
}

// The mask pixels of other blobs within `margin` of blob `label`'s
// bounds, per pixel of the blob.
double stray_share(const BlobImage& image, uint32_t label, size_t margin){
const Blob& blob = image.blobs[label - 1];
const size_t min_x = blob.min_x > margin ? blob.min_x - margin : 0;
const size_t min_y = blob.min_y > margin ? blob.min_y - margin : 0;
const size_t max_x = std::min(blob.max_x + margin, image.width - 1);
const size_t max_y = std::min(blob.max_y + margin, image.height - 1);
size_t stray = 0;
for (size_t y = min_y; y <= max_y; y++){
for (size_t x = min_x; x <= max_x; x++){
const uint32_t here = image.label(x, y);
if (here != 0 && here != label){
stray++;
}
}
}
return double(stray) / double(blob.count);
}

// Whether blob `label` is the chevron, on a screen `scale` times 1080p:
// 32x19 measured, 30-33 x 18-20 on every frame with it.
bool chevron_blob(const BlobImage& image, uint32_t label, double scale){
const Blob& blob = image.blobs[label - 1];
const double width = double(blob.max_x - blob.min_x + 1) / scale;
const double height = double(blob.max_y - blob.min_y + 1) / scale;
if (width < 26 || width > 40 || height < 15 || height > 24){
return false;
}
const double aspect = width / height;
if (aspect < 1.4 || aspect > 2.1){
return false;
}
return stray_share(image, label, size_t(4 * scale + 0.5)) <= 0.09 && v_match(image, label) >= 0.45;
}

// Whether a blob of the pixels of `area` that `in` is the chevron.
template <typename In>
bool chevron_in(const ImageViewRGB32& area, double scale, In&& in){
const BlobImage image = find_blobs(area, in);
for (uint32_t label = 1; label <= image.blobs.size(); label++){
if (chevron_blob(image, label, scale)){
return true;
}
}
return false;
}

} // namespace


AdvanceArrowDetector::AdvanceArrowDetector(Color color)
: m_color(color)
, m_area(1370.0 / 1920, 930.0 / 1080, 110.0 / 1920, 120.0 / 1080)
{}

void AdvanceArrowDetector::make_overlays(VideoOverlaySet& items) const{
items.add(m_color, m_area);
}

// On 1920x1080 frames saved as JPEG (quality 80) the chevron (on the ad's
// subtitles, on white boxes, on the opening's blue box) fills the V
// 0.51-0.71 with at most 6% stray pixels. The closest other shape in size
// and shape, a dark blade of scenery, fills the V 0.48 but has 13% stray
// pixels, so stray pixels are allowed up to 9%; shapes with nothing
// around them fill the V 0.43 or less and are out of the chevron's size.
// JPEG keeps colour at half resolution, so a white chevron over orange
// (the player's shorts) loses blue below 235 at its edges: white asks
// for 220 in blue (any floor from 190 to 230 reads every frame right).
bool AdvanceArrowDetector::detect(const ImageViewRGB32& screen){
const double scale = double(screen.height()) / 1080;
const ImageViewRGB32 area = extract_box_reference(screen, m_area);
const bool white = chevron_in(area, scale, [](Color pixel){
return pixel.red() >= 235 && pixel.green() >= 235 && pixel.blue() >= 220;
});
return white || chevron_in(area, scale, [](Color pixel){
return pixel.red() <= 110 && pixel.green() <= 110 && pixel.blue() <= 110;
});
}


} // namespace PokemonLZA
} // namespace NintendoSwitch
} // namespace PokemonAutomation
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/* PokemonLZA AutoStory - Advance arrow detector
*
* From: https://github.com/PokemonAutomation/
*
* The "press A to go on" chevron at the bottom right of a dialogue box or
* of the boxless subtitles of the ad early in the story.
*
*/

#ifndef PokemonAutomation_PokemonLZA_AutoStoryAdvanceArrowDetector_H
#define PokemonAutomation_PokemonLZA_AutoStoryAdvanceArrowDetector_H

#include "Common/Cpp/Color.h"
#include "CommonFramework/ImageTools/ImageBoxes.h"
#include "CommonTools/VisualDetector.h"

namespace PokemonAutomation{
namespace NintendoSwitch{
namespace PokemonLZA{


// The "press A to go on" chevron (a V about 32x19 on 1920x1080) at the
// right end of a dialogue box and of boxless subtitles: dark on a white
// box, white on a blue box and on the ad's subtitles. Found by shape in
// x 1370-1480, y 930-1050: a blob of white (red and green 235+, blue 220+) or dark
// (every channel 110 or under) pixels the size of the chevron, filling a
// V, with no other pixel of its colour around it. Text is not read, so
// any language matches.
class AdvanceArrowDetector : public StaticScreenDetector{
public:
explicit AdvanceArrowDetector(Color color = COLOR_GREEN);

virtual void make_overlays(VideoOverlaySet& items) const override;
virtual bool detect(const ImageViewRGB32& screen) override;

private:
Color m_color;
ImageFloatBox m_area;
};



} // namespace PokemonLZA
} // namespace NintendoSwitch
} // namespace PokemonAutomation
#endif
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
/* PokemonLZA AutoStory - Autosave detector
*
* From: https://github.com/PokemonAutomation/
*
*/

#include <initializer_list>
#include "CommonFramework/ImageTypes/ImageViewRGB32.h"
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
#include "PokemonLZA_AutoStoryPixelShares.h"
#include "PokemonLZA_AutoStoryAutosaveDetector.h"

namespace PokemonAutomation{
namespace NintendoSwitch{
namespace PokemonLZA{


namespace{

// The icon, and the strips around it (left, above, below) clear of white
// and cyan, `dx` pixels (of 1920) right of where the Chinese badge has
// them.
bool autosave_icon_at(
const ImageViewRGB32& screen, const ImageFloatBox& icon,
std::initializer_list<ImageFloatBox> around, double dx
){
const ImageFloatBox icon_here = shifted(icon, dx);
const double white = white_share(screen, icon_here);
if (white < 0.08 || white > 0.40 || cyan_share(screen, icon_here) < 0.25){
return false;
}
for (const ImageFloatBox& box : around){
const ImageFloatBox here = shifted(box, dx);
if (white_share(screen, here) >= 0.05 || cyan_share(screen, here) >= 0.05){
return false;
}
}
return true;
}

} // namespace


AutosaveDetector::AutosaveDetector(Color color)
: m_color(color)
, m_icon(1603.0 / 1920, 73.0 / 1080, 27.0 / 1920, 31.0 / 1080)
, m_left(1570.0 / 1920, 73.0 / 1080, 28.0 / 1920, 31.0 / 1080)
, m_above(1603.0 / 1920, 56.0 / 1080, 27.0 / 1920, 14.0 / 1080)
, m_below(1603.0 / 1920, 108.0 / 1080, 27.0 / 1920, 14.0 / 1080)
{}

void AutosaveDetector::make_overlays(VideoOverlaySet& items) const{
items.add(m_color, m_icon);
items.add(m_color, m_left);
items.add(m_color, m_above);
items.add(m_color, m_below);
}

// On the badge the icon is 19% white and 48% cyan, with nothing bright
// left of it. A bright cutscene flash passes the icon's bounds too, but is
// 70%+ white and 39%+ cyan left of the icon as well.
// Only the Chinese badge is checked; with a shorter text (English
// "Saving...") a right-aligned badge moves the icon right, so the icon is
// looked for from where it is in Chinese to 240 px right of it.
// Glowing scenery (a lit blade, a bright sky) can match the icon somewhere
// in that span, but glows on past it: the strips 14 px above and below the
// icon are 22%+ white or cyan there and empty around the badge; they must
// stay under 5%.
bool AutosaveDetector::detect(const ImageViewRGB32& screen){
for (double dx = 0; dx <= 240; dx += 4){
if (autosave_icon_at(screen, m_icon, {m_left, m_above, m_below}, dx)){
return true;
}
}
return false;
}


} // namespace PokemonLZA
} // namespace NintendoSwitch
} // namespace PokemonAutomation
Loading
Loading