Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions SerialPrograms/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,11 @@ include(cmake/CoreLib.cmake)
# Create the GUI library
add_library(SerialProgramsLib STATIC ${CORE_LIBRARY_SOURCES} ${LIBRARY_SOURCES})

# Data files built into the program (see cmake/EmbedTextFiles.cmake).
include(cmake/EmbedTextFiles.cmake)
pa_embed_text_files(SerialProgramsLib Source/VideoGames/PokemonLZA/Programs/AutoStory/ReferenceData
VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h PokemonAutomation::NintendoSwitch::PokemonLZA autostory_reference_json)


# Create binary target
if (APPLE)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include "Inference/PokemonLZA_OverworldPartySelectionDetector.h"
#include "Inference/PokemonLZA_PartyBarDetector.h"
#include "Inference/PokemonLZA_SelectionArrowDetector.h"
#include "Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h"

namespace PokemonAutomation{
namespace NintendoSwitch{
Expand All @@ -45,6 +46,7 @@ void add_tests(UnitTestDatabase& database){
add_tests_OverworldPartySelectionDetector(database);
add_tests_PartyBarDetector(database);
add_tests_SelectionArrowDetector(database);
add_tests_AutoStoryDetectors(database);

}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
/* PokemonLZA AutoStory - Advance arrow detector
*
* From: https://github.com/PokemonAutomation/
*
*/

#include <algorithm>
#include <cmath>
#include <memory>
#include "CommonFramework/ImageTypes/BinaryImage.h"
#include "CommonFramework/ImageTypes/ImageViewRGB32.h"
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
#include "CommonTools/Images/BinaryImage_FilterRgb32.h"
#include "Kernels/Waterfill/Kernels_Waterfill_Session.h"
#include "PokemonLZA_AutoStoryAdvanceArrowDetector.h"

namespace PokemonAutomation{
namespace NintendoSwitch{
namespace PokemonLZA{

using namespace Kernels::Waterfill;


namespace{

// How well `blob` (a Waterfill object kept with its pixels) fills a V
// across its bounds: its overlap with the V over their union. Each arm
// runs from 10% in at the top to the middle at the bottom, 22% of the
// width across.
double v_match(const WaterfillObject& blob){
const double width = double(blob.width());
const double height = double(blob.height());
size_t both = 0;
size_t either = 0;
for (size_t y = blob.min_y; y < blob.max_y; y++){
const double v = (double(y - blob.min_y) + 0.5) / height;
for (size_t x = blob.min_x; x < blob.max_x; x++){
const double u = (double(x - blob.min_x) + 0.5) / width;
const bool in_v = std::abs(std::abs(u - 0.5) - (0.40 * (1 - v) + 0.03)) <= 0.11;
const bool in_blob = blob.object->get(x, y);
if (in_v && in_blob){
both++;
}
if (in_v || in_blob){
either++;
}
}
}
return either == 0 ? 0 : double(both) / double(either);
}

// The pixels of `mask` outside `blob` within `margin` of its bounds, per
// pixel of the blob. The blob lies inside the window, so they are the
// window's pixels less its own.
double stray_share(const PackedBinaryMatrix& mask, const WaterfillObject& blob, size_t margin){
const size_t min_x = blob.min_x > margin ? blob.min_x - margin : 0;
const size_t min_y = blob.min_y > margin ? blob.min_y - margin : 0;
const size_t max_x = std::min(blob.max_x + margin, mask.width());
const size_t max_y = std::min(blob.max_y + margin, mask.height());
size_t in_window = 0;
for (size_t y = min_y; y < max_y; y++){
for (size_t x = min_x; x < max_x; x++){
if (mask.get(x, y)){
in_window++;
}
}
}
return double(in_window - blob.area) / double(blob.area);
}

// Whether `blob` of `mask` is the chevron, on a screen `scale` times 1080p:
// 32x19 measured, 30-33 x 18-20 on every frame with it.
bool chevron_blob(const PackedBinaryMatrix& mask, const WaterfillObject& blob, double scale){
const double width = double(blob.width()) / scale;
const double height = double(blob.height()) / scale;
if (width < 26 || width > 40 || height < 15 || height > 24){
return false;
}
const double aspect = width / height;
if (aspect < 1.4 || aspect > 2.1){
return false;
}
return stray_share(mask, blob, size_t(4 * scale + 0.5)) <= 0.09 && v_match(blob) >= 0.45;
}

// Whether a 4-connected blob of the pixels of `area` in the colour range
// `mins`-`maxs` is the chevron.
bool chevron_in(const ImageViewRGB32& area, double scale, uint32_t mins, uint32_t maxs){
const PackedBinaryMatrix mask = compress_rgb32_to_binary_range(area, mins, maxs);
PackedBinaryMatrix unfilled = mask.copy();
const std::unique_ptr<WaterfillSession> session = make_WaterfillSession(unfilled);
const std::unique_ptr<WaterfillIterator> blobs = session->make_iterator(1);
WaterfillObject blob;
while (blobs->find_next(blob, true)){
if (chevron_blob(mask, blob, scale)){
return true;
}
}
return false;
}

} // namespace


AdvanceArrowDetector::AdvanceArrowDetector(Color color)
: m_color(color)
, m_area(1370.0 / 1920, 930.0 / 1080, 110.0 / 1920, 120.0 / 1080)
{}

void AdvanceArrowDetector::make_overlays(VideoOverlaySet& items) const{
items.add(m_color, m_area);
}

// On 1920x1080 frames saved as JPEG (quality 80) the chevron (on the ad's
// subtitles, on white boxes, on the opening's blue box) fills the V
// 0.51-0.71 with at most 6% stray pixels. The closest other shape in size
// and shape, a dark blade of scenery, fills the V 0.48 but has 13% stray
// pixels, so stray pixels are allowed up to 9%; shapes with nothing
// around them fill the V 0.43 or less and are out of the chevron's size.
// JPEG keeps colour at half resolution, so a white chevron over orange
// (the player's shorts) loses blue below 235 at its edges: white asks
// for 220 in blue (any floor from 190 to 230 reads every frame right).
bool AdvanceArrowDetector::detect(const ImageViewRGB32& screen){
const double scale = double(screen.height()) / 1080;
const ImageViewRGB32 area = extract_box_reference(screen, m_area);
// White: red and green 235 or more, blue 220 or more; dark: every
// channel 110 or under.
return chevron_in(area, scale, 0x00ebebdc, 0xffffffff) || chevron_in(area, scale, 0x00000000, 0xff6e6e6e);
}


} // namespace PokemonLZA
} // namespace NintendoSwitch
} // namespace PokemonAutomation
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/* PokemonLZA AutoStory - Advance arrow detector
*
* From: https://github.com/PokemonAutomation/
*
* The "press A to go on" chevron at the bottom right of a dialogue box or
* of the boxless subtitles of the ad early in the story.
*
*/

#ifndef PokemonAutomation_PokemonLZA_AutoStoryAdvanceArrowDetector_H
#define PokemonAutomation_PokemonLZA_AutoStoryAdvanceArrowDetector_H

#include "Common/Cpp/Color.h"
#include "CommonFramework/ImageTools/ImageBoxes.h"
#include "CommonTools/VisualDetector.h"

namespace PokemonAutomation{
namespace NintendoSwitch{
namespace PokemonLZA{


// The "press A to go on" chevron (a V about 32x19 on 1920x1080) at the
// right end of a dialogue box and of boxless subtitles: dark on a white
// box, white on a blue box and on the ad's subtitles. Found by shape in
// x 1370-1480, y 930-1050: a blob of white (red and green 235+, blue 220+) or dark
// (every channel 110 or under) pixels the size of the chevron, filling a
// V, with no other pixel of its colour around it. Text is not read, so
// any language matches.
class AdvanceArrowDetector : public StaticScreenDetector{
public:
explicit AdvanceArrowDetector(Color color = COLOR_GREEN);

virtual void make_overlays(VideoOverlaySet& items) const override;
virtual bool detect(const ImageViewRGB32& screen) override;

private:
Color m_color;
ImageFloatBox m_area;
};



} // namespace PokemonLZA
} // namespace NintendoSwitch
} // namespace PokemonAutomation
#endif
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/* PokemonLZA AutoStory - Autosave detector
*
* From: https://github.com/PokemonAutomation/
*
*/

#include <initializer_list>
#include "CommonFramework/ImageTypes/ImageViewRGB32.h"
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
#include "PokemonLZA_AutoStoryPixelShares.h"
#include "PokemonLZA_AutoStoryAutosaveDetector.h"

namespace PokemonAutomation{
namespace NintendoSwitch{
namespace PokemonLZA{


namespace{

// The icon (white, with a cyan glow, dim or bright), and the strips around
// it (left, above, below) clear of white and bright cyan, `dx` pixels (of
// 1920) right of where the Chinese badge has them.
bool autosave_icon_at(
const ImageViewRGB32& screen, const ImageFloatBox& icon,
std::initializer_list<ImageFloatBox> around, double dx
){
const ImageFloatBox icon_here = shifted(icon, dx);
const double white = white_share(screen, icon_here);
if (white < 0.08 || white > 0.40 || dim_cyan_share(screen, icon_here) < 0.25){
return false;
}
for (const ImageFloatBox& box : around){
const ImageFloatBox here = shifted(box, dx);
if (white_share(screen, here) >= 0.05 || cyan_share(screen, here) >= 0.05){
return false;
}
}
return true;
}

} // namespace


AutosaveDetector::AutosaveDetector(Color color)
: m_color(color)
, m_icon(1603.0 / 1920, 73.0 / 1080, 27.0 / 1920, 31.0 / 1080)
, m_left(1570.0 / 1920, 73.0 / 1080, 28.0 / 1920, 31.0 / 1080)
, m_above(1603.0 / 1920, 56.0 / 1080, 27.0 / 1920, 14.0 / 1080)
, m_below(1603.0 / 1920, 108.0 / 1080, 27.0 / 1920, 14.0 / 1080)
{}

void AutosaveDetector::make_overlays(VideoOverlaySet& items) const{
items.add(m_color, m_icon);
items.add(m_color, m_left);
items.add(m_color, m_above);
items.add(m_color, m_below);
}

// On the badge the icon is 19% white and 48% cyan, with nothing bright
// left of it. A bright cutscene flash passes the icon's bounds too, but is
// 70%+ white and 39%+ cyan left of the icon as well.
// Only the Chinese badge is checked; with a shorter text (English
// "Saving...") a right-aligned badge moves the icon right, so the icon is
// looked for from where it is in Chinese to 240 px right of it.
// Glowing scenery (a lit blade, a bright sky) can match the icon somewhere
// in that span, but glows on past it: the strips 14 px above and below the
// icon are 22%+ white or cyan there and empty around the badge; they must
// stay under 5%.
// The icon's glow is dim over a dark or lamp-lit scene (about 28, 70, 85
// on 02.04's check-in wall, frame 0075 of a
// run in Japanese, icon at dx 36), so the icon's glow is dim_cyan_share():
// of the 3569 PLZA captures on the console it finds 11 more
// badges, every one a real badge, and no new false one.
bool AutosaveDetector::detect(const ImageViewRGB32& screen){
for (double dx = 0; dx <= 240; dx += 4){
if (autosave_icon_at(screen, m_icon, {m_left, m_above, m_below}, dx)){
return true;
}
}
return false;
}


} // namespace PokemonLZA
} // namespace NintendoSwitch
} // namespace PokemonAutomation
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/* PokemonLZA AutoStory - Autosave detector
*
* From: https://github.com/PokemonAutomation/
*
* The autosave badge, a pencil and "Saving...", which shows for 1-3 s
* after the game autosaves.
*
*/

#ifndef PokemonAutomation_PokemonLZA_AutoStoryAutosaveDetector_H
#define PokemonAutomation_PokemonLZA_AutoStoryAutosaveDetector_H

#include "Common/Cpp/Color.h"
#include "CommonFramework/ImageTools/ImageBoxes.h"
#include "CommonTools/VisualDetector.h"

namespace PokemonAutomation{
namespace NintendoSwitch{
namespace PokemonLZA{


// The autosave badge at the top right: a white pencil with a cyan glow at
// x 1603-1630, y 73-104 on 1920x1080 (the glow dim over a dark scene),
// and nothing white or cyan just left of it, above it or below it. Only
// the icon is read, so any language matches.
class AutosaveDetector : public StaticScreenDetector{
public:
explicit AutosaveDetector(Color color = COLOR_YELLOW);

virtual void make_overlays(VideoOverlaySet& items) const override;
virtual bool detect(const ImageViewRGB32& screen) override;

private:
Color m_color;
ImageFloatBox m_icon;
ImageFloatBox m_left;
ImageFloatBox m_above;
ImageFloatBox m_below;
};



} // namespace PokemonLZA
} // namespace NintendoSwitch
} // namespace PokemonAutomation
#endif
Loading
Loading