Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 94 additions & 0 deletions SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/* Screen Event Types
*
* From: https://github.com/PokemonAutomation/
*
* The words a program uses for what the screen shows: a fade, a dialogue
* box, a battle starting. ScreenEvents.h watches a stream for them.
*
*/

#ifndef PokemonAutomation_CommonTools_ScreenEvents_ScreenEventTypes_H
#define PokemonAutomation_CommonTools_ScreenEvents_ScreenEventTypes_H

#include <cstddef>
#include <cstdint>
#include <initializer_list>
#include <string>
#include <string_view>

namespace PokemonAutomation{


// One thing a frame shows. A game reports the ones its detectors can see.
enum class ScreenEvent : uint8_t{
FADE_BLACK, // A uniform dark screen: a door, a cave mouth, a scene cut.
FADE_WHITE, // A uniform white screen: a cave exit.
LOCATION_BANNER, // A location banner. An observer given an expected
// map reports only that map's.
DIALOG_BOX, // A dialogue box.
CHOICE_BOX, // A choice (Yes/No, a menu of answers).
INTERACT_PROMPT, // A button prompt to interact ("A Talk").
BATTLE_STARTED, // A battle shows (its menu, its HUD or its opening).
BATTLE_ENDED, // A battle's closing screen.
PLAYER_CAN_MOVE, // The overworld with the player in control.
SCREEN_STILL, // The screen barely changed since the frame before.
SWITCH_HOME_MENU, // The Switch's HOME menu: the game is not on the screen.
};
// How many ScreenEvent values there are.
constexpr size_t SCREEN_EVENT_COUNT = 11;
static_assert((size_t)ScreenEvent::SWITCH_HOME_MENU + 1 == SCREEN_EVENT_COUNT, "SWITCH_HOME_MENU is the last ScreenEvent");
// "dialog box", for the log.
std::string screen_event_name(ScreenEvent event);


// How a screen's fade looks: the kind of FADE_BLACK / FADE_WHITE.
enum class WarpFade{
NONE,
BLACK, // a door's black, or a cave mouth's dim fade
WHITE, // a cave exit's white, with no battle after it
};
// "black fade", for the log.
std::string warp_fade_name(WarpFade fade);


// The events of one frame, or of all the frames a watcher saw.
class ScreenEvents{
public:
ScreenEvents() = default;
ScreenEvents(std::initializer_list<ScreenEvent> events);

bool has(ScreenEvent event) const{
return (m_bits & bit(event)) != 0;
}
// Whether any of `events` is in this set.
bool any_of(const ScreenEvents& events) const{
return (m_bits & events.m_bits) != 0;
}
// Adds `event`, or removes it when `on` is false (as std::bitset::set).
void set(ScreenEvent event, bool on);
void merge(const ScreenEvents& events){
m_bits |= events.m_bits;
}

uint32_t bits() const{
return m_bits;
}
static ScreenEvents from_bits(uint32_t bits);

bool operator==(const ScreenEvents& other) const = default;

private:
static uint32_t bit(ScreenEvent event){
return uint32_t(1) << (uint32_t)event;
}

uint32_t m_bits = 0;
};
// "dialog box, player can move" (joined by `separator`); "nothing" when
// empty.
std::string describe(const ScreenEvents& events, std::string_view separator = ", ");



} // namespace PokemonAutomation
#endif
125 changes: 125 additions & 0 deletions SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
/* Screen Events
*
* From: https://github.com/PokemonAutomation/
*
*/

#include "CommonFramework/ImageTools/ImageBoxes.h"
#include "CommonFramework/ImageTools/ImageDiff.h"
#include "CommonFramework/ImageTools/ImageStats.h"
#include "CommonTools/Images/SolidColorTest.h"
#include "ScreenEventTypes.h"
#include "ScreenFades.h"
#include "ScreenEvents.h"

namespace PokemonAutomation{


std::string screen_event_name(ScreenEvent event){
switch (event){
case ScreenEvent::FADE_BLACK: return "black fade";
case ScreenEvent::FADE_WHITE: return "white fade";
case ScreenEvent::LOCATION_BANNER: return "location banner";
case ScreenEvent::DIALOG_BOX: return "dialog box";
case ScreenEvent::CHOICE_BOX: return "choice box";
case ScreenEvent::INTERACT_PROMPT: return "interact prompt";
case ScreenEvent::BATTLE_STARTED: return "battle started";
case ScreenEvent::BATTLE_ENDED: return "battle ended";
case ScreenEvent::PLAYER_CAN_MOVE: return "player can move";
case ScreenEvent::SCREEN_STILL: return "screen still";
case ScreenEvent::SWITCH_HOME_MENU: return "Switch HOME menu";
}
return "?";
}

std::string warp_fade_name(WarpFade fade){
switch (fade){
case WarpFade::NONE: return "no fade";
case WarpFade::BLACK: return "black fade";
case WarpFade::WHITE: return "white fade";
}
return "?";
}


ScreenEvents::ScreenEvents(std::initializer_list<ScreenEvent> events){
for (ScreenEvent event : events){
set(event, true);
}
}
void ScreenEvents::set(ScreenEvent event, bool on){
if (on){
m_bits |= bit(event);
}else{
m_bits &= ~bit(event);
}
}
ScreenEvents ScreenEvents::from_bits(uint32_t bits){
ScreenEvents out;
out.m_bits = bits & ((uint32_t(1) << SCREEN_EVENT_COUNT) - 1);
return out;
}

ScreenEvents fade_events(WarpFade fade){
switch (fade){
case WarpFade::NONE:
return {};
case WarpFade::BLACK:
return {ScreenEvent::FADE_BLACK};
case WarpFade::WHITE:
return {ScreenEvent::FADE_WHITE};
}
return {};
}
WarpFade fade_in(const ScreenEvents& events){
if (events.has(ScreenEvent::FADE_BLACK)){
return WarpFade::BLACK;
}
if (events.has(ScreenEvent::FADE_WHITE)){
return WarpFade::WHITE;
}
return WarpFade::NONE;
}
std::string describe(const ScreenEvents& events, std::string_view separator){
std::string out;
for (size_t c = 0; c < SCREEN_EVENT_COUNT; c++){
const ScreenEvent event = (ScreenEvent)c;
if (!events.has(event)){
continue;
}
if (!out.empty()){
out += separator;
}
out += screen_event_name(event);
}
return out.empty() ? "nothing" : out;
}


// Black is is_black() (SolidColorTest, BlackScreenDetector's test) with
// the fade's thresholds.
// White is not is_white(): that also wants an even grey, and a fade to
// white can be tinted.
WarpFade fade_of(const ImageStats& middle){
if (is_black(middle, FADE_MAX_DARK_SUM, FADE_MAX_STDDEV_SUM)){
return WarpFade::BLACK;
}
if (middle.stddev.sum() > FADE_MAX_STDDEV_SUM){
return WarpFade::NONE;
}
return middle.average.sum() >= FADE_MIN_WHITE_SUM ? WarpFade::WHITE : WarpFade::NONE;
}
WarpFade fade_of_screen(const ImageViewRGB32& screen){
return fade_of(image_stats(extract_box_reference(screen, ImageFloatBox(0.1, 0.1, 0.8, 0.8))));
}


bool same_screen(const VideoSnapshot& a, const VideoSnapshot& b){
if (!a || !b || a->width() != b->width() || a->height() != b->height()){
return false;
}
return ImageMatch::pixel_RMSD(a, b) < 2.0;
}


} // namespace PokemonAutomation
49 changes: 49 additions & 0 deletions SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/* Screen Events
*
* From: https://github.com/PokemonAutomation/
*
* What an autonomous program can see on the screen, in words every game
* shares: a fade, a location banner, a dialogue box, a battle starting.
* Each game implements ScreenEventObserver with its own detectors; the
* judgements built on these events are then the same for all of them.
*
* An observer is asked frame by frame.
*
*/

#ifndef PokemonAutomation_CommonTools_ScreenEvents_ScreenEvents_H
#define PokemonAutomation_CommonTools_ScreenEvents_ScreenEvents_H

#include "Common/Cpp/Time.h"
#include "CommonFramework/VideoPipeline/VideoFeed.h"
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
#include "ScreenEventTypes.h"

namespace PokemonAutomation{


// A game's detectors, as events. Observers may keep state between frames
// (the frame before, for SCREEN_STILL; when a banner was last read, to
// gate OCR), so one observer watches one stream at a time.
class ScreenEventObserver{
public:
virtual ~ScreenEventObserver() = default;

// What `frame` shows. `now` is the caller's clock, for observers that
// rate-limit expensive reads. A null frame shows nothing.
virtual ScreenEvents observe(const VideoSnapshot& frame, WallClock now) = 0;

// The boxes the detectors read, for the video overlay.
virtual void make_overlays(VideoOverlaySet& items) const = 0;
};


// Whether `a` and `b` both have a frame of the same size within the
// still-screen RMSD threshold (2.0): the screen barely changed
// (ScreenEvent::SCREEN_STILL).
bool same_screen(const VideoSnapshot& a, const VideoSnapshot& b);



} // namespace PokemonAutomation
#endif
Loading
Loading