Skip to content
Draft
90 changes: 90 additions & 0 deletions SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventLog.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/* Screen Event Log
*
* From: https://github.com/PokemonAutomation/
*
* The screen events of a stream of frames, in order and timed.
*
*/

#ifndef PokemonAutomation_CommonTools_ScreenEvents_ScreenEventLog_H
#define PokemonAutomation_CommonTools_ScreenEvents_ScreenEventLog_H

#include <cstddef>
#include <cstdint>
#include <deque>
#include <optional>
#include <string>
#include "Common/Cpp/Time.h"
#include "ScreenEventTypes.h"

namespace PokemonAutomation{


// Whether an event started or stopped showing.
enum class ScreenEventEdge : uint8_t{
ROSE,
FELL,
};

// One event starting or stopping to show, at the frame that showed it.
struct ScreenEventChange{
WallClock time;
ScreenEvent event;
ScreenEventEdge edge;

bool operator==(const ScreenEventChange& other) const = default;
};

// How many changes a log keeps. A walk's events change a few times a
// second at most (a fade's black, a box, the screen going still), so this
// is a minute or more of a busy scene.
constexpr size_t SCREEN_EVENT_LOG_CAPACITY = 256;

// The events of a stream of frames, in order and timed: what rose and fell
// when, not only what was seen (ScreenEventWatcher::seen()). The judges that
// care about order read it: a flash then a battle is an encounter, a box
// then a battle a trainer's sight.
//
// It keeps the newest `capacity` changes and counts the ones it dropped.
// Not thread-safe; ScreenEventWatcher guards its own.
class ScreenEventLog{

Check warning on line 50 in SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventLog.h

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

PokemonAutomation::ScreenEventLog: public_methods=6 (advisory above 5, limit 7)
public:
explicit ScreenEventLog(size_t capacity = SCREEN_EVENT_LOG_CAPACITY);

// Adds a frame: a change for each event that rose or fell since the
// frame before (the first frame's events all rise). `now` must not go
// back in time.
void record(const ScreenEvents& frame, WallClock now);

// The changes kept, oldest first.
const std::deque<ScreenEventChange>& changes() const{
return m_changes;
}
// How many changes were dropped, oldest first, to keep the capacity.
size_t dropped() const{
return m_dropped;
}
// The events of the last frame recorded.
ScreenEvents showing() const{
return m_showing;
}
// When the last frame was recorded; none before the first.
std::optional<WallClock> last_frame() const{
return m_last_frame;
}
void clear();

private:
size_t m_capacity;
std::deque<ScreenEventChange> m_changes;
size_t m_dropped = 0;
ScreenEvents m_showing;
std::optional<WallClock> m_last_frame;
};
// "+1200 ms dialog box rose", times from `origin`, one change per line.
std::string describe(const ScreenEventLog& log, WallClock origin);



} // namespace PokemonAutomation
#endif
94 changes: 94 additions & 0 deletions SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/* Screen Event Types

Check warning on line 1 in SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

<header>: header_symbols=7 (advisory above 6, limit 8)
*
* From: https://github.com/PokemonAutomation/
*
* The words a program uses for what the screen shows: a fade, a dialogue
* box, a battle starting. ScreenEvents.h watches a stream for them.
*
*/

#ifndef PokemonAutomation_CommonTools_ScreenEvents_ScreenEventTypes_H
#define PokemonAutomation_CommonTools_ScreenEvents_ScreenEventTypes_H

#include <cstddef>
#include <cstdint>
#include <initializer_list>
#include <string>
#include <string_view>

namespace PokemonAutomation{


// One thing a frame shows. A game reports the ones its detectors can see.
enum class ScreenEvent : uint8_t{
FADE_BLACK, // A uniform dark screen: a door, a cave mouth, a scene cut.
FADE_WHITE, // A uniform white screen: a cave exit.
LOCATION_BANNER, // A location banner. An observer given an expected
// map reports only that map's.
DIALOG_BOX, // A dialogue box.
CHOICE_BOX, // A choice (Yes/No, a menu of answers).
INTERACT_PROMPT, // A button prompt to interact ("A Talk").
BATTLE_STARTED, // A battle shows (its menu, its HUD or its opening).
BATTLE_ENDED, // A battle's closing screen.
PLAYER_CAN_MOVE, // The overworld with the player in control.
SCREEN_STILL, // The screen barely changed since the frame before.
SWITCH_HOME_MENU, // The Switch's HOME menu: the game is not on the screen.
};
// How many ScreenEvent values there are.
constexpr size_t SCREEN_EVENT_COUNT = 11;
static_assert((size_t)ScreenEvent::SWITCH_HOME_MENU + 1 == SCREEN_EVENT_COUNT, "SWITCH_HOME_MENU is the last ScreenEvent");
// "dialog box", for the log.
std::string screen_event_name(ScreenEvent event);


// How a screen's fade looks: the kind of FADE_BLACK / FADE_WHITE.
enum class WarpFade{
NONE,
BLACK, // a door's black, or a cave mouth's dim fade
WHITE, // a cave exit's white, with no battle after it
};
// "black fade", for the log.
std::string warp_fade_name(WarpFade fade);


// The events of one frame, or of all the frames a watcher saw.
class ScreenEvents{

Check warning on line 55 in SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

PokemonAutomation::ScreenEvents: public_methods=6 (advisory above 5, limit 7)
public:
ScreenEvents() = default;
ScreenEvents(std::initializer_list<ScreenEvent> events);

bool has(ScreenEvent event) const{
return (m_bits & bit(event)) != 0;
}
// Whether any of `events` is in this set.
bool any_of(const ScreenEvents& events) const{
return (m_bits & events.m_bits) != 0;
}
// Adds `event`, or removes it when `on` is false (as std::bitset::set).
void set(ScreenEvent event, bool on);
void merge(const ScreenEvents& events){
m_bits |= events.m_bits;
}

uint32_t bits() const{
return m_bits;
}
static ScreenEvents from_bits(uint32_t bits);

bool operator==(const ScreenEvents& other) const = default;

private:
static uint32_t bit(ScreenEvent event){
return uint32_t(1) << (uint32_t)event;
}

uint32_t m_bits = 0;
};
// "dialog box, player can move" (joined by `separator`); "nothing" when
// empty.
std::string describe(const ScreenEvents& events, std::string_view separator = ", ");



} // namespace PokemonAutomation
#endif
195 changes: 195 additions & 0 deletions SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
/* Screen Events
*
* From: https://github.com/PokemonAutomation/
*
*/

#include <chrono>
#include <format>
#include "CommonFramework/ImageTools/ImageBoxes.h"
#include "CommonFramework/ImageTools/ImageStats.h"
#include "CommonTools/Images/SolidColorTest.h"
#include "ScreenEventLog.h"
#include "ScreenEventTypes.h"
#include "ScreenFades.h"
#include "ScreenEvents.h"

namespace PokemonAutomation{


std::string screen_event_name(ScreenEvent event){
switch (event){
case ScreenEvent::FADE_BLACK: return "black fade";
case ScreenEvent::FADE_WHITE: return "white fade";
case ScreenEvent::LOCATION_BANNER: return "location banner";
case ScreenEvent::DIALOG_BOX: return "dialog box";
case ScreenEvent::CHOICE_BOX: return "choice box";
case ScreenEvent::INTERACT_PROMPT: return "interact prompt";
case ScreenEvent::BATTLE_STARTED: return "battle started";
case ScreenEvent::BATTLE_ENDED: return "battle ended";
case ScreenEvent::PLAYER_CAN_MOVE: return "player can move";
case ScreenEvent::SCREEN_STILL: return "screen still";
case ScreenEvent::SWITCH_HOME_MENU: return "Switch HOME menu";
}
return "?";
}

std::string warp_fade_name(WarpFade fade){
switch (fade){
case WarpFade::NONE: return "no fade";
case WarpFade::BLACK: return "black fade";
case WarpFade::WHITE: return "white fade";
}
return "?";
}


ScreenEvents::ScreenEvents(std::initializer_list<ScreenEvent> events){
for (ScreenEvent event : events){
set(event, true);
}
}
void ScreenEvents::set(ScreenEvent event, bool on){
if (on){
m_bits |= bit(event);
}else{
m_bits &= ~bit(event);
}
}
ScreenEvents ScreenEvents::from_bits(uint32_t bits){
ScreenEvents out;
out.m_bits = bits & ((uint32_t(1) << SCREEN_EVENT_COUNT) - 1);
return out;
}

ScreenEvents fade_events(WarpFade fade){
switch (fade){
case WarpFade::NONE:
return {};
case WarpFade::BLACK:
return {ScreenEvent::FADE_BLACK};
case WarpFade::WHITE:
return {ScreenEvent::FADE_WHITE};
}
return {};
}
WarpFade fade_in(const ScreenEvents& events){
if (events.has(ScreenEvent::FADE_BLACK)){
return WarpFade::BLACK;
}
if (events.has(ScreenEvent::FADE_WHITE)){
return WarpFade::WHITE;
}
return WarpFade::NONE;
}
std::string describe(const ScreenEvents& events, std::string_view separator){
std::string out;
for (size_t c = 0; c < SCREEN_EVENT_COUNT; c++){
const ScreenEvent event = (ScreenEvent)c;
if (!events.has(event)){
continue;
}
if (!out.empty()){
out += separator;
}
out += screen_event_name(event);
}
return out.empty() ? "nothing" : out;
}


// Black is is_black() (SolidColorTest, BlackScreenDetector's test) with
// the fade's thresholds.
// White is not is_white(): that also wants an even grey, and a fade to
// white can be tinted.
WarpFade fade_of(const ImageStats& middle){
if (is_black(middle, FADE_MAX_DARK_SUM, FADE_MAX_STDDEV_SUM)){
return WarpFade::BLACK;
}
if (middle.stddev.sum() > FADE_MAX_STDDEV_SUM){
return WarpFade::NONE;
}
return middle.average.sum() >= FADE_MIN_WHITE_SUM ? WarpFade::WHITE : WarpFade::NONE;
}
WarpFade fade_of_screen(const ImageViewRGB32& screen){
return fade_of(image_stats(extract_box_reference(screen, ImageFloatBox(0.1, 0.1, 0.8, 0.8))));
}


ScreenEventLog::ScreenEventLog(size_t capacity)
: m_capacity(capacity)
{}
void ScreenEventLog::record(const ScreenEvents& frame, WallClock now){
const uint32_t changed = frame.bits() ^ m_showing.bits();
for (size_t c = 0; c < SCREEN_EVENT_COUNT; c++){
if ((changed & (uint32_t(1) << c)) == 0){
continue;
}
const ScreenEvent event = (ScreenEvent)c;
m_changes.push_back({now, event, frame.has(event) ? ScreenEventEdge::ROSE : ScreenEventEdge::FELL});
}
while (m_changes.size() > m_capacity){
m_changes.pop_front();
m_dropped++;
}
m_showing = frame;
m_last_frame = now;
}
void ScreenEventLog::clear(){
m_changes.clear();
m_dropped = 0;
m_showing = ScreenEvents();
m_last_frame.reset();
}
std::string describe(const ScreenEventLog& log, WallClock origin){
std::string out;
if (log.dropped() > 0){
out += std::format("({} older changes dropped)\n", log.dropped());
}
for (const ScreenEventChange& change : log.changes()){
out += std::format(
"{:+} ms {} {}\n",
std::chrono::duration_cast<Milliseconds>(change.time - origin).count(),
screen_event_name(change.event),
change.edge == ScreenEventEdge::ROSE ? "rose" : "fell"
);
}
return out.empty() ? "no changes" : out;
}


ScreenEventWatcher::ScreenEventWatcher(ScreenEventObserver& observer, ScreenEvents stop_on)
: VisualInferenceCallback("ScreenEventWatcher")
, m_observer(observer)
, m_stop_on(stop_on)
{}
void ScreenEventWatcher::make_overlays(VideoOverlaySet& items) const{
m_observer.make_overlays(items);
}
bool ScreenEventWatcher::process_frame(const VideoSnapshot& frame){
const ScreenEvents events = m_observer.observe(frame, frame.timestamp);
m_seen.fetch_or(events.bits(), std::memory_order_acq_rel);
{
LockGuard<Mutex> lg(m_log_lock);
m_log.record(events, frame.timestamp);
}
if (!events.any_of(m_stop_on)){
return false;
}
m_stopped_on.store(events.bits(), std::memory_order_release);
return true;
}
ScreenEventLog ScreenEventWatcher::log() const{
LockGuard<Mutex> lg(m_log_lock);
return m_log;
}
void ScreenEventWatcher::reset(){
m_seen.store(0, std::memory_order_release);
m_stopped_on.store(0, std::memory_order_release);
LockGuard<Mutex> lg(m_log_lock);
m_log.clear();
}



} // namespace PokemonAutomation
Loading
Loading