Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 90 additions & 0 deletions SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventLog.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/* Screen Event Log
*
* From: https://github.com/PokemonAutomation/
*
* The screen events of a stream of frames, in order and timed.
*
*/

#ifndef PokemonAutomation_CommonTools_ScreenEvents_ScreenEventLog_H
#define PokemonAutomation_CommonTools_ScreenEvents_ScreenEventLog_H

#include <cstddef>
#include <cstdint>
#include <deque>
#include <optional>
#include <string>
#include "Common/Cpp/Time.h"
#include "ScreenEventTypes.h"

namespace PokemonAutomation{


// Whether an event started or stopped showing.
enum class ScreenEventEdge : uint8_t{
ROSE,
FELL,
};

// One event starting or stopping to show, at the frame that showed it.
struct ScreenEventChange{
WallClock time;
ScreenEvent event;
ScreenEventEdge edge;

bool operator==(const ScreenEventChange& other) const = default;
};

// How many changes a log keeps. A walk's events change a few times a
// second at most (a fade's black, a box, the screen going still), so this
// is a minute or more of a busy scene.
constexpr size_t SCREEN_EVENT_LOG_CAPACITY = 256;

// The events of a stream of frames, in order and timed: what rose and fell
// when, not only what was seen (ScreenEventWatcher::seen()). The judges that
// care about order read it: a flash then a battle is an encounter, a box
// then a battle a trainer's sight.
//
// It keeps the newest `capacity` changes and counts the ones it dropped.
// Not thread-safe; ScreenEventWatcher guards its own.
class ScreenEventLog{

Check warning on line 50 in SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventLog.h

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

PokemonAutomation::ScreenEventLog: public_methods=6 (advisory above 5, limit 7)

Check warning on line 50 in SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventLog.h

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

PokemonAutomation::ScreenEventLog: public_methods=6 (advisory above 5, limit 7)
public:
explicit ScreenEventLog(size_t capacity = SCREEN_EVENT_LOG_CAPACITY);

// Adds a frame: a change for each event that rose or fell since the
// frame before (the first frame's events all rise). `now` must not go
// back in time.
void record(const ScreenEvents& frame, WallClock now);

// The changes kept, oldest first.
const std::deque<ScreenEventChange>& changes() const{
return m_changes;
}
// How many changes were dropped, oldest first, to keep the capacity.
size_t dropped() const{
return m_dropped;
}
// The events of the last frame recorded.
ScreenEvents showing() const{
return m_showing;
}
// When the last frame was recorded; none before the first.
std::optional<WallClock> last_frame() const{
return m_last_frame;
}
void clear();

private:
size_t m_capacity;
std::deque<ScreenEventChange> m_changes;
size_t m_dropped = 0;
ScreenEvents m_showing;
std::optional<WallClock> m_last_frame;
};
// "+1200 ms dialog box rose", times from `origin`, one change per line.
std::string describe(const ScreenEventLog& log, WallClock origin);



} // namespace PokemonAutomation
#endif
94 changes: 94 additions & 0 deletions SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/* Screen Event Types

Check warning on line 1 in SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

<header>: header_symbols=7 (advisory above 6, limit 8)

Check warning on line 1 in SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

<header>: header_symbols=7 (advisory above 6, limit 8)
*
* From: https://github.com/PokemonAutomation/
*
* The words a program uses for what the screen shows: a fade, a dialogue
* box, a battle starting. ScreenEvents.h watches a stream for them.
*
*/

#ifndef PokemonAutomation_CommonTools_ScreenEvents_ScreenEventTypes_H
#define PokemonAutomation_CommonTools_ScreenEvents_ScreenEventTypes_H

#include <cstddef>
#include <cstdint>
#include <initializer_list>
#include <string>
#include <string_view>

namespace PokemonAutomation{


// One thing a frame shows. A game reports the ones its detectors can see.
enum class ScreenEvent : uint8_t{
FADE_BLACK, // A uniform dark screen: a door, a cave mouth, a scene cut.
FADE_WHITE, // A uniform white screen: a cave exit.
LOCATION_BANNER, // A location banner. An observer given an expected
// map reports only that map's.
DIALOG_BOX, // A dialogue box.
CHOICE_BOX, // A choice (Yes/No, a menu of answers).
INTERACT_PROMPT, // A button prompt to interact ("A Talk").
BATTLE_STARTED, // A battle shows (its menu, its HUD or its opening).
BATTLE_ENDED, // A battle's closing screen.
PLAYER_CAN_MOVE, // The overworld with the player in control.
SCREEN_STILL, // The screen barely changed since the frame before.
SWITCH_HOME_MENU, // The Switch's HOME menu: the game is not on the screen.
};
// How many ScreenEvent values there are.
constexpr size_t SCREEN_EVENT_COUNT = 11;
static_assert((size_t)ScreenEvent::SWITCH_HOME_MENU + 1 == SCREEN_EVENT_COUNT, "SWITCH_HOME_MENU is the last ScreenEvent");
// "dialog box", for the log.
std::string screen_event_name(ScreenEvent event);


// How a screen's fade looks: the kind of FADE_BLACK / FADE_WHITE.
enum class WarpFade{
NONE,
BLACK, // a door's black, or a cave mouth's dim fade
WHITE, // a cave exit's white, with no battle after it
};
// "black fade", for the log.
std::string warp_fade_name(WarpFade fade);


// The events of one frame, or of all the frames a watcher saw.
class ScreenEvents{

Check warning on line 55 in SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

PokemonAutomation::ScreenEvents: public_methods=6 (advisory above 5, limit 7)

Check warning on line 55 in SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

PokemonAutomation::ScreenEvents: public_methods=6 (advisory above 5, limit 7)
public:
ScreenEvents() = default;
ScreenEvents(std::initializer_list<ScreenEvent> events);

bool has(ScreenEvent event) const{
return (m_bits & bit(event)) != 0;
}
// Whether any of `events` is in this set.
bool any_of(const ScreenEvents& events) const{
return (m_bits & events.m_bits) != 0;
}
// Adds `event`, or removes it when `on` is false (as std::bitset::set).
void set(ScreenEvent event, bool on);
void merge(const ScreenEvents& events){
m_bits |= events.m_bits;
}

uint32_t bits() const{
return m_bits;
}
static ScreenEvents from_bits(uint32_t bits);

bool operator==(const ScreenEvents& other) const = default;

private:
static uint32_t bit(ScreenEvent event){
return uint32_t(1) << (uint32_t)event;
}

uint32_t m_bits = 0;
};
// "dialog box, player can move" (joined by `separator`); "nothing" when
// empty.
std::string describe(const ScreenEvents& events, std::string_view separator = ", ");



} // namespace PokemonAutomation
#endif
195 changes: 195 additions & 0 deletions SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
/* Screen Events
*
* From: https://github.com/PokemonAutomation/
*
*/

#include <chrono>
#include <format>
#include "CommonFramework/ImageTools/ImageBoxes.h"
#include "CommonFramework/ImageTools/ImageStats.h"
#include "CommonTools/Images/SolidColorTest.h"
#include "ScreenEventLog.h"
#include "ScreenEventTypes.h"
#include "ScreenFades.h"
#include "ScreenEvents.h"

namespace PokemonAutomation{


std::string screen_event_name(ScreenEvent event){
switch (event){
case ScreenEvent::FADE_BLACK: return "black fade";
case ScreenEvent::FADE_WHITE: return "white fade";
case ScreenEvent::LOCATION_BANNER: return "location banner";
case ScreenEvent::DIALOG_BOX: return "dialog box";
case ScreenEvent::CHOICE_BOX: return "choice box";
case ScreenEvent::INTERACT_PROMPT: return "interact prompt";
case ScreenEvent::BATTLE_STARTED: return "battle started";
case ScreenEvent::BATTLE_ENDED: return "battle ended";
case ScreenEvent::PLAYER_CAN_MOVE: return "player can move";
case ScreenEvent::SCREEN_STILL: return "screen still";
case ScreenEvent::SWITCH_HOME_MENU: return "Switch HOME menu";
}
return "?";
}

std::string warp_fade_name(WarpFade fade){
switch (fade){
case WarpFade::NONE: return "no fade";
case WarpFade::BLACK: return "black fade";
case WarpFade::WHITE: return "white fade";
}
return "?";
}


ScreenEvents::ScreenEvents(std::initializer_list<ScreenEvent> events){
for (ScreenEvent event : events){
set(event, true);
}
}
void ScreenEvents::set(ScreenEvent event, bool on){
if (on){
m_bits |= bit(event);
}else{
m_bits &= ~bit(event);
}
}
ScreenEvents ScreenEvents::from_bits(uint32_t bits){
ScreenEvents out;
out.m_bits = bits & ((uint32_t(1) << SCREEN_EVENT_COUNT) - 1);
return out;
}

ScreenEvents fade_events(WarpFade fade){
switch (fade){
case WarpFade::NONE:
return {};
case WarpFade::BLACK:
return {ScreenEvent::FADE_BLACK};
case WarpFade::WHITE:
return {ScreenEvent::FADE_WHITE};
}
return {};
}
WarpFade fade_in(const ScreenEvents& events){
if (events.has(ScreenEvent::FADE_BLACK)){
return WarpFade::BLACK;
}
if (events.has(ScreenEvent::FADE_WHITE)){
return WarpFade::WHITE;
}
return WarpFade::NONE;
}
std::string describe(const ScreenEvents& events, std::string_view separator){
std::string out;
for (size_t c = 0; c < SCREEN_EVENT_COUNT; c++){
const ScreenEvent event = (ScreenEvent)c;
if (!events.has(event)){
continue;
}
if (!out.empty()){
out += separator;
}
out += screen_event_name(event);
}
return out.empty() ? "nothing" : out;
}


// Black is is_black() (SolidColorTest, BlackScreenDetector's test) with
// the fade's thresholds.
// White is not is_white(): that also wants an even grey, and a fade to
// white can be tinted.
WarpFade fade_of(const ImageStats& middle){
if (is_black(middle, FADE_MAX_DARK_SUM, FADE_MAX_STDDEV_SUM)){
return WarpFade::BLACK;
}
if (middle.stddev.sum() > FADE_MAX_STDDEV_SUM){
return WarpFade::NONE;
}
return middle.average.sum() >= FADE_MIN_WHITE_SUM ? WarpFade::WHITE : WarpFade::NONE;
}
WarpFade fade_of_screen(const ImageViewRGB32& screen){
return fade_of(image_stats(extract_box_reference(screen, ImageFloatBox(0.1, 0.1, 0.8, 0.8))));
}


ScreenEventLog::ScreenEventLog(size_t capacity)
: m_capacity(capacity)
{}
void ScreenEventLog::record(const ScreenEvents& frame, WallClock now){
const uint32_t changed = frame.bits() ^ m_showing.bits();
for (size_t c = 0; c < SCREEN_EVENT_COUNT; c++){
if ((changed & (uint32_t(1) << c)) == 0){
continue;
}
const ScreenEvent event = (ScreenEvent)c;
m_changes.push_back({now, event, frame.has(event) ? ScreenEventEdge::ROSE : ScreenEventEdge::FELL});
}
while (m_changes.size() > m_capacity){
m_changes.pop_front();
m_dropped++;
}
m_showing = frame;
m_last_frame = now;
}
void ScreenEventLog::clear(){
m_changes.clear();
m_dropped = 0;
m_showing = ScreenEvents();
m_last_frame.reset();
}
std::string describe(const ScreenEventLog& log, WallClock origin){
std::string out;
if (log.dropped() > 0){
out += std::format("({} older changes dropped)\n", log.dropped());
}
for (const ScreenEventChange& change : log.changes()){
out += std::format(
"{:+} ms {} {}\n",
std::chrono::duration_cast<Milliseconds>(change.time - origin).count(),
screen_event_name(change.event),
change.edge == ScreenEventEdge::ROSE ? "rose" : "fell"
);
}
return out.empty() ? "no changes" : out;
}


ScreenEventWatcher::ScreenEventWatcher(ScreenEventObserver& observer, ScreenEvents stop_on)
: VisualInferenceCallback("ScreenEventWatcher")
, m_observer(observer)
, m_stop_on(stop_on)
{}
void ScreenEventWatcher::make_overlays(VideoOverlaySet& items) const{
m_observer.make_overlays(items);
}
bool ScreenEventWatcher::process_frame(const VideoSnapshot& frame){
const ScreenEvents events = m_observer.observe(frame, frame.timestamp);
m_seen.fetch_or(events.bits(), std::memory_order_acq_rel);
{
LockGuard<Mutex> lg(m_log_lock);
m_log.record(events, frame.timestamp);
}
if (!events.any_of(m_stop_on)){
return false;
}
m_stopped_on.store(events.bits(), std::memory_order_release);
return true;
}
ScreenEventLog ScreenEventWatcher::log() const{
LockGuard<Mutex> lg(m_log_lock);
return m_log;
}
void ScreenEventWatcher::reset(){
m_seen.store(0, std::memory_order_release);
m_stopped_on.store(0, std::memory_order_release);
LockGuard<Mutex> lg(m_log_lock);
m_log.clear();
}



} // namespace PokemonAutomation
Loading
Loading