Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
e9f0efe
AutoStory: screen state and interrupt tracking, split out of #122
dayfine Nov 10, 2026
d39d8ce
Merge nav/story-engine into nav/screen-state (#312 review: Types/, da…
dayfine Nov 10, 2026
942e468
Merge nav/story-engine into nav/screen-state (structured step args)
dayfine Nov 10, 2026
28df3b2
Merge nav/story-engine into nav/screen-state (story range options and…
dayfine Nov 10, 2026
936b808
Merge nav/story-engine into nav/screen-state (range end lock mode)
dayfine Nov 10, 2026
7b73ef0
Merge nav/story-engine into nav/screen-state (one story test registra…
dayfine Nov 10, 2026
bd2a594
Merge nav/story-engine (one-line route plan) into nav/screen-state
dayfine Nov 10, 2026
4652d59
Merge remote-tracking branch 'origin/nav/story-engine' into review312…
dayfine Nov 10, 2026
aa90844
Merge nav/story-engine (run report / range options split) into nav/sc…
dayfine Nov 10, 2026
87385d5
AutoStory: split over-limit headers, drop char* for code health
dayfine Nov 10, 2026
b2b00c8
Merge branch 'nav/story-engine' into nav/screen-state
dayfine Nov 10, 2026
d7048f1
Screen state: std::format, outputs last, one test header per suite
dayfine Nov 10, 2026
1ae7cf0
Merge nav/story-engine into nav/screen-state
dayfine Nov 10, 2026
23bc092
Merge nav/story-engine into nav/screen-state
dayfine Nov 10, 2026
ba65152
Merge nav/story-engine into nav/screen-state
dayfine Nov 11, 2026
f4b149c
AutoStory core: move the 2D scene judges and interrupts out of nav/sc…
dayfine Nov 11, 2026
cae4730
Merge origin/nav/story-engine into nav/screen-state
dayfine Nov 12, 2026
6082102
Merge branch 'nav/story-engine' into nav/screen-state
dayfine Nov 12, 2026
4fd9aca
Merge origin/nav/story-engine into nav/screen-state
dayfine Nov 12, 2026
8ed7494
Merge branch 'nav/story-engine' into nav/screen-state
dayfine Nov 13, 2026
4a7953b
Merge branch 'nav/story-engine' into nav/screen-state
dayfine Nov 13, 2026
b72eca8
Merge remote-tracking branch 'origin/nav/story-engine' into HEAD
dayfine Nov 13, 2026
105a593
Merge origin/nav/story-engine into nav/screen-state: #45 review fixes
dayfine Nov 13, 2026
bdd16da
Merge origin/nav/story-engine into nav/screen-state: #45 review fixes
dayfine Nov 13, 2026
070c0a3
Merge origin/nav/story-engine into nav/screen-state: shared checkpoin…
dayfine Nov 13, 2026
60d4a9d
Merge origin/nav/story-engine into nav/screen-state
dayfine Nov 13, 2026
7fccccf
CommonTools: move the screen events out of AutoStory
dayfine Nov 13, 2026
f69e766
Merge origin/nav/story-engine into nav/screen-state: StoryRunCounts m…
dayfine Nov 13, 2026
91395e5
Merge nav/story-engine into nav/screen-state
dayfine Nov 14, 2026
bed0945
Merge nav/story-engine into nav/screen-state: add_checkpoint_prefix (…
dayfine Nov 14, 2026
a90d330
Merge nav/story-engine into nav/screen-state
dayfine Nov 14, 2026
e699019
Merge nav/story-engine into nav/screen-state
dayfine Nov 14, 2026
929298e
Merge nav/story-engine into nav/screen-state: drop the stale native c…
dayfine Nov 14, 2026
ee797a4
ScreenEvents tests: the case's name outside its input (code health te…
dayfine Nov 14, 2026
a44ab05
ScreenEvents: comments say why without lab runs or PR numbers (#313 r…
dayfine Nov 14, 2026
2cb688b
Merge nav/story-engine into nav/screen-state: end pose after a single…
dayfine Nov 14, 2026
6ccd314
Merge nav/shared into nav/screen-state: #312 merged
dayfine Nov 14, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 90 additions & 0 deletions SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventLog.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/* Screen Event Log
*
* From: https://github.com/PokemonAutomation/
*
* The screen events of a stream of frames, in order and timed.
*
*/

#ifndef PokemonAutomation_CommonTools_ScreenEvents_ScreenEventLog_H
#define PokemonAutomation_CommonTools_ScreenEvents_ScreenEventLog_H

#include <cstddef>
#include <cstdint>
#include <deque>
#include <optional>
#include <string>
#include "Common/Cpp/Time.h"
#include "ScreenEventTypes.h"

namespace PokemonAutomation{


// Whether an event started or stopped showing.
enum class ScreenEventEdge : uint8_t{
ROSE,
FELL,
};

// One event starting or stopping to show, at the frame that showed it.
struct ScreenEventChange{
WallClock time;
ScreenEvent event;
ScreenEventEdge edge;

bool operator==(const ScreenEventChange& other) const = default;
};

// How many changes a log keeps. A walk's events change a few times a
// second at most (a fade's black, a box, the screen going still), so this
// is a minute or more of a busy scene.
constexpr size_t SCREEN_EVENT_LOG_CAPACITY = 256;

// The events of a stream of frames, in order and timed: what rose and fell
// when, not only what was seen (ScreenEventWatcher::seen()). The judges that
// care about order read it: a flash then a battle is an encounter, a box
// then a battle a trainer's sight.
//
// It keeps the newest `capacity` changes and counts the ones it dropped.
// Not thread-safe; ScreenEventWatcher guards its own.
class ScreenEventLog{

Check warning on line 50 in SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventLog.h

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

PokemonAutomation::ScreenEventLog: public_methods=6 (advisory above 5, limit 7)
public:
explicit ScreenEventLog(size_t capacity = SCREEN_EVENT_LOG_CAPACITY);

// Adds a frame: a change for each event that rose or fell since the
// frame before (the first frame's events all rise). `now` must not go
// back in time.
void record(const ScreenEvents& frame, WallClock now);

// The changes kept, oldest first.
const std::deque<ScreenEventChange>& changes() const{
return m_changes;
}
// How many changes were dropped, oldest first, to keep the capacity.
size_t dropped() const{
return m_dropped;
}
// The events of the last frame recorded.
ScreenEvents showing() const{
return m_showing;
}
// When the last frame was recorded; none before the first.
std::optional<WallClock> last_frame() const{
return m_last_frame;
}
void clear();

private:
size_t m_capacity;
std::deque<ScreenEventChange> m_changes;
size_t m_dropped = 0;
ScreenEvents m_showing;
std::optional<WallClock> m_last_frame;
};
// "+1200 ms dialog box rose", times from `origin`, one change per line.
std::string describe(const ScreenEventLog& log, WallClock origin);



} // namespace PokemonAutomation
#endif
91 changes: 91 additions & 0 deletions SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/* Screen Event Types

Check warning on line 1 in SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

<header>: header_symbols=7 (advisory above 6, limit 8)
*
* From: https://github.com/PokemonAutomation/
*
* The words a program uses for what the screen shows: a fade, a dialogue
* box, a battle starting. ScreenEvents.h watches a stream for them.
*
*/

#ifndef PokemonAutomation_CommonTools_ScreenEvents_ScreenEventTypes_H
#define PokemonAutomation_CommonTools_ScreenEvents_ScreenEventTypes_H

#include <cstddef>
#include <cstdint>
#include <initializer_list>
#include <string>

namespace PokemonAutomation{


// One thing a frame shows. A game reports the ones its detectors can see.
enum class ScreenEvent : uint8_t{
FADE_BLACK, // A uniform dark screen: a door, a cave mouth, a scene cut.
FADE_WHITE, // A uniform white screen: a cave exit.
LOCATION_BANNER, // A location banner. An observer given an expected
// map reports only that map's.
DIALOG_BOX, // A dialogue box.
CHOICE_BOX, // A choice (Yes/No, a menu of answers).
INTERACT_PROMPT, // A button prompt to interact ("A Talk").
BATTLE_STARTED, // A battle shows (its menu, its HUD or its opening).
BATTLE_ENDED, // A battle's closing screen.
PLAYER_CAN_MOVE, // The overworld with the player in control.
SCREEN_STILL, // The screen barely changed since the frame before.
SWITCH_HOME_MENU, // The Switch's HOME menu: the game is not on the screen.
};
// How many ScreenEvent values there are.
constexpr size_t SCREEN_EVENT_COUNT = 11;
// "dialog box", for the log.
std::string screen_event_name(ScreenEvent event);


// How a screen's fade looks: the kind of FADE_BLACK / FADE_WHITE.
enum class WarpFade{
NONE,
BLACK, // a door's black, or a cave mouth's dim fade
WHITE, // a cave exit's white, with no battle after it
};
// "black fade", for the log.
std::string warp_fade_name(WarpFade fade);


// The events of one frame, or of all the frames a watcher saw.
class ScreenEvents{

Check warning on line 53 in SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

PokemonAutomation::ScreenEvents: public_methods=6 (advisory above 5, limit 7)
public:
ScreenEvents() = default;
ScreenEvents(std::initializer_list<ScreenEvent> events);

bool has(ScreenEvent event) const{
return (m_bits & bit(event)) != 0;
}
// Whether any of `events` is in this set.
bool any_of(const ScreenEvents& events) const{
return (m_bits & events.m_bits) != 0;
}
// Adds `event`, or removes it when `on` is false (as std::bitset::set).
void set(ScreenEvent event, bool on);
void merge(const ScreenEvents& events){
m_bits |= events.m_bits;
}

uint32_t bits() const{
return m_bits;
}
static ScreenEvents from_bits(uint32_t bits);

bool operator==(const ScreenEvents& other) const = default;

private:
static uint32_t bit(ScreenEvent event){
return uint32_t(1) << (uint32_t)event;
}

uint32_t m_bits = 0;
};
// "dialog box, player can move"; "nothing" when empty.
std::string describe(const ScreenEvents& events);



} // namespace PokemonAutomation
#endif
188 changes: 188 additions & 0 deletions SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
/* Screen Events
*
* From: https://github.com/PokemonAutomation/
*
*/

#include <chrono>
#include <format>
#include "CommonFramework/ImageTools/ImageBoxes.h"
#include "CommonFramework/ImageTools/ImageStats.h"
#include "ScreenEventLog.h"
#include "ScreenEventTypes.h"
#include "ScreenFades.h"
#include "ScreenEvents.h"

namespace PokemonAutomation{


std::string screen_event_name(ScreenEvent event){
switch (event){
case ScreenEvent::FADE_BLACK: return "black fade";
case ScreenEvent::FADE_WHITE: return "white fade";
case ScreenEvent::LOCATION_BANNER: return "location banner";
case ScreenEvent::DIALOG_BOX: return "dialog box";
case ScreenEvent::CHOICE_BOX: return "choice box";
case ScreenEvent::INTERACT_PROMPT: return "interact prompt";
case ScreenEvent::BATTLE_STARTED: return "battle started";
case ScreenEvent::BATTLE_ENDED: return "battle ended";
case ScreenEvent::PLAYER_CAN_MOVE: return "player can move";
case ScreenEvent::SCREEN_STILL: return "screen still";
case ScreenEvent::SWITCH_HOME_MENU: return "Switch HOME menu";
}
return "?";
}

std::string warp_fade_name(WarpFade fade){
switch (fade){
case WarpFade::NONE: return "no fade";
case WarpFade::BLACK: return "black fade";
case WarpFade::WHITE: return "white fade";
}
return "?";
}


ScreenEvents::ScreenEvents(std::initializer_list<ScreenEvent> events){
for (ScreenEvent event : events){
set(event, true);
}
}
void ScreenEvents::set(ScreenEvent event, bool on){
if (on){
m_bits |= bit(event);
}else{
m_bits &= ~bit(event);
}
}
ScreenEvents ScreenEvents::from_bits(uint32_t bits){
ScreenEvents out;
out.m_bits = bits & ((uint32_t(1) << SCREEN_EVENT_COUNT) - 1);
return out;
}

ScreenEvents fade_events(WarpFade fade){
switch (fade){
case WarpFade::NONE:
return {};
case WarpFade::BLACK:
return {ScreenEvent::FADE_BLACK};
case WarpFade::WHITE:
return {ScreenEvent::FADE_WHITE};
}
return {};
}
WarpFade fade_in(const ScreenEvents& events){
if (events.has(ScreenEvent::FADE_BLACK)){
return WarpFade::BLACK;
}
if (events.has(ScreenEvent::FADE_WHITE)){
return WarpFade::WHITE;
}
return WarpFade::NONE;
}
std::string describe(const ScreenEvents& events){
std::string out;
for (size_t c = 0; c < SCREEN_EVENT_COUNT; c++){
const ScreenEvent event = (ScreenEvent)c;
if (!events.has(event)){
continue;
}
if (!out.empty()){
out += ", ";
}
out += screen_event_name(event);
}
return out.empty() ? "nothing" : out;
}


WarpFade fade_of(const ImageStats& middle){
if (middle.stddev.sum() > FADE_MAX_STDDEV_SUM){
return WarpFade::NONE;
}
const double sum = middle.average.sum();
return sum <= FADE_MAX_DARK_SUM ? WarpFade::BLACK : sum >= FADE_MIN_WHITE_SUM ? WarpFade::WHITE : WarpFade::NONE;
}
WarpFade fade_of_screen(const ImageViewRGB32& screen){
return fade_of(image_stats(extract_box_reference(screen, ImageFloatBox(0.1, 0.1, 0.8, 0.8))));
}


ScreenEventLog::ScreenEventLog(size_t capacity)
: m_capacity(capacity)
{}
void ScreenEventLog::record(const ScreenEvents& frame, WallClock now){
const uint32_t changed = frame.bits() ^ m_showing.bits();
for (size_t c = 0; c < SCREEN_EVENT_COUNT; c++){
if ((changed & (uint32_t(1) << c)) == 0){
continue;
}
const ScreenEvent event = (ScreenEvent)c;
m_changes.push_back({now, event, frame.has(event) ? ScreenEventEdge::ROSE : ScreenEventEdge::FELL});
}
while (m_changes.size() > m_capacity){
m_changes.pop_front();
m_dropped++;
}
m_showing = frame;
m_last_frame = now;
}
void ScreenEventLog::clear(){
m_changes.clear();
m_dropped = 0;
m_showing = ScreenEvents();
m_last_frame.reset();
}
std::string describe(const ScreenEventLog& log, WallClock origin){
std::string out;
if (log.dropped() > 0){
out += std::format("({} older changes dropped)\n", log.dropped());
}
for (const ScreenEventChange& change : log.changes()){
out += std::format(
"{:+} ms {} {}\n",
std::chrono::duration_cast<Milliseconds>(change.time - origin).count(),
screen_event_name(change.event),
change.edge == ScreenEventEdge::ROSE ? "rose" : "fell"
);
}
return out.empty() ? "no changes" : out;
}


ScreenEventWatcher::ScreenEventWatcher(ScreenEventObserver& observer, ScreenEvents stop_on)
: VisualInferenceCallback("ScreenEventWatcher")
, m_observer(observer)
, m_stop_on(stop_on)
{}
void ScreenEventWatcher::make_overlays(VideoOverlaySet& items) const{
m_observer.make_overlays(items);
}
bool ScreenEventWatcher::process_frame(const VideoSnapshot& frame){
const ScreenEvents events = m_observer.observe(frame, frame.timestamp);
m_seen.fetch_or(events.bits(), std::memory_order_acq_rel);
{
LockGuard<Mutex> lg(m_log_lock);
m_log.record(events, frame.timestamp);
}
if (!events.any_of(m_stop_on)){
return false;
}
m_stopped_on.store(events.bits(), std::memory_order_release);
return true;
}
ScreenEventLog ScreenEventWatcher::log() const{
LockGuard<Mutex> lg(m_log_lock);
return m_log;
}
void ScreenEventWatcher::reset(){
m_seen.store(0, std::memory_order_release);
m_stopped_on.store(0, std::memory_order_release);
LockGuard<Mutex> lg(m_log_lock);
m_log.clear();
}



} // namespace PokemonAutomation
Loading
Loading