Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
177 changes: 177 additions & 0 deletions SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,177 @@
/* Screen Machine
*
* From: https://github.com/PokemonAutomation/
*
*/

#include <algorithm>
#include <format>
#include <string>
#include <utility>
#include "CommonFramework/ImageTools/ImageDiff.h"
#include "ScreenMachine.h"

namespace PokemonAutomation{

using namespace std::chrono_literals;


namespace{

const ScreenState* classify(const std::vector<ScreenState>& states, const ImageViewRGB32& screen){
for (const ScreenState& state : states){
if (state.matches && state.matches(screen)){
return &state;
}
}
return nullptr;
}

} // namespace


bool same_screen(const VideoSnapshot& a, const VideoSnapshot& b){
if (!a || !b || a->width() != b->width() || a->height() != b->height()){
return false;
}
return ImageMatch::pixel_RMSD(a, b) < 2.0;
}

const ScreenState* match_screen_state(const std::vector<ScreenState>& states, const ScreenObservation& observation){
for (const ScreenState& state : states){
if (state.matches_observation && state.matches_observation(observation)){
return &state;
}
}
return nullptr;
}


const ScreenState& read_screen_state(

Check warning on line 50 in SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

PokemonAutomation::read_screen_state( VideoStream & stream , CancellableScope & scope , const std :: vector<ScreenState> & states , StepTrace & trace , VideoSnapshot & frame): function_lines=33 (advisory above 25, limit 50)
VideoStream& stream, CancellableScope& scope,
const std::vector<ScreenState>& states, StepTrace& trace, VideoSnapshot& frame
){
const ScreenState* last = nullptr;
WallClock video_lost = WallClock::max();
for (size_t c = 0; c < 20; c++){
VideoSnapshot screen = stream.video().snapshot();
if (!screen){
// The app resets a frozen capture by itself, which takes a second or two.
const WallClock now = current_time();
video_lost = std::min(video_lost, now);
if (now - video_lost > std::chrono::seconds(10)){
trace.fail("No video from the capture card for 10s.");
}
c--;
scope.wait_for(500ms);
continue;
}
video_lost = WallClock::max();
const ScreenState* now = classify(states, screen);
frame = std::move(screen);
if (now != nullptr && now == last){
return *now;
}
last = now;
scope.wait_for(150ms);
}
if (last == nullptr){
trace.fail("The screen matches none of the known states.");
}
return *last; // Still changing after 3s: go with the latest read.
}


void ScreenMachineChecks::check(const ScreenState& state, VideoSnapshot frame){
const std::optional<std::string> found = problem(state, std::move(frame), current_time());
if (found){
m_trace.fail(*found);
}
}

std::optional<std::string> ScreenMachineChecks::problem(const ScreenState& state, const ScreenObservation& observation){
return problem(state, observation.frame ? *observation.frame : VideoSnapshot(), observation.when);
}

std::optional<std::string> ScreenMachineChecks::unchanged_problem(const ScreenState& state, WallClock when){
if (state.max_still > Milliseconds::zero()){
if (when - m_changed_at > state.max_still){
return std::format("The screen stayed still on \"{}\" for more than {}s.", state.name, state.max_still.count() / 1000);
}
return std::nullopt;
}
if (++m_unchanged >= MAX_UNCHANGED){
return std::format("The screen stopped changing: {} steps on \"{}\" did nothing.", MAX_UNCHANGED, state.name);
}
return std::nullopt;
}

std::optional<std::string> ScreenMachineChecks::problem(const ScreenState& state, VideoSnapshot frame, WallClock when){

Check warning on line 109 in SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

PokemonAutomation::ScreenMachineChecks::problem( const ScreenState & state , VideoSnapshot frame , WallClock when): function_lines=27 (advisory above 25, limit 50)
if (same_screen(m_last_frame, frame)){
std::optional<std::string> stalled = unchanged_problem(state, when);
if (stalled){
return stalled;
}
}else{
m_unchanged = 0;
m_changed_at = when;
}
m_last_frame = std::move(frame);

if (m_previous != nullptr && m_previous != &state && !m_previous->next.empty() &&
std::find(m_previous->next.begin(), m_previous->next.end(), state.name) == m_previous->next.end()
){
return std::format("Unexpected screen after \"{}\": \"{}\".", m_previous->name, state.name);
}
m_in_a_row = m_previous == &state ? m_in_a_row + 1 : 1;
if (m_in_a_row > state.max_in_a_row){
return std::format("Screen \"{}\" came up {} times in a row: the step for it is not getting anywhere.", state.name, m_in_a_row);
}
if (++m_visits[state.name] > state.max_visits){
return std::format("Screen \"{}\" came up more than {} times: the step for it is not working.", state.name, state.max_visits);
}
m_previous = &state;
return std::nullopt;
}


ScreenMachineResult run_screen_machine_on_observations(

Check warning on line 138 in SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

PokemonAutomation::run_screen_machine_on_observations( const std :: function<ScreenObservation()> & observe , CancellableScope & scope , const std :: vector<ScreenState> & states , Milliseconds timeout , StepTrace & trace): function_lines=36 (advisory above 25, limit 50)
const std::function<ScreenObservation()>& observe, CancellableScope& scope,
const std::vector<ScreenState>& states, Milliseconds timeout, StepTrace& trace
){
// Looks before the run stops even when nothing else ends it (looks with
// no time, states with no visit limit).
constexpr size_t MAX_LOOKS = 10000;
ScreenMachineChecks checks(trace);
std::optional<WallClock> start;
for (size_t looks = 0; looks < MAX_LOOKS; looks++){
scope.throw_if_cancelled();
const ScreenObservation observation = observe();
if (!start){
start = observation.when;
}
if (observation.when - *start > timeout){
return {.terminal = "", .failure = std::format("Did not finish within {}s.", timeout.count() / 1000)};
}
const ScreenState* state = match_screen_state(states, observation);
if (state == nullptr){
return {.terminal = "", .failure = "The screen matches none of the known states."};
}
trace.screen(state->name);
std::optional<std::string> found = checks.problem(*state, observation);
if (found){
return {.terminal = "", .failure = std::move(*found)};
}
if (state->act){
state->act();
}
if (state->terminal){
return {.terminal = state->name, .failure = ""};
}
}
return {.terminal = "", .failure = std::format("Did not finish within {} looks.", MAX_LOOKS)};
}



} // namespace PokemonAutomation
176 changes: 176 additions & 0 deletions SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
/* Screen Machine

Check warning on line 1 in SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

<header>: header_symbols=8 (advisory above 6, limit 8)
*
* From: https://github.com/PokemonAutomation/
*
* A closed loop over known screens: read the screen, take the one step that screen
* calls for, read again. Every step follows a read, a screen counts only once two
* frames agree, and each state lists the screens it may lead to, so a dropped or
* wrong input shows up as an unexpected screen or a state that keeps coming back.
*
*/

#ifndef PokemonAutomation_CommonTools_ScreenMachine_H
#define PokemonAutomation_CommonTools_ScreenMachine_H

#include <format>
#include <stdint.h>
#include <functional>
#include <map>
#include <optional>
#include <string>
#include <vector>
#include "Common/Cpp/Time.h"
#include "Common/Cpp/CancellableScope.h"
#include "CommonFramework/ImageTypes/ImageViewRGB32.h"
#include "CommonFramework/Tools/VideoStream.h"
#include "CommonFramework/VideoPipeline/VideoFeed.h"
#include "ScreenObservation.h"
#include "StepTrace.h"

namespace PokemonAutomation{


// Single recognizable screen state, including detection predicate and handler action.
struct ScreenState{
std::string name;
// Whether this is the screen. States are tried in order; the first match wins, so
// the last state should match anything.
std::function<bool(const ImageViewRGB32& screen)> matches;
// Whether this is the screen, from what a look at the game observed
// instead of from its pixels, so the state also runs in a simulation.
// A state may have either or both; read_screen_state() uses `matches`
// and skips a state without it.
std::function<bool(const ScreenObservation& observation)> matches_observation = nullptr;
// The one step to take on this screen; none for a state that only
// waits.
std::function<void()> act;
// The screens the step may lead to besides staying here. Empty: any.
std::vector<std::string> next;
// Fail once the screen has been seen this many times.
size_t max_visits = SIZE_MAX;
// End the run after this state's step.
bool terminal = false;
// Fail once the screen has been seen this many times in a row.
size_t max_in_a_row = SIZE_MAX;
// How long the screen may stay unchanged on this state. A waiting
// state (a fade, a still cutscene, a hold before a screen counts) sets
// it: its looks never count as a stall, and it fails only once the
// screen has not changed for longer than this. Zero: the state's step
// is input that should change the screen, and MAX_UNCHANGED such
// steps that do not are a stall.
Milliseconds max_still = Milliseconds::zero();
};


// Whether `a` and `b` both have a frame of the same size within the
// still-screen RMSD threshold (2.0).
bool same_screen(const VideoSnapshot& a, const VideoSnapshot& b);

// The state on screen, once two frames 150ms apart agree. Waits out a video
// gap of up to 10s. Fails through `trace` if no state matches.
const ScreenState& read_screen_state(
VideoStream& stream, CancellableScope& scope,
const std::vector<ScreenState>& states, StepTrace& trace, VideoSnapshot& frame
);

// The first of `states` whose matches_observation() accepts `observation`;
// null when none does. States without one are skipped.
const ScreenState* match_screen_state(const std::vector<ScreenState>& states, const ScreenObservation& observation);

// The checks between reads: the screen must be one the previous state can lead
// to, no state may come up too often, and steps must keep changing the screen
// (a waiting state's only for its ScreenState::max_still).
class ScreenMachineChecks{
public:
ScreenMachineChecks(StepTrace& trace)
: m_trace(trace)
{}
// Fails through the trace on a problem.
void check(const ScreenState& state, VideoSnapshot frame);
// The problem with `state` coming up on `observation`, if any. The
// screen counts as unchanged only when it and the look before both have
// a frame and the frames match; how long, by the looks' `when`.
std::optional<std::string> problem(const ScreenState& state, const ScreenObservation& observation);

private:
std::optional<std::string> problem(const ScreenState& state, VideoSnapshot frame, WallClock when);
// The stall problem, if any, of `state` on a frame unchanged since `m_changed_at`.
std::optional<std::string> unchanged_problem(const ScreenState& state, WallClock when);

// A step that changes nothing on screen is a dead end (e.g. a menu the states do
// not know): fail after a few instead of running out the visit limit.
static constexpr size_t MAX_UNCHANGED = 8;

StepTrace& m_trace;
std::map<std::string, size_t> m_visits;
const ScreenState* m_previous = nullptr;
VideoSnapshot m_last_frame;
// When the screen last changed (or was first read).
WallClock m_changed_at = WallClock::min();
size_t m_unchanged = 0;
size_t m_in_a_row = 0;
};


// Runs the states until a terminal one has acted. Returns its name. Fails through
// `trace` on an unexpected screen, a state seen too often, a stall or the timeout.
template <typename ControllerContext>
std::string run_screen_machine(
VideoStream& stream, ControllerContext& context,
const std::vector<ScreenState>& states, Milliseconds timeout, StepTrace& trace
);


// How a run of states over observations ended.
struct ScreenMachineResult{
// The terminal state that acted; empty when the run failed.
std::string terminal;
// Why it failed: no state matched, an unexpected screen, a state seen too
// often, a stall, or the timeout.
std::string failure;

bool succeeded() const{
return failure.empty();
}
};

// The same loop as run_screen_machine() over looks from `observe` instead of
// frames, so it runs on a console and in a simulation alike: each look picks
// the first state whose matches_observation() accepts it (one look, no second
// read: an observation is already settled), the trace records it, the checks
// apply, then the state acts. Time is the looks' own (`when`): the run fails
// once `timeout` has passed since the first look, and after 10000 looks in
// any case. A failure is returned, not thrown.
ScreenMachineResult run_screen_machine_on_observations(
const std::function<ScreenObservation()>& observe, CancellableScope& scope,
const std::vector<ScreenState>& states, Milliseconds timeout, StepTrace& trace
);

template <typename ControllerContext>
std::string run_screen_machine(
VideoStream& stream, ControllerContext& context,
const std::vector<ScreenState>& states, Milliseconds timeout, StepTrace& trace
){
ScreenMachineChecks checks(trace);
const WallClock deadline = current_time() + timeout;
while (current_time() < deadline){
context.wait_for_all_requests();
VideoSnapshot frame;
const ScreenState& state = read_screen_state(stream, context, states, trace, frame);
trace.screen(state.name);
checks.check(state, std::move(frame));
if (state.act){
state.act();
}
if (state.terminal){
context.wait_for_all_requests();
return state.name;
}
}
trace.fail(std::format("Did not finish within {}s.", timeout.count() / 1000));
}



} // namespace PokemonAutomation
#endif
Loading
Loading