Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 136 additions & 0 deletions SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
/* Screen Machine
*
* From: https://github.com/PokemonAutomation/
*
*/

#include <format>
#include <map>
#include <string>
#include <utility>
#include "CommonTools/ScreenEvents/ScreenEvents.h"
#include "ScreenMachine.h"

namespace PokemonAutomation{


namespace{


// The first of `states` whose matches() accepts `observation`; null when
// none does.
const ScreenState* match_screen_state(const std::vector<ScreenState>& states, const ScreenObservation& observation){
for (const ScreenState& state : states){
if (state.matches && state.matches(observation)){
return &state;
}
}
return nullptr;
}


// The checks between looks: no state may come up too often, and steps must
// keep changing the screen (a waiting state's only for its
// ScreenState::max_still).
class ScreenMachineChecks{
public:
// A step that changes nothing on screen is a dead end (e.g. a menu the
// states do not know): fail after this many instead of running out the
// visit limit.
static constexpr size_t MAX_UNCHANGED = 8;

// The problem with `state` coming up on `observation`, if any. The
// screen counts as unchanged only when it and the look before both have
// a frame and the frames match (same_screen()); how long, by the looks'
// `when`.
std::optional<std::string> problem(const ScreenState& state, const ScreenObservation& observation);

private:
// The stall problem, if any, of `state` on a frame unchanged since
// `m_changed_at`.
std::optional<std::string> unchanged_problem(const ScreenState& state, WallClock when);

std::map<std::string, size_t, std::less<>> m_visits;
VideoSnapshot m_last_frame;
// When the screen last changed (or was first looked at).
WallClock m_changed_at = WallClock::min();
size_t m_unchanged = 0;
};


std::optional<std::string> ScreenMachineChecks::unchanged_problem(const ScreenState& state, WallClock when){
if (state.max_still > Milliseconds::zero()){
if (when - m_changed_at > state.max_still){
return std::format("The screen stayed still on \"{}\" for more than {}s.", state.name, state.max_still.count() / 1000);
}
return std::nullopt;
}
if (++m_unchanged >= MAX_UNCHANGED){
return std::format("The screen stopped changing: {} steps on \"{}\" did nothing.", MAX_UNCHANGED, state.name);
}
return std::nullopt;
}

std::optional<std::string> ScreenMachineChecks::problem(const ScreenState& state, const ScreenObservation& observation){
VideoSnapshot frame = observation.frame ? *observation.frame : VideoSnapshot();
if (same_screen(m_last_frame, frame)){
std::optional<std::string> stalled = unchanged_problem(state, observation.when);
if (stalled){
return stalled;
}
}else{
m_unchanged = 0;
m_changed_at = observation.when;
}
m_last_frame = std::move(frame);
if (++m_visits[state.name] > state.max_visits){
return std::format("Screen \"{}\" came up more than {} times: the step for it is not working.", state.name, state.max_visits);
}
return std::nullopt;
}


} // namespace



ScreenMachineResult run_screen_machine(
const std::function<ScreenObservation()>& observe, CancellableScope& scope,
const std::vector<ScreenState>& states, Milliseconds timeout, StepTrace& trace
){
// Looks before the run stops even when nothing else ends it (looks with
// no time, states with no visit limit).
constexpr size_t MAX_LOOKS = 10000;
ScreenMachineChecks checks;
std::optional<WallClock> start;
for (size_t looks = 0; looks < MAX_LOOKS; looks++){
scope.throw_if_cancelled();
const ScreenObservation observation = observe();
if (!start){
start = observation.when;
}
if (observation.when - *start > timeout){
return {.terminal = "", .failure = std::format("Did not finish within {}s.", timeout.count() / 1000)};
}
const ScreenState* state = match_screen_state(states, observation);
if (state == nullptr){
return {.terminal = "", .failure = "The screen matches none of the known states."};
}
trace.screen(state->name, observation.when);
std::optional<std::string> found = checks.problem(*state, observation);
if (found){
return {.terminal = "", .failure = std::move(*found)};
}
if (state->act){
state->act();
}
if (state->terminal){
return {.terminal = state->name, .failure = ""};
}
}
return {.terminal = "", .failure = std::format("Did not finish within {} looks.", MAX_LOOKS)};
}



} // namespace PokemonAutomation
81 changes: 81 additions & 0 deletions SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
/* Screen Machine
*
* From: https://github.com/PokemonAutomation/
*
* A closed loop over known screens: look at the game, take the one step that
* screen calls for, look again. Every step follows a look, and each state says
* how often it may come up and how long the screen may stay unchanged on it,
* so a dropped or wrong input shows up as a state that keeps coming back or a
* screen that stops changing. The looks are ScreenObservations, so the same
* machine runs on a console and on a scripted backend.
*
*/

#ifndef PokemonAutomation_CommonTools_ScreenMachine_H
#define PokemonAutomation_CommonTools_ScreenMachine_H

#include <stdint.h>
#include <functional>
#include <optional>
#include <string>
#include <vector>
#include "Common/Cpp/Time.h"
#include "Common/Cpp/CancellableScope.h"
#include "CommonFramework/VideoPipeline/VideoFeed.h"
#include "ScreenObservation.h"
#include "StepTrace.h"

namespace PokemonAutomation{


// One recognizable screen: how to tell it, and the step to take on it.
struct ScreenState{
std::string name;
// Whether a look shows this screen. States are tried in order; the first
// match wins.
std::function<bool(const ScreenObservation& observation)> matches;
// The one step to take on this screen; none for a state that only
// waits.
std::function<void()> act = nullptr;
// Fail once the screen has been seen more than this many times.
size_t max_visits = SIZE_MAX;
// End the run after this state's step.
bool terminal = false;
// How long the screen may stay unchanged on this state. A waiting
// state (a fade, a still cutscene, a hold before a screen counts) sets
// it: its looks never count as a stall, and it fails only once the
// screen has not changed for longer than this. Zero: the state's step
// is input that should change the screen, and
// ScreenMachineChecks::MAX_UNCHANGED (ScreenMachine.cpp) such steps
// that do not are a stall.
Milliseconds max_still = Milliseconds::zero();
};


// How a run of states ended.
struct ScreenMachineResult{
// The terminal state that acted; empty when the run failed.
std::string terminal;
// Why it failed: no state matched, a state seen too often, a stall, or
// the timeout.
std::string failure;

bool succeeded() const{
return failure.empty();
}
};

// Runs `states` over looks from `observe` until a terminal one has acted:
// each look picks the first state that matches it, `trace` records it, the
// checks apply, then the state acts. Time is the looks' own (`when`): the
// run fails once `timeout` has passed since the first look, and after
// 10000 looks in any case. A failure is returned, not thrown.
ScreenMachineResult run_screen_machine(
const std::function<ScreenObservation()>& observe, CancellableScope& scope,
const std::vector<ScreenState>& states, Milliseconds timeout, StepTrace& trace
);



} // namespace PokemonAutomation
#endif
Loading
Loading