Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
ea47c4b
AutoStory: replay recorded frames through the real watchers
dayfine Nov 9, 2026
d86938f
Merge branch 'autostory/frame-replay' into frlg/frame-replay-0027
dayfine Nov 9, 2026
1cfecb8
FRLG AutoStory: replay the 00.27 Route 10 door failures from their fr…
dayfine Nov 9, 2026
ecc08f8
FRLG frame replay: name the one-tile hold from the walk's constants
dayfine Nov 9, 2026
3b19431
Merge branch 'autostory/frame-replay' into frlg/frame-replay-0027
dayfine Nov 9, 2026
1d0cef5
AutoStory frame replay: designated test cases; keep the runner's regi…
dayfine Nov 9, 2026
1af5d31
FRLG frame replay: designated cases; register them with the screen ev…
dayfine Nov 9, 2026
3f4414b
Merge nav/story-steps into autostory/frame-replay
dayfine Nov 9, 2026
2a9ecac
AutoStory frame replay: the warp watcher watches along the hold
dayfine Nov 9, 2026
db9088a
Merge frlg/stage8-rocktunnel into frlg/frame-replay-0027 (warp fade w…
dayfine Nov 9, 2026
ea406c2
Merge autostory/frame-replay into frlg/frame-replay-0027
dayfine Nov 9, 2026
2a97556
FRLG frame replay: replay the warp watcher as the walk now runs it
dayfine Nov 9, 2026
ad654be
AutoStory frame replay: how to cut a failure capture's video into a case
dayfine Nov 9, 2026
7f5bea9
Merge origin/frlg/stage8-rocktunnel into frlg/frame-replay-0027
dayfine Nov 10, 2026
a4106b0
Merge frlg/stage8-rocktunnel into frlg/frame-replay-0027
dayfine Nov 10, 2026
09f12c3
Merge pull request #343 from dayfine/frlg/frame-replay-0027-json
dayfine Nov 10, 2026
0935898
Merge frlg/stage8-rocktunnel (a30397d52) into frlg/frame-replay-0027
dayfine Nov 11, 2026
32b0c36
Restore the blank line in ComputerPrograms_UnitTestRunner.cpp
dayfine Nov 11, 2026
338b7cd
Merge origin/nav/grid-story-steps into autostory/frame-replay
dayfine Nov 12, 2026
4b28332
Merge origin/frlg/stage8-rocktunnel into frlg/frame-replay-0027
dayfine Nov 12, 2026
5a674bd
Merge origin/autostory/frame-replay into frlg/frame-replay-0027
dayfine Nov 12, 2026
74f0852
Merge origin/frlg/stage8-rocktunnel into frlg/frame-replay-0027
dayfine Nov 12, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,10 @@
#include "Engine/Pokemon_StoryEngine_Tests.h"
#include "Pokemon/Options/Pokemon_StoryRangeOptions_Tests.h"
#include "Pokemon_AutoStorySelect_Tests.h"
#include "Pokemon_FrameReplay_Tests.h"
#include "Pokemon_StoryRun_Tests.h"
#include "Pokemon_AutoStoryStats_Tests.h"
#include "Pokemon_FrameReplay_Tests.h"
#include "Pokemon_Interrupts_Tests.h"
#include "Pokemon_SceneJudges_Tests.h"
#include "Pokemon_ScreenEvents_Tests.h"
Expand Down Expand Up @@ -59,6 +61,7 @@ inline void add_tests_AutoStory(UnitTestDatabase& database){
add_tests_SceneJudges(database);
add_tests_Interrupts(database);
add_tests_Transitions(database);
add_tests_FrameReplay(database);
add_tests_StoryActions(database);
add_tests_StoryInterrupts(database);
}
Expand Down
121 changes: 121 additions & 0 deletions SerialPrograms/Source/Pokemon/AutoStory/Pokemon_FrameReplay.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
/* Frame Replay
*
* From: https://github.com/PokemonAutomation/
*
*/

#include <algorithm>
#include <cctype>
#include "Common/Cpp/Exceptions.h"
#include "Common/Cpp/Filesystem/Filesystem.h"
#include "Pokemon_FrameReplay.h"

namespace PokemonAutomation{
namespace Pokemon{


namespace{

// The time a frame file's name gives ("1250.png" -> 1250), or -1 when the
// name is not all digits or the file not a .png.
long long frame_time_of(const Filesystem::Path& file){
if (file.extension().string() != ".png"){
return -1;
}
const std::string stem = file.stem().string();
if (stem.empty() || stem.size() > 9){
return -1;
}
for (char c : stem){
if (!std::isdigit((unsigned char)c)){
return -1;
}
}
return std::stoll(stem);
}

} // namespace


ReplayFrameSource::ReplayFrameSource(std::vector<ReplayFrame> frames)
: m_frames(std::move(frames))
{
std::stable_sort(m_frames.begin(), m_frames.end(), [](const ReplayFrame& a, const ReplayFrame& b){
return a.at < b.at;
});
}

ReplayFrameSource ReplayFrameSource::from_folder(const std::string& folder){
std::vector<ReplayFrame> frames;
if (Filesystem::is_directory(folder)){
for (const Filesystem::Path& file : Filesystem::list_directory(folder)){
const long long ms = frame_time_of(file);
if (ms >= 0){
frames.push_back(ReplayFrame{.at = Milliseconds(ms), .file = file.string()});
}
}
}
if (frames.empty()){
throw FileException(nullptr, PA_CURRENT_FUNCTION, "No <ms>.png frames to replay.", folder);
}
return ReplayFrameSource(std::move(frames));
}

Milliseconds ReplayFrameSource::last_frame() const{
return m_frames.empty() ? Milliseconds(0) : m_frames.back().at;
}

VideoSnapshot ReplayFrameSource::frame_at(Milliseconds at, WallClock origin) const{
auto after = std::upper_bound(m_frames.begin(), m_frames.end(), at, [](Milliseconds time, const ReplayFrame& frame){
return time < frame.at;
});
if (after == m_frames.begin()){
return VideoSnapshot();
}
ReplayFrame& frame = *(after - 1);
if (!frame.image){
frame.image = std::make_shared<const ImageRGB32>(frame.file);
}
VideoSnapshot out;
out.frame = frame.image;
out.timestamp = origin + at;
return out;
}


ReplayHit replay_until(
const ReplayFrameSource& source, ReplayWindow window,
const std::vector<VisualInferenceCallback*>& callbacks,
WallClock origin, Milliseconds period
){
for (Milliseconds now = window.start; now < window.end; now += period){
const VideoSnapshot frame = source.frame_at(now, origin);
if (!frame){
continue;
}
for (size_t c = 0; c < callbacks.size(); c++){
if (callbacks[c]->process_frame(frame)){
return ReplayHit{.index = (int)c, .at = now};
}
}
}
return ReplayHit{};
}

ScreenEventLog replay_events(

Check warning on line 105 in SerialPrograms/Source/Pokemon/AutoStory/Pokemon_FrameReplay.cpp

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

style: replay_events: output parameter 'ScreenEventObserver& observer' comes before an input; put inputs (by value, const&) first and non-const references last (Google "Function Parameter Ordering")
const ReplayFrameSource& source, ReplayWindow window,
ScreenEventObserver& observer,
WallClock origin, Milliseconds period
){
ScreenEventLog log;
for (Milliseconds now = window.start; now < window.end; now += period){
const VideoSnapshot frame = source.frame_at(now, origin);
log.record(observer.observe(frame, origin + now), origin + now);
}
return log;
}



} // namespace Pokemon
} // namespace PokemonAutomation
125 changes: 125 additions & 0 deletions SerialPrograms/Source/Pokemon/AutoStory/Pokemon_FrameReplay.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
/* Frame Replay

Check warning on line 1 in SerialPrograms/Source/Pokemon/AutoStory/Pokemon_FrameReplay.h

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

<header>: header_symbols=7 (advisory above 6, limit 8)
*
* From: https://github.com/PokemonAutomation/
*
* Recorded console frames, replayed through the real detectors on a
* virtual clock, so a console failure becomes a deterministic test. The
* story sims script what a screen shows (a label per frame); a replay
* shows what the console showed and lets the game's own watchers and
* ScreenEventObserver judge it.
*
* A recording is a folder of frames named by their time in ms from its
* start ("0.png", "1250.png"), as the captured transitions
* (Pokemon_Transitions_Tests.cpp). A frame shows until the next one.
* Recordings are too big for this repo; tests read them from
* CommandLineTests/<game>/FrameReplay/<case>/ and skip when it is missing.
*
* From the stream history (an error report's Video.mp4, kept when the
* "Stream History" setting is on), cut the seconds around the failure at
* 20 frames a second, then rename frame N to its time, (N - 1) * 50 ms:
* ffmpeg -ss <start s> -t <seconds> -i Video.mp4 -vf fps=20 %d.png
*
* A failed AutoStory event saves its own (Pokemon_AutoStoryFailureCapture.h)
* in USER_FILE_PATH()/<game>/AutoStoryFailures/<time>-<checkpoint>-<event>/,
* whatever the error-report settings. Its Video.mp4 ends when the event
* failed, so the last seconds are the case; Failure.json names the event
* and its last log lines, which say where in those seconds each action
* started. For the last 20 s, into the case folder:
* ffmpeg -sseof -20 -i Video.mp4 -vf fps=20 %d.png
* for f in *.png; do n=${f%.png}; mv "$f" "x$(( (n - 1) * 50 )).png"; done
* for f in x*.png; do mv "$f" "${f#x}"; done
*
* One thread, no real time: the caller says which window of the recording
* each action watched, and the replay feeds those frames at the inference
* period, as run_until / wait_until would.
*
*/

#ifndef PokemonAutomation_Pokemon_FrameReplay_H
#define PokemonAutomation_Pokemon_FrameReplay_H

#include <memory>
#include <string>
#include <vector>
#include "Common/Cpp/Time.h"
#include "CommonFramework/ImageTypes/ImageRGB32.h"
#include "CommonFramework/VideoPipeline/VideoFeed.h"
#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h"
#include "Pokemon_ScreenEvents.h"

namespace PokemonAutomation{
namespace Pokemon{


// One recorded frame: when it showed, from the recording's start, and the
// image, or the file it loads from on first use.
struct ReplayFrame{
Milliseconds at = Milliseconds(0);
std::string file = {};
std::shared_ptr<const ImageRGB32> image = {};
};


// Recorded frames, served by virtual time. Not thread-safe: images load
// on first use.
class ReplayFrameSource{
public:
explicit ReplayFrameSource(std::vector<ReplayFrame> frames);

// Every "<ms>.png" in `folder`. Throws FileException if it has none.
static ReplayFrameSource from_folder(const std::string& folder);

size_t size() const{
return m_frames.size();
}
// When the last frame showed.
Milliseconds last_frame() const;

// The frame showing at `at`: the last one at or before it, stamped
// `origin + at`. None before the first frame.
VideoSnapshot frame_at(Milliseconds at, WallClock origin) const;

private:
mutable std::vector<ReplayFrame> m_frames;
};


// How often a replay looks: the inference loop's default period.
constexpr Milliseconds REPLAY_PERIOD = Milliseconds(50);

// Part of a recording an action watched, in ms from its start, end
// excluded: the hold of a leg run_until watched, or a wait_until after it.
struct ReplayWindow{
Milliseconds start = Milliseconds(0);
Milliseconds end = Milliseconds(0);
};

// The first callback that stopped a replay, and when.
struct ReplayHit{
// Its index in the callbacks; -1 if none did.
int index = -1;
Milliseconds at = Milliseconds(0);
};

// Feeds `callbacks` the frames of `window`, one look per `period`, each
// callback in order on each look, until one returns true: wait_until on a
// recording. Frames are stamped `origin + time`.
ReplayHit replay_until(
const ReplayFrameSource& source, ReplayWindow window,
const std::vector<VisualInferenceCallback*>& callbacks,
WallClock origin, Milliseconds period = REPLAY_PERIOD
);

// What `observer` sees over `window`, one look per `period`, as a
// ScreenEventWatcher's log; the judges (Pokemon_Interrupts.h) read it.
ScreenEventLog replay_events(

Check warning on line 115 in SerialPrograms/Source/Pokemon/AutoStory/Pokemon_FrameReplay.h

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

style: replay_events: output parameter 'ScreenEventObserver& observer' comes before an input; put inputs (by value, const&) first and non-const references last (Google "Function Parameter Ordering")
const ReplayFrameSource& source, ReplayWindow window,
ScreenEventObserver& observer,
WallClock origin, Milliseconds period = REPLAY_PERIOD
);



} // namespace Pokemon
} // namespace PokemonAutomation
#endif
Loading
Loading