From 0e0af12d8d32816766b761263ba3ddaf4497567a Mon Sep 17 00:00:00 2001 From: dayfine Date: Fri, 13 Nov 2026 15:10:37 -0800 Subject: [PATCH 1/2] Pokemon Navigation: leg trace and replay of console failures The shared [Leg] record writer and reader (LegTracer), the fault-file reader for replaying a console failure on a sim (LegReplay), their unit tests, and the Python grader and fault-file tools under devtools/maps. Taken unchanged from the FRLG walker commit and nav/leg-replay (#447) so both games build on one copy. (cherry picked from the shared parts of commit 9499c3f9bb0befee849815a3f233faf747f3d7ad and devtools/maps/leg_replay.py from 98c0b6177) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../Pokemon/Navigation/Pokemon_LegReplay.cpp | 133 +++++ .../Pokemon/Navigation/Pokemon_LegReplay.h | 107 ++++ .../Pokemon/Navigation/Pokemon_LegTrace.cpp | 267 ++++++++++ .../Pokemon/Navigation/Pokemon_LegTrace.h | 167 +++++++ .../Navigation/Pokemon_LegTrace_Tests.cpp | 167 +++++++ .../Navigation/Pokemon_LegTrace_Tests.h | 23 + .../Navigation/Pokemon_RouteLog_Tests.cpp | 3 + SerialPrograms/cmake/SourceFiles.cmake | 6 + devtools/maps/leg_replay.py | 111 +++++ devtools/maps/leg_trace.py | 463 ++++++++++++++++++ devtools/maps/tests/test_leg_trace.py | 228 +++++++++ 11 files changed, 1675 insertions(+) create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegReplay.cpp create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegReplay.h create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace.cpp create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace.h create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace_Tests.cpp create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace_Tests.h create mode 100644 devtools/maps/leg_replay.py create mode 100644 devtools/maps/leg_trace.py create mode 100644 devtools/maps/tests/test_leg_trace.py diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegReplay.cpp b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegReplay.cpp new file mode 100644 index 0000000000..d593f1949c --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegReplay.cpp @@ -0,0 +1,133 @@ +/* Leg Replay + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "Common/Cpp/Json/JsonArray.h" +#include "Common/Cpp/Json/JsonObject.h" +#include "Common/Cpp/Json/JsonValue.h" +#include "Pokemon_LegReplay.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + +namespace{ + +[[noreturn]] void bad(const std::string& name, const std::string& what){ + throw std::runtime_error("console failure " + name + ": " + what); +} + +Direction direction_of(const std::string& name, const std::string& text){ + for (const Direction direction : ALL_DIRECTIONS){ + if (direction_name(direction) == text){ + return direction; + } + } + bad(name, "unknown direction \"" + text + "\""); +} + +TilePos tile_of(const std::string& name, const JsonArray& array){ + if (array.size() != 2){ + bad(name, "a tile is [x, y]"); + } + return {(int)array[0].to_integer_throw(), (int)array[1].to_integer_throw()}; +} + +ReplayFade fade_of(const std::string& name, const std::string& text){ + if (text == "unseen"){ + return ReplayFade::UNSEEN; + } + if (text == "black"){ + return ReplayFade::BLACK; + } + if (text == "dim"){ + return ReplayFade::DIM; + } + if (text == "white"){ + return ReplayFade::WHITE; + } + bad(name, "unknown fade \"" + text + "\""); +} + +ReplayFault fault_of(const std::string& name, const JsonObject& json){ + const std::string kind = json.get_string_throw("kind"); + ReplayFault fault; + if (kind == "fade_missed" || kind == "banner_missed"){ + fault.kind = kind == "fade_missed" ? ReplayFaultKind::FADE_MISSED : ReplayFaultKind::BANNER_MISSED; + fault.index = (size_t)json.get_integer_throw("index"); + }else if (kind == "moved_further" || kind == "hold_through_warp" || kind == "scene_ends_further"){ + fault.kind = kind == "moved_further" ? ReplayFaultKind::MOVED_FURTHER + : kind == "hold_through_warp" ? ReplayFaultKind::HOLD_THROUGH_WARP : ReplayFaultKind::SCENE_ENDS_FURTHER; + fault.step = json.get_string_throw("step"); + fault.direction = direction_of(name, json.get_string_throw("direction")); + fault.tiles = (size_t)json.get_integer_default("tiles", 1); + fault.fade = fade_of(name, json.get_string_default("fade", "unseen")); + fault.far_pose_of = json.get_string_default("far_pose_of", ""); + }else if (kind == "moved_less"){ + fault.kind = ReplayFaultKind::MOVED_LESS; + fault.step = json.get_string_throw("step"); + fault.index = (size_t)json.get_integer_throw("index"); + fault.tiles = (size_t)json.get_integer_throw("tiles"); + }else if (kind == "person_stands"){ + fault.kind = ReplayFaultKind::PERSON_STANDS; + fault.name = json.get_string_throw("name"); + fault.tile = tile_of(name, json.get_array_throw("tile")); + }else{ + bad(name, "unknown fault kind \"" + kind + "\""); + } + return fault; +} + +} // namespace + + +std::string replay_fault_kind_name(ReplayFaultKind kind){ + switch (kind){ + case ReplayFaultKind::FADE_MISSED: + return "fade_missed"; + case ReplayFaultKind::BANNER_MISSED: + return "banner_missed"; + case ReplayFaultKind::MOVED_FURTHER: + return "moved_further"; + case ReplayFaultKind::HOLD_THROUGH_WARP: + return "hold_through_warp"; + case ReplayFaultKind::SCENE_ENDS_FURTHER: + return "scene_ends_further"; + case ReplayFaultKind::PERSON_STANDS: + return "person_stands"; + case ReplayFaultKind::MOVED_LESS: + return "moved_less"; + } + return ""; +} + +ConsoleFailure parse_console_failure(const std::string& name, const std::string& text){ + const JsonValue value = parse_json(text); + const JsonObject& json = value.to_object_throw(); + ConsoleFailure out; + out.name = name; + out.checkpoint = json.get_string_throw("checkpoint"); + if (const JsonObject* start = json.get_object("start")){ + out.start = ReplayTile{start->get_string_throw("map"), tile_of(name, start->get_array_throw("tile"))}; + out.start_facing = direction_of(name, start->get_string_throw("facing")); + } + for (const auto& item : json.get_array_throw("faults")){ + out.faults.push_back(fault_of(name, item.to_object_throw())); + } + const JsonObject& failed = json.get_object_throw("failed"); + out.failed_step = failed.get_string_throw("step"); + out.failed_error = failed.get_string_throw("error"); + out.frame = json.get_string_throw("frame"); + if (const JsonArray* evidence = json.get_array("evidence")){ + for (const auto& line : *evidence){ + out.evidence.push_back(line.to_string_throw()); + } + } + return out; +} + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegReplay.h b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegReplay.h new file mode 100644 index 0000000000..4f699912c1 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegReplay.h @@ -0,0 +1,107 @@ +/* Leg Replay + * + * From: https://github.com/PokemonAutomation/ + * + * A console failure as a small fault file (the files of a game's ConsoleFailures folder beside + * a game's simulation tests), made from a console log's [Leg] records and + * raw walk facts by devtools/maps/leg_replay.py. The file keeps only what + * the simulation cannot work out from its map: where the run started if + * not on its checkpoint's start, the detections it missed, the walks that + * went further than counted, and the people the map lacks. Trainers, NPCs + * and walls come from the map. Where the failure happened and the frame it + * was checked against ride along for the reader. + * + * {"checkpoint": "01.02", + * "faults": [{"kind": "hold_through_warp", "step": "past_curtis", + * "direction": "east", "fade": "unseen", "far_pose_of": "into_oreburgh"}], + * "failed": {"step": "into_oreburgh", "error": "no_banner"}, +// code-health: allow header_symbols -- the fault file model and its reader are one unit, a split would scatter it + */ + +#ifndef PokemonAutomation_Pokemon_LegReplay_H +#define PokemonAutomation_Pokemon_LegReplay_H + +#include +#include +#include +#include +#include "Pokemon_Direction.h" +#include "Pokemon_Grid.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// A tile of a named map. +struct ReplayTile{ + std::string map; + TilePos tile; +}; + +// What went differently on the console than the program counted on. +enum class ReplayFaultKind{ + FADE_MISSED, // the `index`th fade of the run goes unseen + BANNER_MISSED, // the `index`th map banner of the run goes unseen + MOVED_FURTHER, // the step's counted taps went `tiles` further (onto a warp, `far_pose_of`) + HOLD_THROUGH_WARP, // the step's hold toward `direction` ran on through a warp (`far_pose_of`) + SCENE_ENDS_FURTHER, // the player stood `tiles` toward `direction` past where the scene step leaves them + PERSON_STANDS, // `name`, a person of the map (who wanders or stays put), stood on `tile` + MOVED_LESS, // the `index`th hold of `step`'s walk (from 1) moved `tiles` fewer than it was held for, with + // nothing on screen: the walker took the move as seen +}; +// "fade_missed", "banner_missed", "moved_further", "hold_through_warp", "scene_ends_further", "person_stands", +// "moved_less". +std::string replay_fault_kind_name(ReplayFaultKind kind); + +// How the walk saw the fade of a warp it went through. +enum class ReplayFade{ + UNSEEN, + BLACK, + DIM, + WHITE, +}; + +// One fault; the fields of its kind (above) are the ones that count. +struct ReplayFault{ + ReplayFaultKind kind = ReplayFaultKind::FADE_MISSED; + // FADE_MISSED, BANNER_MISSED: counted within the checkpoint, from 0. + // MOVED_LESS: counted within the step's walk, from 1. + size_t index = 0; + // MOVED_FURTHER, HOLD_THROUGH_WARP, SCENE_ENDS_FURTHER, MOVED_LESS. + std::string step; + Direction direction = Direction::EAST; + size_t tiles = 1; + ReplayFade fade = ReplayFade::UNSEEN; + bool banner_up = false; + // The step of the checkpoint whose pose after it is where the warp lands. + std::string far_pose_of; + // PERSON_STANDS. + std::string name; + TilePos tile; +}; + +// A console failure: the checkpoint it ran, its faults and what it failed at. +struct ConsoleFailure{ + std::string name; + std::string checkpoint; + // Where the run starts if not on the checkpoint's start. + std::optional start; + std::optional start_facing; + std::vector faults; + // What the console did: the step and the start of its error. + std::string failed_step; + std::string failed_error; + // The frame the failure was checked against, a file name in the lab's + // failure screenshots. + std::string frame; + // The log lines the faults were worked out from. + std::vector evidence; +}; + +// Reads a fault file; throws on a missing or unknown field. +ConsoleFailure parse_console_failure(const std::string& name, const std::string& text); + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace.cpp b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace.cpp new file mode 100644 index 0000000000..8043c6c8f4 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace.cpp @@ -0,0 +1,267 @@ +/* Leg Trace + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "Common/Cpp/Json/JsonArray.h" +#include "Common/Cpp/Json/JsonObject.h" +#include "Common/Cpp/Json/JsonValue.h" +#include "Pokemon_LegTrace.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +std::string_view leg_end_kind_name(LegEndKind kind){ + switch (kind){ + case LegEndKind::OPEN: + return "open"; + case LegEndKind::WALL: + return "wall"; + case LegEndKind::TILE: + return "tile"; + } + return "?"; +} + +std::string_view leg_end_known_name(LegEndKnown known){ + switch (known){ + case LegEndKnown::WALL: + return "wall"; + case LegEndKnown::FADE: + return "fade"; + case LegEndKnown::DIALOGUE: + return "dialogue"; + case LegEndKnown::COUNTED: + return "counted"; + case LegEndKnown::DEAD_RECKONED: + return "dead_reckoned"; + case LegEndKnown::SEEN: + return "seen"; + case LegEndKnown::UNKNOWN: + return "unknown"; + } + return "?"; +} + +std::string_view leg_interrupt_name(LegInterrupt interrupt){ + switch (interrupt){ + case LegInterrupt::WILD_BATTLE: + return "wild_battle"; + case LegInterrupt::TRAINER: + return "trainer"; + case LegInterrupt::SCENE: + return "scene"; + case LegInterrupt::DIALOGUE: + return "dialogue"; + case LegInterrupt::BLOCKED: + return "blocked"; + } + return "?"; +} + + +namespace{ + +// [["Route 202", 23, 25], ...] +JsonArray tiles_json(const std::vector& tiles){ + JsonArray out; + for (const RouteTile& tile : tiles){ + JsonArray entry; + entry.push_back(tile.map); + entry.push_back((int64_t)tile.pos.x); + entry.push_back((int64_t)tile.pos.y); + out.push_back(std::move(entry)); + } + return out; +} + +} // namespace + +std::string leg_trace_json(const LegRecord& record){ + JsonObject out; + out["checkpoint"] = record.checkpoint; + out["event"] = record.event; + out["leg"] = (int64_t)record.leg; + out["dir"] = direction_name(record.plan.direction); + out["tiles"] = (int64_t)record.plan.tiles; + out["end_kind"] = std::string(leg_end_kind_name(record.plan.end_kind)); + out["planned_end"] = tiles_json(record.plan.ends); + if (record.moved){ + out["moved"] = (int64_t)*record.moved; + } + out["chunks"] = (int64_t)record.chunks; + out["ms"] = record.walked_ms; + out["end"] = tiles_json(record.end); + out["known"] = std::string(leg_end_known_name(record.known)); + JsonArray interrupts; + for (LegInterrupt interrupt : record.interrupts){ + interrupts.push_back(std::string(leg_interrupt_name(interrupt))); + } + out["interrupts"] = std::move(interrupts); + if (!record.note.empty()){ + out["note"] = record.note; + } + return out.dump(-1); +} + +std::string leg_trace_line(const LegRecord& record){ + return std::string(LEG_TRACE_TAG) + leg_trace_json(record); +} + + +namespace{ + +std::optional parse_int(std::string_view text){ + int value = 0; + const auto result = std::from_chars(text.data(), text.data() + text.size(), value); + if (result.ec != std::errc() || result.ptr != text.data() + text.size()){ + return std::nullopt; + } + return value; +} + +// "Route 202 (23, 25)". +std::optional parse_route_tile(std::string_view text){ + const size_t open = text.rfind(" ("); + const size_t comma = text.rfind(", "); + if (open == std::string_view::npos || comma == std::string_view::npos || comma < open || !text.ends_with(")")){ + return std::nullopt; + } + const std::optional x = parse_int(text.substr(open + 2, comma - open - 2)); + const std::optional y = parse_int(text.substr(comma + 2, text.size() - comma - 3)); + if (open == 0 || !x || !y){ + return std::nullopt; + } + return RouteTile{std::string(text.substr(0, open)), TilePos{*x, *y}}; +} + +std::optional parse_direction(std::string_view text){ + for (Direction direction : ALL_DIRECTIONS){ + if (text == direction_name(direction)){ + return direction; + } + } + return std::nullopt; +} + +} // namespace + +std::vector parse_route_tiles(std::string_view text){ + std::vector out; + while (!text.empty()){ + // Each tile ends at its ")"; ", " separates them. + const size_t close = text.find(')'); + if (close == std::string_view::npos){ + return {}; + } + const std::optional tile = parse_route_tile(text.substr(0, close + 1)); + if (!tile){ + return {}; + } + out.push_back(*tile); + text.remove_prefix(close + 1); + if (text.starts_with(", ")){ + text.remove_prefix(2); + }else if (!text.empty()){ + return {}; + } + } + return out; +} + +std::optional parse_route_leg_line(std::string_view text){ + // "Leg /: () to " + if (!text.starts_with("Leg ")){ + return std::nullopt; + } + const size_t colon = text.find(": "); + if (colon == std::string_view::npos){ + return std::nullopt; + } + text.remove_prefix(colon + 2); + const size_t space = text.find(' '); + const size_t open = text.find(" ("); + const size_t close = text.find(") to "); + if (space == std::string_view::npos || open == std::string_view::npos || close == std::string_view::npos + || !(space <= open && open < close)){ + return std::nullopt; + } + const std::optional direction = parse_direction(text.substr(0, space)); + const std::optional tiles = parse_int(text.substr(space + 1, open - space - 1)); + const std::string_view end = text.substr(open + 2, close - open - 2); + if (!direction || !tiles || *tiles < 0){ + return std::nullopt; + } + LegPlan plan{.direction = *direction, .tiles = (size_t)*tiles}; + if (end == "to wall"){ + plan.end_kind = LegEndKind::WALL; + }else if (end == "warp"){ + plan.end_kind = LegEndKind::TILE; + }else if (end == "open"){ + plan.end_kind = LegEndKind::OPEN; + }else{ + return std::nullopt; + } + plan.ends = parse_route_tiles(text.substr(close + 5)); + return plan; +} + + +void LegTracer::begin_event(const std::string& checkpoint, const std::string& event){ + end_event(); + m_checkpoint = checkpoint; + m_event = event; + m_legs = 0; +} + +void LegTracer::end_event(){ + if (m_leg){ + end_leg({}, LegEndKnown::UNKNOWN); + } +} + +void LegTracer::begin_leg(LegPlan plan){ + end_event(); + m_leg = LegRecord{.checkpoint = m_checkpoint, .event = m_event, .leg = ++m_legs, .plan = std::move(plan)}; +} + +void LegTracer::walked(const LegWalk& walk){ + if (!m_leg){ + return; + } + m_leg->chunks += walk.chunks; + m_leg->walked_ms += walk.walked_ms; + for (size_t i = 0; i < walk.wild_battles; i++){ + m_leg->interrupts.push_back(LegInterrupt::WILD_BATTLE); + } + if (walk.tapped){ + m_leg->moved = m_leg->moved.value_or(0) + *walk.tapped; + } +} + +void LegTracer::interrupted(LegInterrupt interrupt){ + if (m_leg){ + m_leg->interrupts.push_back(interrupt); + } +} + +void LegTracer::end_leg(std::vector end, LegEndKnown known, std::string note){ + if (!m_leg){ + return; + } + LegRecord record = std::move(*m_leg); + m_leg.reset(); + record.end = std::move(end); + record.known = known; + record.note = std::move(note); + m_emit(leg_trace_line(record)); +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace.h b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace.h new file mode 100644 index 0000000000..d177c6e09b --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace.h @@ -0,0 +1,167 @@ +/* Leg Trace + * + * From: https://github.com/PokemonAutomation/ + * + * One structured record per walked leg, shared by the games' walkers and + * their simulations, so a console run's route can be compared leg by leg + * with the simulation's (devtools/maps/leg_trace.py). Each leg is one log + * line beside the free-text route lines: + * + * [Leg] {"checkpoint":"00.04","event":"to_the_signs_row","leg":1, + * "dir":"north","tiles":1,"end_kind":"tile","moved":1,...} + * + * A record holds what was planned (direction, tiles, how the leg ends and + * the tiles it may end on), what was measured (taps that moved, the + * walk's moved chunks and held time), where the leg ended and how that is + * known, and what interrupted it. + * +// code-health: allow header_symbols -- leg trace diagnostic models and route execution logging types + */ + +#ifndef PokemonAutomation_Pokemon_LegTrace_H +#define PokemonAutomation_Pokemon_LegTrace_H + +#include +#include +#include +#include +#include +#include +#include +#include "Pokemon_Grid.h" +#include "Pokemon_RouteLog.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// Prefix tag of a leg record's log line. +inline constexpr std::string_view LEG_TRACE_TAG = "[Leg] "; + +// How a leg is planned to end. +enum class LegEndKind{ + OPEN, // timed or tapped into the open + WALL, // held until a wall stops it + TILE, // counted taps onto an exact tile +}; +// "open", "wall", "tile". +std::string_view leg_end_kind_name(LegEndKind kind); + +// How the end of a leg is known. +enum class LegEndKnown{ + WALL, // a wall stopped the walk + FADE, // a warp's fade or banner ended it + DIALOGUE, // a dialogue stopped it + COUNTED, // taps that moved, counted against the map + DEAD_RECKONED, // taken to have ended as planned + SEEN, // stopped short, where the walker found itself + UNKNOWN, // not known: the leg was cut off +}; +// "wall", "fade", "dialogue", "counted", "dead_reckoned", "seen", "unknown". +std::string_view leg_end_known_name(LegEndKnown known); + +// What stopped or delayed a leg on the way. +enum class LegInterrupt{ + WILD_BATTLE, + TRAINER, + SCENE, + DIALOGUE, + BLOCKED, // someone the map lacks stood in the way +}; +// "wild_battle", "trainer", "scene", "dialogue", "blocked". +std::string_view leg_interrupt_name(LegInterrupt interrupt); + +// What a leg is planned to do. +struct LegPlan{ + Direction direction = Direction::NORTH; + size_t tiles = 0; + LegEndKind end_kind = LegEndKind::OPEN; + // Every tile the leg may end on; empty where the plan does not say. + std::vector ends; +}; + +// One walked leg. +struct LegRecord{ + std::string checkpoint; + std::string event; + // 1-based, counted through the event, replans included. + size_t leg = 0; + LegPlan plan; + // The taps that moved, where the leg counts them. + std::optional moved; + // What the walks of the leg report: chunks of screen movement, and the + // time the direction was held. + size_t chunks = 0; + int64_t walked_ms = 0; + // The tiles the leg ended on, as the walker knows them. + std::vector end; + LegEndKnown known = LegEndKnown::UNKNOWN; + std::vector interrupts; + // Why the leg ended other than as planned ("replan", "warp"), or "". + std::string note; +}; + +// The record as one line of compact JSON, keys sorted. +std::string leg_trace_json(const LegRecord& record); +// LEG_TRACE_TAG and the record's JSON. +std::string leg_trace_line(const LegRecord& record); + + +// The route log's text read back: "Route 202 (23, 25), Route 202 (30, 26)". +// Nothing for text that is not a list of tiles. +std::vector parse_route_tiles(std::string_view text); +// A route_leg_line() read back: "Leg 1/2: east 15 (to wall) to ". +// A warp leg ends at its tiles, as a tile leg. +std::optional parse_route_leg_line(std::string_view text); + + +// What one walk of a leg reports. +struct LegWalk{ + // Chunks of screen movement, and the time the direction was held. + size_t chunks = 0; + int64_t walked_ms = 0; + size_t wild_battles = 0; + // The taps that moved, for a walk of taps. + std::optional tapped; +}; + +// Builds a walker's leg records from what it does, and hands each, as its +// log line, to `emit` once the leg ends. A leg still open when the next +// starts, or when its event ends, is emitted with its end unknown. +class LegTracer{ +public: + explicit LegTracer(std::function emit) + : m_emit(std::move(emit)) + {} + + // The legs from here on are `event`'s of `checkpoint`. + void begin_event(const std::string& checkpoint, const std::string& event); + // Emits the leg still open, if one is. + void end_event(); + + // A leg starts. + void begin_leg(LegPlan plan); + // The open leg's plan; null if no leg is open. + const LegPlan* open_leg() const{ + return m_leg ? &m_leg->plan : nullptr; + } + // A walk of the open leg ended. A leg can take several (a walk on to + // the wall after a battle); they add up. + void walked(const LegWalk& walk); + void interrupted(LegInterrupt interrupt); + // The open leg ended on `end`, known by `known`; emits it. + void end_leg(std::vector end, LegEndKnown known, std::string note = ""); + +private: + std::function m_emit; + std::string m_checkpoint; + std::string m_event; + size_t m_legs = 0; + std::optional m_leg; +}; + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace_Tests.cpp b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace_Tests.cpp new file mode 100644 index 0000000000..dc2c889309 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace_Tests.cpp @@ -0,0 +1,167 @@ +/* Leg Trace Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The leg record's line word for word, the route log's leg lines read + * back, and the tracer's records for a leg that ends, one cut off by the + * next, and one with a wild battle on the way. + * + */ + +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "Pokemon_LegTrace.h" +#include "Pokemon_LegTrace_Tests.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + +struct LegTraceInput{ + std::string name; + std::function actual; +}; +struct LegTraceExpectation{ + std::string expected; +}; +struct LegTraceTestCase{ + LegTraceInput input; + LegTraceExpectation expectation; +}; + +class Test_LegTrace : public UnitTest{ +public: + explicit Test_LegTrace(LegTraceTestCase scenario) + : UnitTest("Pokemon::LegTrace - " + scenario.input.name) + , m_scenario(std::move(scenario)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::string actual = m_scenario.input.actual(); + if (actual == m_scenario.expectation.expected){ + return true; + } + return "expected:\n" + m_scenario.expectation.expected + "\ngot:\n" + actual; + } + +private: + LegTraceTestCase m_scenario; +}; + +std::string tiles_text(const std::vector& tiles){ + return tiles.empty() ? "none" : describe(tiles); +} + +std::string plan_text(const std::optional& plan){ + if (!plan){ + return "none"; + } + return std::string(direction_name(plan->direction)) + " " + std::to_string(plan->tiles) + " " + + std::string(leg_end_kind_name(plan->end_kind)) + " -> " + tiles_text(plan->ends); +} + +// The lines a tracer emits for `script`, one a line. +std::string traced(const std::function& script){ + std::string out; + LegTracer tracer([&](const std::string& line){ out += line + "\n"; }); + script(tracer); + return out; +} + +const LegTraceTestCase CASES[] = { + { + .input = {.name = "a tap leg's line", .actual = []{ + LegRecord record{ + .checkpoint = "00.04", .event = "to_the_signs_row", .leg = 1, + .plan = {.direction = Direction::NORTH, .tiles = 1, .end_kind = LegEndKind::TILE, + .ends = {{"Route 201", {57, 4}}}}, + .moved = 1, .chunks = 1, .walked_ms = 800, + .end = {{"Route 201", {57, 4}}}, .known = LegEndKnown::COUNTED, + }; + return leg_trace_line(record); + }}, + .expectation = {.expected = + "[Leg] {\"checkpoint\":\"00.04\",\"chunks\":1,\"dir\":\"north\",\"end\":[[\"Route 201\",57,4]]," + "\"end_kind\":\"tile\",\"event\":\"to_the_signs_row\",\"interrupts\":[],\"known\":\"counted\"," + "\"leg\":1,\"moved\":1,\"ms\":800,\"planned_end\":[[\"Route 201\",57,4]],\"tiles\":1}"}, + }, + { + .input = {.name = "tiles read back", .actual = []{ + return tiles_text(parse_route_tiles("Route 202 (23, 25), Mom's House (30, -1)")); + }}, + .expectation = {.expected = "Route 202 (23, 25), Mom's House (30, -1)"}, + }, + { + .input = {.name = "no tiles in other text", .actual = []{ + return tiles_text(parse_route_tiles("an unknown tile")); + }}, + .expectation = {.expected = "none"}, + }, + { + .input = {.name = "a wall leg line read back", .actual = []{ + return plan_text(parse_route_leg_line("Leg 1/2: east 15 (to wall) to Route 202 (23, 25), Route 202 (30, 26)")); + }}, + .expectation = {.expected = "east 15 wall -> Route 202 (23, 25), Route 202 (30, 26)"}, + }, + { + .input = {.name = "an open leg line read back", .actual = []{ + return plan_text(parse_route_leg_line("Leg 3/3: west 1 (open) to Twinleaf Town (7, 9)")); + }}, + .expectation = {.expected = "west 1 open -> Twinleaf Town (7, 9)"}, + }, + { + .input = {.name = "not a leg line", .actual = []{ + return plan_text(parse_route_leg_line("[Route] Turning west.")); + }}, + .expectation = {.expected = "none"}, + }, + { + .input = {.name = "a leg cut off by the next, and by its event's end", .actual = []{ + return traced([](LegTracer& tracer){ + tracer.begin_event("00.04", "beside_mom"); + tracer.begin_leg({.direction = Direction::WEST, .tiles = 2, .end_kind = LegEndKind::OPEN}); + tracer.begin_leg({.direction = Direction::NORTH, .tiles = 1, .end_kind = LegEndKind::WALL}); + tracer.walked({.chunks = 3, .walked_ms = 640}); + tracer.begin_event("00.04", "mom"); + }); + }}, + .expectation = {.expected = + "[Leg] {\"checkpoint\":\"00.04\",\"chunks\":0,\"dir\":\"west\",\"end\":[],\"end_kind\":\"open\"," + "\"event\":\"beside_mom\",\"interrupts\":[],\"known\":\"unknown\",\"leg\":1,\"ms\":0,\"planned_end\":[],\"tiles\":2}\n" + "[Leg] {\"checkpoint\":\"00.04\",\"chunks\":3,\"dir\":\"north\",\"end\":[],\"end_kind\":\"wall\"," + "\"event\":\"beside_mom\",\"interrupts\":[],\"known\":\"unknown\",\"leg\":2,\"ms\":640,\"planned_end\":[],\"tiles\":1}\n"}, + }, + { + .input = {.name = "a wild battle on the way, two walks to the wall", .actual = []{ + return traced([](LegTracer& tracer){ + tracer.begin_event("00.04b", "to_catch_grass"); + tracer.begin_leg({.direction = Direction::SOUTH, .tiles = 4, .end_kind = LegEndKind::WALL}); + tracer.walked({.chunks = 2, .walked_ms = 500, .wild_battles = 1}); + tracer.walked({.chunks = 2, .walked_ms = 400}); + tracer.end_leg({{"Route 202", {20, 27}}}, LegEndKnown::WALL); + tracer.end_event(); + }); + }}, + .expectation = {.expected = + "[Leg] {\"checkpoint\":\"00.04b\",\"chunks\":4,\"dir\":\"south\",\"end\":[[\"Route 202\",20,27]],\"end_kind\":\"wall\"," + "\"event\":\"to_catch_grass\",\"interrupts\":[\"wild_battle\"],\"known\":\"wall\",\"leg\":1,\"ms\":900,\"planned_end\":[],\"tiles\":4}\n"}, + }, +}; + +} // namespace + + +void add_tests_LegTrace(UnitTestDatabase& database){ + for (const auto& scenario : CASES){ + database.add(scenario); + } +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace_Tests.h b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace_Tests.h new file mode 100644 index 0000000000..a81561fa7f --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace_Tests.h @@ -0,0 +1,23 @@ +/* Leg Trace Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_Pokemon_LegTrace_Tests_H +#define PokemonAutomation_Pokemon_LegTrace_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// Registers leg trace record and tracer unit tests. +void add_tests_LegTrace(UnitTestDatabase& database); + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_RouteLog_Tests.cpp b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_RouteLog_Tests.cpp index 843fc32db0..2039c5c62e 100644 --- a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_RouteLog_Tests.cpp +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_RouteLog_Tests.cpp @@ -13,6 +13,7 @@ #include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/TestRunners/UnitTest.h" +#include "Pokemon_LegTrace_Tests.h" #include "Pokemon_RouteLog.h" #include "Pokemon_RouteLog_Tests.h" @@ -198,6 +199,8 @@ void add_tests_RouteLog(UnitTestDatabase& database){ for (const auto& scenario : CASES){ database.add(scenario); } + // The leg records read the route lines back. + add_tests_LegTrace(database); } diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index bc315ba159..6355f135b9 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1487,6 +1487,12 @@ file(GLOB LIBRARY_SOURCES Source/Pokemon/Navigation/Pokemon_LegWalker.h Source/Pokemon/Navigation/Pokemon_LegWalker_Tests.cpp Source/Pokemon/Navigation/Pokemon_LegWalker_Tests.h + Source/Pokemon/Navigation/Pokemon_LegReplay.cpp + Source/Pokemon/Navigation/Pokemon_LegReplay.h + Source/Pokemon/Navigation/Pokemon_LegTrace.cpp + Source/Pokemon/Navigation/Pokemon_LegTrace.h + Source/Pokemon/Navigation/Pokemon_LegTrace_Tests.cpp + Source/Pokemon/Navigation/Pokemon_LegTrace_Tests.h Source/Pokemon/Navigation/Pokemon_MapData.cpp Source/Pokemon/Navigation/Pokemon_MapData.h Source/Pokemon/Navigation/Pokemon_MapData_Tests.cpp diff --git a/devtools/maps/leg_replay.py b/devtools/maps/leg_replay.py new file mode 100644 index 0000000000..61e25fa238 --- /dev/null +++ b/devtools/maps/leg_replay.py @@ -0,0 +1,111 @@ +"""A console failure as a fault file for the simulation (Pokemon_LegReplay.h). + + python3 -m devtools.maps.leg_replay LOG --failed-at 'HH:MM:SS' [--date YYYY-MM-DD] + [--fault JSON]... [--pin 'step: Map x,y'] [--frames DIR] [--build SHA] -o FILE + +Reads the failure the program logged at that time ("[Event] 01.02/11 +into_oreburgh: failed: ..."), the [Leg] records and route lines of the run up +to it, and the failure frame nearest in time, and writes the fault file: the +checkpoint, where it failed, the frame, and the log lines the faults were +worked out from. The faults are the ones given with --fault (the file's own +JSON, see Pokemon_LegReplay.h); the tool derives none the log cannot show by +itself, so it stops (exit 2) and prints the facts it found when none are +given, asking for one pinned position (--pin) taken by hand from the frame. +""" + +import argparse +import glob +import json +import os +import re +import sys +from datetime import datetime + +STAMP = re.compile(r'^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d)\.\d+ - ') +FAILED = re.compile(r'\[Event\] (\d\d\.\d\d[a-z]?)/(\d+) (\w+): failed: (.*)$') +FACTS = ('[Leg] ', 'someone the map lacks', 'a trainer the map lacks', 'find us again', + 'no fade and no', 'blocked after', 'talked to us') + + +def failure_at(lines, stamp): + """(index, checkpoint, step, error) of the failure logged at `stamp`.""" + for i, line in enumerate(lines): + m = FAILED.search(line) + if m and line.startswith(stamp): + return i, m.group(1), m.group(3), m.group(4).strip() + return None + + +def run_facts(lines, end, checkpoint): + """The facts of the checkpoint's run that ends at line `end`.""" + start = end + while start > 0 and f'[Checkpoint] {checkpoint}: start' not in lines[start]: + start -= 1 + out = [] + for line in lines[start:end + 1]: + if any(tag in line for tag in FACTS) and 'Watcher' not in line: + out.append(STAMP.sub('', line).replace('[Console 0]: ', '').strip()) + return out + + +def nearest_frame(frames, stamp): + """The failure frame closest in time to `stamp` (file names: YYYYMMDD-HHMMSS...).""" + want = datetime.strptime(stamp, '%Y-%m-%d %H:%M:%S') + best = None + for path in glob.glob(os.path.join(frames, '*.*')): + m = re.match(r'(\d{8}-\d{6})', os.path.basename(path)) + if not m: + continue + gap = abs((datetime.strptime(m.group(1), '%Y%m%d-%H%M%S') - want).total_seconds()) + if best is None or gap < best[0]: + best = (gap, os.path.basename(path)) + return best[1] if best and best[0] <= 120 else '' + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__.split('\n')[0]) + p.add_argument('log') + p.add_argument('--failed-at', required=True) + p.add_argument('--date', default=datetime.now().strftime('%Y-%m-%d')) + p.add_argument('--fault', action='append', default=[]) + p.add_argument('--pin', help="'step: Map x,y' read from the frame") + p.add_argument('--frames') + p.add_argument('--build', default='') + p.add_argument('-o', '--out') + a = p.parse_args(argv) + stamp = f'{a.date} {a.failed_at}' + with open(a.log, errors='replace') as f: + lines = [l.rstrip('\n') for l in f] + found = failure_at(lines, stamp) + if not found: + print(f'no failure logged at {stamp}', file=sys.stderr) + return 1 + index, checkpoint, step, error = found + facts = run_facts(lines, index, checkpoint) + if not a.fault and not a.pin: + print(f'{checkpoint} failed at {step}: {error}\nthe log shows, not enough to place the player:', file=sys.stderr) + for fact in facts[-30:]: + print(' ' + fact[:200], file=sys.stderr) + print("pin one position from the frame (--pin 'step: Map x,y') or give the faults (--fault).", file=sys.stderr) + return 2 + out = { + 'about': f'lab-B {stamp}, {checkpoint} {step}' + (f', build {a.build}' if a.build else ''), + 'checkpoint': checkpoint, + 'faults': [json.loads(x) for x in a.fault], + 'failed': {'step': step, 'error': error}, + 'frame': nearest_frame(a.frames, stamp) if a.frames else '', + 'evidence': facts[-12:], + } + if a.pin: + out['pin'] = a.pin + text = json.dumps(out, indent=2) + '\n' + if a.out: + with open(a.out, 'w') as f: + f.write(text) + else: + sys.stdout.write(text) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/devtools/maps/leg_trace.py b/devtools/maps/leg_trace.py new file mode 100644 index 0000000000..2f09325e75 --- /dev/null +++ b/devtools/maps/leg_trace.py @@ -0,0 +1,463 @@ +"""Leg traces: a console run's route read back from its log, and graded +against the simulation's route for the same checkpoints. + +A walker writes one record per leg (SerialPrograms Pokemon_LegTrace.h): + + [Leg] {"checkpoint":"00.04","event":"to_the_signs_row","leg":1,"dir":"north", + "tiles":1,"end_kind":"tile","planned_end":[...],"moved":1,"chunks":1, + "ms":800,"end":[["Route 201",57,4]],"known":"counted","interrupts":[]} + +A trace is {"legs": [record, ...]}; the simulation's are checked in as +golden files (Simulation/leg_trace_.golden.json). Logs written before +the records existed are read from their route lines instead (replay_text()): +plan legs, their "ended at" lines, the story's tap and dialogue runs. + +The diff aligns the two traces by checkpoint, event and leg, grades each +leg, and fails only on LARGE: + + OK the same plan, and the end the same, or for an open leg within + OPEN_BAND_TILES of the simulation's + drift outside that band, but a later leg of the event that re-localizes + (a wall, a count, a fade) ends where the simulation's does; or + a leg of a replan after a leg stopped short, until the route + re-syncs + LARGE outside the band and never re-synced, a different map, legs or + events one side lacks, a recovery loop, or a checkpoint whose + walking time is over TIME_RATIO_LARGE times the simulation's + masked a leg with an interruption (wild battle, trainer, scene, + someone in the way), and + the console's legs after it until one ends where a simulation + leg does + + python3 -m devtools.maps.leg_trace replay run.log -o console.json + python3 -m devtools.maps.leg_trace diff run.log Simulation/leg_trace_00.04.golden.json +""" +import argparse +import json +import re +import sys +from dataclasses import dataclass, field + +# Thresholds, in one place. +# An open leg (timed or tapped into the open) ending this many tiles or +# fewer from the simulation's end is OK: a later wall clip absorbs it. +OPEN_BAND_TILES = 2 +# A checkpoint's walking time over this times the simulation's is LARGE... +TIME_RATIO_LARGE = 1.5 +# ...when it is also at least this much longer (short checkpoints jitter). +TIME_MIN_EXTRA_MS = 3000 +# An event with this many more legs than the simulation's is a recovery loop. +LOOP_EXTRA_LEGS = 6 +# How a leg's end is known when it re-localizes the walker. +RELOCALIZING = {'wall', 'counted', 'fade', 'seen', 'dialogue'} + +OK = 'OK' +DRIFT = 'drift' +LARGE = 'LARGE' +MASKED = 'masked' + +LEG_TAG = '[Leg] ' +DIRECTION = r'(north|south|east|west)' +TILE = re.compile(r'([^,()]+?) \((-?\d+), (-?\d+)\)') +CHECKPOINT = re.compile(r'\[Checkpoint\] (\S+): start') +EVENT = re.compile(r'\[Event\] (\S+)/\d+ (\S+): start') +PLAN_LEG = re.compile(r'(?:\[Route\] )?Leg \d+/\d+: ' + DIRECTION + r' (\d+) \((to wall|open|warp)\) to (.*?)\s*$') +LEG_END = re.compile(r'\[Route\] .*: ended at (.*) \((walked|assumed)[^)]*\)\.\s*$') +REPLAN = re.compile(r'\[Route\] .*, at (.*); planning again\.\s*$') +TAPS = re.compile(r'\[(\S+)\] (\S+): taps ' + DIRECTION + r' moved (\d+) of (\d+)(?: tiles\.|: )') +TO_DIALOGUE = re.compile(r'\[(\S+)\] (\S+): ran ' + DIRECTION + + r' to a dialogue: (\d+) moved chunks, (\d+) tiles at most, ([^,]+), (\d+) wild battle') +WILD = re.compile(r'\[Walk\] .*wild battle[^;]*; running\.') +TRAINER = re.compile(r'\[Route\] .* stopped us about ') +BLOCKED = re.compile(r'; waiting for the way to clear\.|: someone is in the way; talking to them\.') +WALKED = re.compile(r'\[Walk\] .*: \w+ after (\d+)ms, (\d+) moved chunks, (\d+) wild battle') + + +def tiles_of(text): + return [[m.group(1).strip(), int(m.group(2)), int(m.group(3))] for m in TILE.finditer(text)] + + +def replay(lines, attempt=None): + """The trace of a log: its [Leg] records, or if it has none, what its + route lines say (replay_text()). A log can hold several runs of a + checkpoint: each record gets its run's "attempt" (1, 2, ... a + checkpoint), and only `attempt` is kept (-1: each checkpoint's last; + None: all).""" + legs = [] + attempts = {} + for line in lines: + m = CHECKPOINT.search(line) + if m: + attempts[m.group(1)] = attempts.get(m.group(1), 0) + 1 + at = line.find(LEG_TAG) + if at >= 0: + record = json.loads(line[at + len(LEG_TAG):]) + record.setdefault('attempt', attempts.get(record['checkpoint'], 1)) + legs.append(record) + trace = {'legs': legs} if legs else replay_text(lines) + return pick_attempt(trace, attempt) + + +def pick_attempt(trace, attempt): + """`trace` with only run `attempt` of each checkpoint (-1: the last).""" + if attempt is None: + return trace + last = {} + for leg in trace['legs']: + last[leg['checkpoint']] = max(last.get(leg['checkpoint'], 0), leg.get('attempt', 1)) + def wanted(checkpoint, run): + return run == (last.get(checkpoint, 0) if attempt == -1 else attempt) + out = {'legs': [leg for leg in trace['legs'] if wanted(leg['checkpoint'], leg.get('attempt', 1))]} + if 'events' in trace: + out['events'] = [e for e in trace['events'] if wanted(e[0], e[2])] + return out + + +def replay_text(lines): + """A trace from a log's free-text route lines: no held time, and tap + legs without their end tiles (the log does not name them).""" + legs = [] + state = {'checkpoint': '', 'event': '', 'count': 0, 'open': None} + attempts = {} + events = [] + + def start(record): + close(None, 'unknown') + state['count'] += 1 + record.update({'checkpoint': state['checkpoint'], 'event': state['event'], 'leg': state['count'], + 'attempt': attempts.get(state['checkpoint'], 1)}) + record.setdefault('chunks', 0) + record.setdefault('ms', 0) + record.setdefault('interrupts', []) + record.setdefault('planned_end', []) + state['open'] = record + + def close(end, known, note=''): + record = state['open'] + if record is None: + return + record['end'] = end or [] + record['known'] = known + if note: + record['note'] = note + legs.append(record) + state['open'] = None + + for line in lines: + m = CHECKPOINT.search(line) + if m: + close(None, 'unknown') + attempts[m.group(1)] = attempts.get(m.group(1), 0) + 1 + continue + m = EVENT.search(line) + if m: + close(None, 'unknown') + state.update({'checkpoint': m.group(1), 'event': m.group(2), 'count': 0}) + events.append([m.group(1), m.group(2), attempts.get(m.group(1), 1)]) + continue + m = PLAN_LEG.search(line) + if m: + kind = {'to wall': 'wall', 'open': 'open', 'warp': 'tile'}[m.group(3)] + start({'dir': m.group(1), 'tiles': int(m.group(2)), 'end_kind': kind, 'planned_end': tiles_of(m.group(4))}) + continue + m = LEG_END.search(line) + if m and state['open'] is not None: + assumed = m.group(2) == 'assumed' + if assumed and 'wild_battle' not in state['open']['interrupts']: + state['open']['interrupts'].append('wild_battle') + wall = state['open']['end_kind'] == 'wall' and not assumed + close(tiles_of(m.group(1)), 'wall' if wall else 'dead_reckoned') + continue + m = REPLAN.search(line) + if m and state['open'] is not None: + close(tiles_of(m.group(1)), 'seen', 'replan') + continue + m = TAPS.search(line) + if m: + start({'dir': m.group(3), 'tiles': int(m.group(5)), 'end_kind': 'tile', 'moved': int(m.group(4)), + 'chunks': int(m.group(4))}) + close(None, 'counted') + continue + m = TO_DIALOGUE.search(line) + if m: + start({'dir': m.group(3), 'tiles': int(m.group(5)), 'end_kind': 'open', 'chunks': int(m.group(4))}) + if int(m.group(7)) > 0: + state['open']['interrupts'].append('wild_battle') + close(None, 'dialogue' if 'dialog' in m.group(6) else 'unknown') + continue + if state['open'] is not None and WILD.search(line): + state['open']['interrupts'].append('wild_battle') + elif state['open'] is not None and TRAINER.search(line): + state['open']['interrupts'].append('trainer') + elif state['open'] is not None and BLOCKED.search(line): + state['open']['interrupts'].append('blocked') + elif state['open'] is not None and WALKED.search(line): + m = WALKED.search(line) + record = state['open'] + record['ms'] += int(m.group(1)) + record['chunks'] += int(m.group(2)) + if record['end_kind'] == 'open': + record['moved'] = record.get('moved', 0) + int(m.group(2)) + if int(m.group(3)) > 0 and 'wild_battle' not in record['interrupts']: + record['interrupts'].append('wild_battle') + close(None, 'unknown') + # The events the log ran: one with no leg lines walked legs the text + # does not describe (a LEGS step to the walls). + return {'legs': legs, 'events': events} + + +@dataclass +class LegGrade: + checkpoint: str + event: str + console: int # the console's leg number, 0 for none + sim: int # the simulation's leg number, 0 for none + grade: str + why: str = '' + + +@dataclass +class Report: + grades: list = field(default_factory=list) + # Checkpoint-level findings: (checkpoint, grade, why). + findings: list = field(default_factory=list) + + def large(self): + return any(g.grade == LARGE for g in self.grades) or any(f[1] == LARGE for f in self.findings) + + def first_divergence(self): + for g in self.grades: + if g.grade in (DRIFT, LARGE): + return g + return None + + def summary(self): + out = [] + checkpoints = [] + for g in self.grades: + if g.checkpoint not in checkpoints: + checkpoints.append(g.checkpoint) + for checkpoint in checkpoints: + mine = [g for g in self.grades if g.checkpoint == checkpoint] + counts = {k: sum(1 for g in mine if g.grade == k) for k in (OK, DRIFT, LARGE, MASKED)} + extra = [f for f in self.findings if f[0] == checkpoint] + large = counts[LARGE] + sum(1 for f in extra if f[1] == LARGE) + line = '{}: {} legs: {} OK, {} drift, {} LARGE, {} masked'.format( + checkpoint, len(mine), counts[OK], counts[DRIFT], large, counts[MASKED]) + first = next((g for g in mine if g.grade in (DRIFT, LARGE)), None) + if first: + line += '; first divergence: {} leg {} ({}): {}'.format(first.event, first.console or '-', first.grade, first.why) + for f in extra: + line += '; {}: {}'.format(f[1], f[2]) + out.append(line) + return out + + +def _key(tile): + return (tile[0], int(tile[1]), int(tile[2])) + + +def _maps(leg): + return {t[0] for t in leg.get('end', [])} + + +def _distance(a, b): + """The fewest tiles between a tile of `a` and one of `b` on the same map; + None if they share no map.""" + best = None + for x in a: + for y in b: + if x[0] == y[0]: + d = abs(int(x[1]) - int(y[1])) + abs(int(x[2]) - int(y[2])) + best = d if best is None else min(best, d) + return best + + +def _same_plan(c, s): + return c['dir'] == s['dir'] and c['tiles'] == s['tiles'] and c['end_kind'] == s['end_kind'] + + +def _same_end(c, s): + return {_key(t) for t in c.get('end', [])} == {_key(t) for t in s.get('end', [])} + + +def _relocalizes(c, s): + """Whether console leg `c` ends, re-localized, where sim leg `s` does.""" + return c.get('known') in RELOCALIZING and c.get('end') and _same_end(c, s) + + +def grade_leg(c, s): + """(grade, why) of console leg `c` against sim leg `s`, of the same plan; + DRIFT here means "outside the band": the caller decides whether a later + leg re-syncs it.""" + if c.get('end') and s.get('end') and _maps(c) != _maps(s): + return LARGE, 'ended on {} where the simulation ended on {}'.format(sorted(_maps(c)), sorted(_maps(s))) + if c['end_kind'] == 'tile' and 'moved' in c and 'moved' in s and c['moved'] != s['moved']: + return DRIFT, 'moved {} of {} tiles, the simulation {}'.format(c['moved'], c['tiles'], s['moved']) + # An open leg's end is dead-reckoned from its plan: the taps that moved + # are what it measured. + if c['end_kind'] == 'open' and 'moved' in c and 'moved' in s and abs(c['moved'] - s['moved']) > OPEN_BAND_TILES: + return DRIFT, 'open leg moved {} of {} tiles, the simulation {}'.format(c['moved'], c['tiles'], s['moved']) + if not c.get('end') or not s.get('end') or _same_end(c, s): + return OK, '' + distance = _distance(c['end'], s['end']) + if c['end_kind'] == 'open' and distance is not None and distance <= OPEN_BAND_TILES: + return OK, 'open leg {} tile(s) off'.format(distance) + return DRIFT, 'ended {} tile(s) from the simulation: {} vs {}'.format(distance, c['end'], s['end']) + + +def _resync(console, i, sim, j): + """After an interruption: the first console leg from `i` that ends where + a sim leg from `j` does, as (console index, sim index), or None. The + legs go on from the ones after both.""" + for k in range(i, len(console)): + for m in range(j, len(sim)): + if console[k].get('end') and _same_end(console[k], sim[m]) and not console[k].get('interrupts'): + return k, m + return None + + +def _grade_event(checkpoint, event, console, sim, grades): + pending = [] # grades outside the band, waiting for a re-sync + i = j = 0 + + def add(c, s, grade, why=''): + g = LegGrade(checkpoint, event, c['leg'] if c else 0, s['leg'] if s else 0, grade, why) + grades.append(g) + return g + + while i < len(console) and j < len(sim): + c, s = console[i], sim[j] + if c.get('interrupts') or s.get('interrupts'): + add(c, s, MASKED, 'interrupted: ' + ', '.join(c.get('interrupts', []) + s.get('interrupts', []))) + found = _resync(console, i + 1, sim, j) + if found is None: + for rest in console[i + 1:]: + add(rest, None, MASKED, 'after an interruption, never re-synced') + return + k, m = found + for rest in console[i + 1:k]: + add(rest, None, MASKED, 'after an interruption') + add(console[k], sim[m], MASKED, 're-synced: ends where the simulation leg {} does'.format(sim[m]['leg'])) + i, j = k + 1, m + 1 + continue + if not _same_plan(c, s): + add(c, s, LARGE, 'planned {} {} ({}), the simulation {} {} ({})'.format( + c['dir'], c['tiles'], c['end_kind'], s['dir'], s['tiles'], s['end_kind'])) + i += 1 + j += 1 + continue + if c.get('note') == 'replan': + # Stopped short and planned again: the legs until the route + # re-syncs are drift, or LARGE if it never does. + grade, why = grade_leg(c, s) + add(c, s, grade if grade != DRIFT else LARGE, ('stopped short and planned again; ' + why).rstrip('; ')) + found = _resync(console, i + 1, sim, j + 1) + if found is None: + for rest in console[i + 1:]: + add(rest, None, LARGE, 'after a replan, never re-synced') + return + k, m = found + for rest in console[i + 1:k]: + add(rest, None, DRIFT, 'a replanned leg; re-synced at leg {}'.format(console[k]['leg'])) + add(console[k], sim[m], OK, 're-synced: ends where the simulation leg {} does'.format(sim[m]['leg'])) + i, j = k + 1, m + 1 + continue + grade, why = grade_leg(c, s) + if grade == OK and _relocalizes(c, s): + for g in pending: + g.grade, g.why = DRIFT, g.why + '; re-synced at leg {}'.format(c['leg']) + pending = [] + g = add(c, s, grade, why) + if grade == DRIFT: + pending.append(g) + i += 1 + j += 1 + for g in pending: + g.grade, g.why = LARGE, g.why + '; never re-synced' + for c in console[i:]: + add(c, None, LARGE, 'a leg the simulation does not walk') + for s in sim[j:]: + add(None, s, LARGE, 'a simulation leg the console did not walk') + + +def _by_event(legs): + out = {} + for leg in legs: + out.setdefault((leg['checkpoint'], leg['event']), []).append(leg) + return out + + +def diff(console, sim, checkpoints=None): + """Grades `console`'s legs (a trace) against `sim`'s, for `checkpoints` + (default: those the simulation trace has).""" + report = Report() + sim_legs = [leg for leg in sim['legs'] if checkpoints is None or leg['checkpoint'] in checkpoints] + wanted = {leg['checkpoint'] for leg in sim_legs} + console_legs = [leg for leg in console['legs'] if leg['checkpoint'] in wanted] + c_events = _by_event(console_legs) + s_events = _by_event(sim_legs) + unrecorded = {(e[0], e[1]) for e in console.get('events', [])} - set(c_events) + for key, s in s_events.items(): + c = c_events.get(key, []) + if key in unrecorded: + for leg in s: + report.grades.append(LegGrade(key[0], key[1], 0, leg['leg'], MASKED, 'the log ran the event but has no lines for its legs')) + continue + if len(c) > len(s) + LOOP_EXTRA_LEGS: + report.findings.append((key[0], LARGE, '{} walked {} legs, the simulation {}: a recovery loop'.format( + key[1], len(c), len(s)))) + _grade_event(key[0], key[1], c, s, report.grades) + for key, c in c_events.items(): + if key not in s_events: + report.findings.append((key[0], LARGE, 'event {} walks {} legs the simulation does not'.format(key[1], len(c)))) + for checkpoint in sorted(wanted): + c_ms = sum(leg.get('ms', 0) for leg in console_legs if leg['checkpoint'] == checkpoint and not leg.get('interrupts')) + s_ms = sum(leg.get('ms', 0) for leg in sim_legs if leg['checkpoint'] == checkpoint) + if s_ms > 0 and c_ms > TIME_RATIO_LARGE * s_ms and c_ms - s_ms >= TIME_MIN_EXTRA_MS: + report.findings.append((checkpoint, LARGE, 'walked {} ms, the simulation {} ms'.format(c_ms, s_ms))) + return report + + +def _load(path, attempt=None): + with open(path, encoding='utf-8', errors='replace') as f: + text = f.read() + if text.lstrip().startswith('{'): + return pick_attempt(json.loads(text), attempt) + return replay(text.splitlines(), attempt) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.split('\n')[0]) + sub = parser.add_subparsers(dest='command', required=True) + r = sub.add_parser('replay', help='a log as a trace') + r.add_argument('log') + r.add_argument('-o', '--out') + r.add_argument('--attempt', type=int, help='only this run of each checkpoint (-1: the last)') + d = sub.add_parser('diff', help='grade a console log or trace against a simulation trace') + d.add_argument('console') + d.add_argument('sim') + d.add_argument('--checkpoint', action='append') + d.add_argument('--attempt', type=int, default=-1, help='the console run of each checkpoint to grade (default -1: the last)') + d.add_argument('-v', '--verbose', action='store_true', help='every leg, not only those off') + args = parser.parse_args(argv) + if args.command == 'replay': + text = json.dumps(_load(args.log, args.attempt), indent=1) + if args.out: + with open(args.out, 'w', encoding='utf-8') as f: + f.write(text + '\n') + else: + print(text) + return 0 + report = diff(_load(args.console, args.attempt), _load(args.sim), args.checkpoint) + for g in report.grades: + if args.verbose or g.grade != OK: + print('{} {} console leg {} / sim leg {}: {} {}'.format( + g.checkpoint, g.event, g.console or '-', g.sim or '-', g.grade, g.why).rstrip()) + for line in report.summary(): + print(line) + return 1 if report.large() else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/devtools/maps/tests/test_leg_trace.py b/devtools/maps/tests/test_leg_trace.py new file mode 100644 index 0000000000..5f4af7f304 --- /dev/null +++ b/devtools/maps/tests/test_leg_trace.py @@ -0,0 +1,228 @@ +import contextlib +import io +import json +import os +import tempfile +import unittest + +from devtools.maps import leg_trace +from devtools.maps.leg_trace import DRIFT, LARGE, MASKED, OK + +GOLDEN = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'SerialPrograms', 'Source', 'PokemonBDSP', + 'Programs', 'AutoStory', 'Simulation', 'leg_trace_00.04.golden.json') + + +def leg(event, number, direction, tiles, kind, end, known=None, ms=1000, interrupts=(), checkpoint='00.04', **extra): + record = {'checkpoint': checkpoint, 'event': event, 'leg': number, 'dir': direction, 'tiles': tiles, + 'end_kind': kind, 'planned_end': end, 'chunks': tiles, 'ms': ms, 'end': end, + 'known': known or ('wall' if kind == 'wall' else 'dead_reckoned'), 'interrupts': list(interrupts)} + record.update(extra) + return record + + +def at(x, y, map_name='Route 201'): + return [[map_name, x, y]] + + +# The simulation's walk of one event: west to a wall, a tile south, west in +# the open, north to a wall. +SIM = {'legs': [ + leg('to_ledge_column', 1, 'west', 7, 'wall', at(56, 4)), + leg('to_ledge_column', 2, 'south', 1, 'open', at(56, 5)), + leg('to_ledge_column', 3, 'west', 5, 'open', at(51, 5)), + leg('to_ledge_column', 4, 'north', 3, 'wall', at(51, 2)), +]} + + +def console(*changes): + """SIM's legs with `changes` (index, {field: value}) applied.""" + legs = json.loads(json.dumps(SIM['legs'])) + for index, change in changes: + legs[index].update(change) + return {'legs': legs} + + +def grades(report): + return [(g.console, g.sim, g.grade) for g in report.grades] + + +class ReplayTest(unittest.TestCase): + def test_records_are_read_back(self): + lines = [ + '2026-11-09 18:41:00.1 - [Console 0]: [Route] Turning north.', + '2026-11-09 18:41:00.2 - [Console 0]: ' + leg_trace.LEG_TAG + json.dumps(SIM['legs'][0]), + '2026-11-09 18:41:00.3 - [Console 0]: ' + leg_trace.LEG_TAG + json.dumps(SIM['legs'][1]), + ] + self.assertEqual(leg_trace.replay(lines), {'legs': [dict(x, attempt=1) for x in SIM['legs'][:2]]}) + + def test_route_lines_without_records(self): + lines = ''' +[Program]: [Event] 00.04/4 to_the_signs_row: start +[Console 0]: [Route] Turning north. +[Console 0]: [00.04] to_the_signs_row: taps north moved 1 of 1 tiles. +[Program]: [Event] 00.04/5 to_ledge_column: start +[Console 0]: [Route] to ledge column: plan from Route 201 (57, 4), Route 201 (58, 4): west 7 (to wall) to Route 201 (56, 4); south 1 (open) to Route 201 (56, 5); west 2 (to wall) to Route 201 (54, 5) +[Console 0]: [Route] Leg 1/3: west 7 (to wall) to Route 201 (56, 4) +[Console 0]: [Walk] to ledge column: west 7 (to wall): wild battle; running. +[Console 0]: [Route] to ledge column: west 7 (to wall): ended at Route 201 (56, 4) (walked). +[Console 0]: [Route] Leg 2/3: south 1 (open) to Route 201 (56, 5) +[Console 0]: [Walk] to ledge column: south 1 (open): blocked after 800ms, 0 moved chunks, 0 wild battle(s). +[Console 0]: [Route] to ledge column: south 1 (open): blocked after 0 of 1 tiles; waiting for the way to clear. +[Console 0]: [Route] to ledge column: south 1 (open): blocked after 0 of 1, at Route 201 (56, 4); planning again. +[Program]: [Event] 00.04/18 into_moms_call: start +[Console 0]: [00.04] into_moms_call: ran west to a dialogue: 2 moved chunks, 1 tiles at most, dialog, 0 wild battle(s). +'''.splitlines() + legs = leg_trace.replay(lines)['legs'] + self.assertEqual([(x['event'], x['leg'], x['dir'], x['tiles'], x['end_kind'], x['known']) for x in legs], [ + ('to_the_signs_row', 1, 'north', 1, 'tile', 'counted'), + ('to_ledge_column', 1, 'west', 7, 'wall', 'wall'), + ('to_ledge_column', 2, 'south', 1, 'open', 'seen'), + ('into_moms_call', 1, 'west', 1, 'open', 'dialogue'), + ]) + self.assertEqual(legs[0]['moved'], 1) + self.assertEqual(legs[1]['interrupts'], ['wild_battle']) + self.assertEqual(legs[1]['end'], at(56, 4)) + self.assertEqual(legs[2]['note'], 'replan') + self.assertEqual((legs[2]['interrupts'], legs[2]['ms'], legs[2]['moved']), (['blocked'], 800, 0)) + self.assertEqual(legs[2]['checkpoint'], '00.04') + + + def test_runs_of_a_checkpoint_are_told_apart(self): + record = json.dumps(SIM['legs'][0]) + lines = ['[Checkpoint] 00.04: start, 23 events', leg_trace.LEG_TAG + record, + '[Checkpoint] 00.04: start, 23 events', leg_trace.LEG_TAG + record, leg_trace.LEG_TAG + record] + self.assertEqual([x['attempt'] for x in leg_trace.replay(lines)['legs']], [1, 2, 2]) + self.assertEqual(len(leg_trace.replay(lines, attempt=1)['legs']), 1) + self.assertEqual(len(leg_trace.replay(lines, attempt=-1)['legs']), 2) + + +class DiffTest(unittest.TestCase): + def test_same_route_is_ok(self): + report = leg_trace.diff(console(), SIM) + self.assertEqual([g.grade for g in report.grades], [OK] * 4) + self.assertFalse(report.large()) + self.assertEqual(report.summary(), ['00.04: 4 legs: 4 OK, 0 drift, 0 LARGE, 0 masked']) + + def test_open_leg_inside_its_band_is_ok(self): + report = leg_trace.diff(console((2, {'end': at(49, 5)})), SIM) + self.assertEqual([g.grade for g in report.grades], [OK] * 4) + + def test_open_leg_off_then_a_wall_resyncs_is_drift(self): + report = leg_trace.diff(console((2, {'end': at(46, 5)})), SIM) + self.assertEqual([g.grade for g in report.grades], [OK, OK, DRIFT, OK]) + self.assertFalse(report.large()) + self.assertIn('re-synced at leg 4', report.grades[2].why) + + def test_open_leg_taps_short_then_a_wall_resyncs_is_drift(self): + report = leg_trace.diff(console((2, {'moved': 1}), (2, {'moved': 5})), {'legs': console((2, {'moved': 5}))['legs']}) + self.assertEqual([g.grade for g in report.grades], [OK] * 4) + short = console((2, {'moved': 1})) + sim = console((2, {'moved': 5})) + report = leg_trace.diff(short, sim) + self.assertEqual([g.grade for g in report.grades], [OK, OK, DRIFT, OK]) + self.assertIn('moved 1 of 5 tiles, the simulation 5', report.grades[2].why) + + def test_wall_leg_off_never_resynced_is_large(self): + report = leg_trace.diff(console((3, {'end': at(51, 3)})), SIM) + self.assertEqual([g.grade for g in report.grades], [OK, OK, OK, LARGE]) + self.assertTrue(report.large()) + self.assertEqual(report.first_divergence().console, 4) + + def test_different_map_is_large(self): + report = leg_trace.diff(console((0, {'end': at(56, 4, 'Twinleaf Town')})), SIM) + self.assertEqual(report.grades[0].grade, LARGE) + + def test_interruption_mid_leg_is_masked_and_resyncs(self): + # A wild battle on the open leg west: the console replans from where + # it is (two legs) and joins the simulation's route at the wall. + legs = json.loads(json.dumps(SIM['legs'][:2])) + legs += [ + leg('to_ledge_column', 3, 'west', 5, 'open', at(53, 5), interrupts=['wild_battle'], known='seen', note='replan'), + leg('to_ledge_column', 4, 'west', 2, 'open', at(51, 5)), + leg('to_ledge_column', 5, 'north', 3, 'wall', at(51, 2)), + ] + report = leg_trace.diff({'legs': legs}, SIM) + self.assertEqual(grades(report), [(1, 1, OK), (2, 2, OK), (3, 3, MASKED), (4, 3, MASKED), (5, 4, OK)]) + self.assertFalse(report.large()) + + def test_interruption_never_resynced_masks_the_rest(self): + legs = json.loads(json.dumps(SIM['legs'][:2])) + legs += [leg('to_ledge_column', 3, 'west', 5, 'open', at(53, 5), interrupts=['trainer'])] + legs += [leg('to_ledge_column', 4, 'north', 9, 'wall', at(53, 0))] + report = leg_trace.diff({'legs': legs}, SIM) + self.assertEqual([g.grade for g in report.grades], [OK, OK, MASKED, MASKED]) + self.assertFalse(report.large()) + + def test_replan_that_resyncs_is_drift(self): + legs = json.loads(json.dumps(SIM['legs'][:2])) + legs += [ + leg('to_ledge_column', 3, 'west', 5, 'open', at(51, 5), known='seen', note='replan'), + leg('to_ledge_column', 4, 'north', 1, 'wall', at(51, 4)), + leg('to_ledge_column', 5, 'north', 2, 'wall', at(51, 2)), + ] + report = leg_trace.diff({'legs': legs}, SIM) + self.assertEqual(grades(report), [(1, 1, OK), (2, 2, OK), (3, 3, OK), (4, 0, DRIFT), (5, 4, OK)]) + self.assertFalse(report.large()) + + def test_event_without_leg_lines_in_an_old_log_is_masked(self): + old = {'legs': SIM['legs'][:1], 'events': [['00.04', 'to_ledge_column', 1], ['00.04', 'into_the_east_corner', 1]]} + sim = {'legs': SIM['legs'][:1] + [leg('into_the_east_corner', 1, 'north', 4, 'wall', [])]} + report = leg_trace.diff(old, sim) + self.assertEqual([g.grade for g in report.grades], [OK, MASKED]) + + def test_missing_and_extra_legs_are_large(self): + report = leg_trace.diff({'legs': SIM['legs'][:3]}, SIM) + self.assertEqual(grades(report)[-1], (0, 4, LARGE)) + extra = {'legs': SIM['legs'] + [leg('to_ledge_column', 5, 'east', 1, 'open', at(52, 2))]} + self.assertEqual(grades(leg_trace.diff(extra, SIM))[-1], (5, 0, LARGE)) + + def test_different_plan_is_large(self): + report = leg_trace.diff(console((1, {'tiles': 2})), SIM) + self.assertEqual(report.grades[1].grade, LARGE) + + def test_event_the_simulation_lacks_is_large(self): + extra = {'legs': SIM['legs'] + [leg('to_ledge_column_recovery', 1, 'east', 1, 'open', at(52, 2))]} + report = leg_trace.diff(extra, SIM) + self.assertTrue(report.large()) + self.assertIn('to_ledge_column_recovery', report.summary()[0]) + + def test_recovery_loop_is_large(self): + loop = [leg('to_ledge_column', 5 + i, 'east' if i % 2 else 'west', 1, 'open', at(51, 5), interrupts=['wild_battle']) + for i in range(leg_trace.LOOP_EXTRA_LEGS + 1)] + report = leg_trace.diff({'legs': SIM['legs'] + loop}, SIM) + self.assertTrue(any('recovery loop' in f[2] for f in report.findings)) + + def test_slow_checkpoint_is_large(self): + slow = console(*[(i, {'ms': 3000}) for i in range(4)]) + report = leg_trace.diff(slow, SIM) + self.assertTrue(report.large()) + self.assertIn('walked 12000 ms, the simulation 4000 ms', report.summary()[0]) + + def test_slow_checkpoint_within_the_ratio_is_ok(self): + report = leg_trace.diff(console(*[(i, {'ms': 1400}) for i in range(4)]), SIM) + self.assertFalse(report.large()) + + +class GoldenTest(unittest.TestCase): + @unittest.skipUnless(os.path.exists(GOLDEN), 'the BDSP golden trace is not on this branch') + def test_golden_trace_against_itself(self): + with open(GOLDEN, encoding='utf-8') as f: + sim = json.load(f) + self.assertTrue(sim['legs']) + self.assertFalse(leg_trace.diff(sim, sim).large()) + + def test_cli_exit_code_is_large(self): + with tempfile.TemporaryDirectory() as folder: + off = os.path.join(folder, 'console.json') + same = os.path.join(folder, 'same.json') + sim = os.path.join(folder, 'sim.json') + for path, trace in ((off, console((3, {'end': at(51, 3)}))), (same, console()), (sim, SIM)): + with open(path, 'w', encoding='utf-8') as f: + json.dump(trace, f) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(leg_trace.main(['diff', off, sim]), 1) + self.assertEqual(leg_trace.main(['diff', same, sim]), 0) + + +if __name__ == '__main__': + unittest.main() From ac99c8950ce85f4d5f5433dadfcdfb0bf2284487 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sat, 14 Nov 2026 20:54:07 -0800 Subject: [PATCH 2/2] Pokemon Navigation: shared fault sweep runner, seeded rows and known-failing list SimSweep: sweep_result() (the log lines and message both games' sweeps print), run_sweep() over SweepVariants, sweep_clean_first() and variant_failure(). SimSeeds: run_seeds() with uint32 seeds, 30 by default, logging mean and worst per measure. KnownFailing: the pass-while-failing wrapper and stale() for entries that name no test. Unit tests for each; adoption in BDSP and FRLG follows per game. --- .../ComputerPrograms_UnitTestRunner.cpp | 6 + .../Simulation/Pokemon_KnownFailing.cpp | 54 ++++ .../Simulation/Pokemon_KnownFailing.h | 58 ++++ .../Simulation/Pokemon_KnownFailing_Tests.cpp | 209 +++++++++++++ .../Simulation/Pokemon_KnownFailing_Tests.h | 23 ++ .../Simulation/Pokemon_SimSeeds.cpp | 63 ++++ .../Navigation/Simulation/Pokemon_SimSeeds.h | 71 +++++ .../Simulation/Pokemon_SimSeeds_Tests.cpp | 197 ++++++++++++ .../Simulation/Pokemon_SimSeeds_Tests.h | 23 ++ .../Simulation/Pokemon_SimSweep.cpp | 68 +++++ .../Navigation/Simulation/Pokemon_SimSweep.h | 80 +++++ .../Simulation/Pokemon_SimSweep_Tests.cpp | 283 ++++++++++++++++++ .../Simulation/Pokemon_SimSweep_Tests.h | 23 ++ SerialPrograms/cmake/SourceFiles.cmake | 12 + 14 files changed, 1170 insertions(+) create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.cpp create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.h create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.cpp create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.h create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.cpp create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.h create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds_Tests.cpp create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds_Tests.h create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSweep.cpp create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSweep.h create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSweep_Tests.cpp create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSweep_Tests.h diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index 16bf093d82..59d2de190d 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -43,7 +43,10 @@ #include "Pokemon/Navigation/Pokemon_RouteLog_Tests.h" #include "Pokemon/Navigation/Pokemon_RuntimeMap_Tests.h" #include "Pokemon/Navigation/Pokemon_WallRecovery_Tests.h" +#include "Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.h" #include "Pokemon/Navigation/Simulation/Pokemon_SimGolden_Tests.h" +#include "Pokemon/Navigation/Simulation/Pokemon_SimSeeds_Tests.h" +#include "Pokemon/Navigation/Simulation/Pokemon_SimSweep_Tests.h" #include "VideoGames/PokemonHome/PokemonHome_Tests.h" #include "VideoGames/PokemonSwSh/PokemonSwSh_Tests.h" #include "VideoGames/PokemonLA/PokemonLA_Tests.h" @@ -91,6 +94,9 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){ Pokemon::add_tests_RuntimeMap(ret); Pokemon::add_tests_RouteLog(ret); Pokemon::add_tests_SimGolden(ret); + Pokemon::add_tests_SimSweep(ret); + Pokemon::add_tests_SimSeeds(ret); + Pokemon::add_tests_KnownFailing(ret); Pokemon::add_tests_AutoStory(ret); NintendoSwitch::add_tests_CheckOnlineDetector(ret); NintendoSwitch::add_tests_FailedToConnectDetector(ret); diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.cpp b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.cpp new file mode 100644 index 0000000000..e01bbe4e07 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.cpp @@ -0,0 +1,54 @@ +/* Known Failing + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Pokemon_KnownFailing.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +KnownFailing::KnownFailing(std::map fixes) + : m_fixes(std::make_move_iterator(fixes.begin()), std::make_move_iterator(fixes.end())) +{} + +UnitTestResult KnownFailing::run(Logger& logger, std::string_view name, const std::function& body) const{ + UnitTestResult result; + try{ + result = body(); + }catch (Exception& e){ + result = std::format("{}: {}", e.name(), e.message()); + } + const auto known = m_fixes.find(name); + if (known == m_fixes.end()){ + return result; + } + if (result.result == UnitTestResult::PASSED){ + return std::format("on KNOWN_FAILING ({}) but passes now: take it off the list.", known->second); + } + logger.log(std::format("Known failing until {}: {}", known->second, result.message)); + return true; +} + +std::vector KnownFailing::stale(const std::set& names) const{ + std::vector out; + for (const auto& [name, fix] : m_fixes){ + if (!names.contains(name)){ + out.push_back(name); + } + } + return out; +} + + + +} // namespace Pokemon + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.h b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.h new file mode 100644 index 0000000000..b6684a4ccb --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.h @@ -0,0 +1,58 @@ +/* Known Failing + * + * From: https://github.com/PokemonAutomation/ + * + * The list of sim tests the program fails today, each with the change + * likely to fix it, shared by the games' fault sweeps and console-failure + * cases. A test on the list passes while it fails, so the suite stays + * green with the failure logged, and fails once it passes, so the entry + * comes off. stale() finds the entries that name no test: a renamed test + * would otherwise drop off the list silently and fail the suite. + * + */ + +#ifndef PokemonAutomation_Pokemon_KnownFailing_H +#define PokemonAutomation_Pokemon_KnownFailing_H + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "Common/Cpp/TestRunners/UnitTest.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// The tests known to fail, by name, with their likely fixes. +class KnownFailing{ +public: + // `fixes`: by test name (or case name, as the game keys it), the + // change likely to fix it. + explicit KnownFailing(std::map fixes); + + // Runs test `name`'s `body`, an Exception it throws being its failure, + // and gives its result as the list sees it: as is if `name` is not + // listed; a pass that logs the failure if it is and fails; a failure + // "on KNOWN_FAILING () but passes now: take it off the list." if + // it is and passes. + UnitTestResult run(Logger& logger, std::string_view name, const std::function& body) const; + + // The listed names not in `names` (the tests that exist), in order. + std::vector stale(const std::set& names) const; + +private: + std::map> m_fixes; +}; + + + +} // namespace Pokemon + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.cpp b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.cpp new file mode 100644 index 0000000000..ff7dfbd7b6 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.cpp @@ -0,0 +1,209 @@ +/* Known Failing Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The known-failing list on fake tests: one on the list passes while it + * fails and fails once it passes, one off the list keeps its result, an + * exception is a failure, and entries naming no test are stale. + * + */ + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "Pokemon_KnownFailing.h" +#include "Pokemon_KnownFailing_Tests.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + +class LinesLogger : public Logger{ +public: + using Logger::log; + virtual void log(const std::string& msg, Color) override{ + lines.push_back(msg); + } + + std::vector lines; +}; + +const KnownFailing& known_failing(){ + static const KnownFailing list({ + {"listed", "the planner's goal kinds"}, + {"listed and gone", "a fix for a test since renamed"}, + }); + return list; +} + +// How the fake test's body ends. +enum class BodyOutcome{ + PASSES, + FAILS, + THROWS, +}; + +struct KnownFailingCase{ + struct Input{ + std::string test; + BodyOutcome body; + }; + struct Expectation{ + bool passes; + std::string message; + std::vector log; + }; + + std::string name; + Input input; + Expectation expectation; +}; + +const KnownFailingCase KNOWN_FAILING_CASES[] = { + { + .name = "a listed test passes while it fails", + .input = {.test = "listed", .body = BodyOutcome::FAILS}, + .expectation = {.passes = true, .message = "", .log = {"Known failing until the planner's goal kinds: stopped at the gate."}}, + }, + { + .name = "a listed test fails once it passes", + .input = {.test = "listed", .body = BodyOutcome::PASSES}, + .expectation = { + .passes = false, + .message = "on KNOWN_FAILING (the planner's goal kinds) but passes now: take it off the list.", + .log = {}, + }, + }, + { + .name = "a listed test's exception is its known failure", + .input = {.test = "listed", .body = BodyOutcome::THROWS}, + .expectation = {.passes = true, .message = "", .log = {"Known failing until the planner's goal kinds: ParseException: no row 99.99"}}, + }, + { + .name = "an unlisted test keeps its failure", + .input = {.test = "unlisted", .body = BodyOutcome::FAILS}, + .expectation = {.passes = false, .message = "stopped at the gate.", .log = {}}, + }, + { + .name = "an unlisted test keeps its pass", + .input = {.test = "unlisted", .body = BodyOutcome::PASSES}, + .expectation = {.passes = true, .message = "", .log = {}}, + }, + { + .name = "an unlisted test's exception is its failure", + .input = {.test = "unlisted", .body = BodyOutcome::THROWS}, + .expectation = {.passes = false, .message = "ParseException: no row 99.99", .log = {}}, + }, +}; + +UnitTestResult fake_body(BodyOutcome outcome){ + switch (outcome){ + case BodyOutcome::PASSES: + return true; + case BodyOutcome::FAILS: + return "stopped at the gate."; + case BodyOutcome::THROWS: + throw ParseException("no row 99.99"); + } + return true; +} + +class Test_KnownFailing : public UnitTest{ +public: + explicit Test_KnownFailing(KnownFailingCase test_case) + : UnitTest(std::format("Pokemon::Navigation - Known failing: {}", test_case.name)) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + LinesLogger logger; + const UnitTestResult result = known_failing().run(logger, m_case.input.test, [this]{ + return fake_body(m_case.input.body); + }); + const KnownFailingCase::Expectation& expected = m_case.expectation; + const bool passed = result.result == UnitTestResult::PASSED; + if (passed != expected.passes){ + return std::format("expected {}, got \"{}\".", expected.passes ? "a pass" : "a failure", result.message); + } + if (result.message != expected.message){ + return std::format("message: expected \"{}\", got \"{}\".", expected.message, result.message); + } + if (logger.lines != expected.log){ + return std::format("log: expected {} lines, got {}: \"{}\".", expected.log.size(), logger.lines.size(), logger.lines.empty() ? "" : logger.lines[0]); + } + return true; + } + +private: + KnownFailingCase m_case; +}; + + +struct StaleCase{ + struct Input{ + std::set tests; + }; + struct Expectation{ + std::vector stale; + }; + + std::string name; + Input input; + Expectation expectation; +}; + +const StaleCase STALE_CASES[] = { + { + .name = "an entry naming no test is stale", + .input = {.tests = {"listed", "unlisted"}}, + .expectation = {.stale = {"listed and gone"}}, + }, + { + .name = "entries naming tests are not", + .input = {.tests = {"listed", "listed and gone"}}, + .expectation = {.stale = {}}, + }, +}; + +class Test_KnownFailingStale : public UnitTest{ +public: + explicit Test_KnownFailingStale(StaleCase test_case) + : UnitTest(std::format("Pokemon::Navigation - Known failing: {}", test_case.name)) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::vector stale = known_failing().stale(m_case.input.tests); + if (stale != m_case.expectation.stale){ + return std::format("expected {} stale entries, got {}.", m_case.expectation.stale.size(), stale.size()); + } + return true; + } + +private: + StaleCase m_case; +}; + +} // namespace + + +void add_tests_KnownFailing(UnitTestDatabase& database){ + for (const KnownFailingCase& item : KNOWN_FAILING_CASES){ + database.add(item); + } + for (const StaleCase& item : STALE_CASES){ + database.add(item); + } +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.h b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.h new file mode 100644 index 0000000000..49b7baa538 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.h @@ -0,0 +1,23 @@ +/* Known Failing Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_Pokemon_KnownFailing_Tests_H +#define PokemonAutomation_Pokemon_KnownFailing_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// Registers the known-failing list tests. +void add_tests_KnownFailing(UnitTestDatabase& database); + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.cpp b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.cpp new file mode 100644 index 0000000000..a56b6e9e25 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.cpp @@ -0,0 +1,63 @@ +/* Sim Seeds + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "Pokemon_SimSeeds.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + +// The mean and worst (largest) of `values`, which has one at least. +SeedStats stats_of(const std::vector& values){ + return { + .mean = std::accumulate(values.begin(), values.end(), 0.) / static_cast(values.size()), + .worst = *std::max_element(values.begin(), values.end()), + }; +} + +} // namespace + + +SeedsResult run_seeds( + Logger& logger, std::string_view label, const std::function& run_one, + uint32_t seeds +){ + SeedsResult out; + out.seeds = seeds; + std::map> passed; + for (uint32_t seed = 1; seed <= seeds; seed++){ + SeedRun run = run_one(seed); + if (run.failure){ + out.failures.push_back({"", std::format("seed {}", seed), std::move(*run.failure)}); + continue; + } + for (const auto& [measure, value] : run.measures){ + passed[measure].push_back(value); + } + } + std::string line = std::format("Seeds {}: passed {}/{}", label, seeds - out.failures.size(), seeds); + for (const auto& [measure, values] : passed){ + const SeedStats stats = stats_of(values); + line += std::format("; {} mean {:.1f}, worst {:.1f}", measure, stats.mean, stats.worst); + out.stats.emplace(measure, stats); + } + logger.log(line); + return out; +} + + + +} // namespace Pokemon + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.h b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.h new file mode 100644 index 0000000000..2fc7088181 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.h @@ -0,0 +1,71 @@ +/* Sim Seeds + * + * From: https://github.com/PokemonAutomation/ + * + * The games' Monte Carlo rows, shared by BDSP's and FRLG's: a seeded + * row (wild battles at the game's odds, wandering people) runs once per + * seed, each seed feeding the sim's std::mt19937. Its failures are a + * fault sweep's (sweep_result()), a variant per seed, and it logs the + * mean and worst of what each seed measured. + * + */ + +#ifndef PokemonAutomation_Pokemon_SimSeeds_H +#define PokemonAutomation_Pokemon_SimSeeds_H + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "Pokemon_SimSweep.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// How many seeds a seeded row runs unless it asks for others. +inline constexpr uint32_t DEFAULT_SWEEP_SEEDS = 30; + +// One seed's run of a row: why it failed, or nothing; and what it measured +// by name ("s", "stops", ...), only what applies to the seed. +struct SeedRun{ + std::optional failure = std::nullopt; + std::map measures = {}; +}; + +// A measure over the seeds that passed and measured it. +struct SeedStats{ + double mean = 0; + double worst = 0; +}; + +// The seeds a row ran, those that failed, and what the rest measured. +struct SeedsResult{ + uint32_t seeds = 0; + // A failure per failed seed, its variant "seed N". + std::vector failures = {}; + // By measure, over the passed seeds; a measure no passed seed has is + // left out. + std::map stats = {}; +}; + +// Runs `run_one` from seeds 1 to `seeds`; logs "Seeds