diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index 16bf093d82..59d2de190d 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -43,7 +43,10 @@ #include "Pokemon/Navigation/Pokemon_RouteLog_Tests.h" #include "Pokemon/Navigation/Pokemon_RuntimeMap_Tests.h" #include "Pokemon/Navigation/Pokemon_WallRecovery_Tests.h" +#include "Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.h" #include "Pokemon/Navigation/Simulation/Pokemon_SimGolden_Tests.h" +#include "Pokemon/Navigation/Simulation/Pokemon_SimSeeds_Tests.h" +#include "Pokemon/Navigation/Simulation/Pokemon_SimSweep_Tests.h" #include "VideoGames/PokemonHome/PokemonHome_Tests.h" #include "VideoGames/PokemonSwSh/PokemonSwSh_Tests.h" #include "VideoGames/PokemonLA/PokemonLA_Tests.h" @@ -91,6 +94,9 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){ Pokemon::add_tests_RuntimeMap(ret); Pokemon::add_tests_RouteLog(ret); Pokemon::add_tests_SimGolden(ret); + Pokemon::add_tests_SimSweep(ret); + Pokemon::add_tests_SimSeeds(ret); + Pokemon::add_tests_KnownFailing(ret); Pokemon::add_tests_AutoStory(ret); NintendoSwitch::add_tests_CheckOnlineDetector(ret); NintendoSwitch::add_tests_FailedToConnectDetector(ret); diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegReplay.cpp b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegReplay.cpp new file mode 100644 index 0000000000..d593f1949c --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegReplay.cpp @@ -0,0 +1,133 @@ +/* Leg Replay + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "Common/Cpp/Json/JsonArray.h" +#include "Common/Cpp/Json/JsonObject.h" +#include "Common/Cpp/Json/JsonValue.h" +#include "Pokemon_LegReplay.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + +namespace{ + +[[noreturn]] void bad(const std::string& name, const std::string& what){ + throw std::runtime_error("console failure " + name + ": " + what); +} + +Direction direction_of(const std::string& name, const std::string& text){ + for (const Direction direction : ALL_DIRECTIONS){ + if (direction_name(direction) == text){ + return direction; + } + } + bad(name, "unknown direction \"" + text + "\""); +} + +TilePos tile_of(const std::string& name, const JsonArray& array){ + if (array.size() != 2){ + bad(name, "a tile is [x, y]"); + } + return {(int)array[0].to_integer_throw(), (int)array[1].to_integer_throw()}; +} + +ReplayFade fade_of(const std::string& name, const std::string& text){ + if (text == "unseen"){ + return ReplayFade::UNSEEN; + } + if (text == "black"){ + return ReplayFade::BLACK; + } + if (text == "dim"){ + return ReplayFade::DIM; + } + if (text == "white"){ + return ReplayFade::WHITE; + } + bad(name, "unknown fade \"" + text + "\""); +} + +ReplayFault fault_of(const std::string& name, const JsonObject& json){ + const std::string kind = json.get_string_throw("kind"); + ReplayFault fault; + if (kind == "fade_missed" || kind == "banner_missed"){ + fault.kind = kind == "fade_missed" ? ReplayFaultKind::FADE_MISSED : ReplayFaultKind::BANNER_MISSED; + fault.index = (size_t)json.get_integer_throw("index"); + }else if (kind == "moved_further" || kind == "hold_through_warp" || kind == "scene_ends_further"){ + fault.kind = kind == "moved_further" ? ReplayFaultKind::MOVED_FURTHER + : kind == "hold_through_warp" ? ReplayFaultKind::HOLD_THROUGH_WARP : ReplayFaultKind::SCENE_ENDS_FURTHER; + fault.step = json.get_string_throw("step"); + fault.direction = direction_of(name, json.get_string_throw("direction")); + fault.tiles = (size_t)json.get_integer_default("tiles", 1); + fault.fade = fade_of(name, json.get_string_default("fade", "unseen")); + fault.far_pose_of = json.get_string_default("far_pose_of", ""); + }else if (kind == "moved_less"){ + fault.kind = ReplayFaultKind::MOVED_LESS; + fault.step = json.get_string_throw("step"); + fault.index = (size_t)json.get_integer_throw("index"); + fault.tiles = (size_t)json.get_integer_throw("tiles"); + }else if (kind == "person_stands"){ + fault.kind = ReplayFaultKind::PERSON_STANDS; + fault.name = json.get_string_throw("name"); + fault.tile = tile_of(name, json.get_array_throw("tile")); + }else{ + bad(name, "unknown fault kind \"" + kind + "\""); + } + return fault; +} + +} // namespace + + +std::string replay_fault_kind_name(ReplayFaultKind kind){ + switch (kind){ + case ReplayFaultKind::FADE_MISSED: + return "fade_missed"; + case ReplayFaultKind::BANNER_MISSED: + return "banner_missed"; + case ReplayFaultKind::MOVED_FURTHER: + return "moved_further"; + case ReplayFaultKind::HOLD_THROUGH_WARP: + return "hold_through_warp"; + case ReplayFaultKind::SCENE_ENDS_FURTHER: + return "scene_ends_further"; + case ReplayFaultKind::PERSON_STANDS: + return "person_stands"; + case ReplayFaultKind::MOVED_LESS: + return "moved_less"; + } + return ""; +} + +ConsoleFailure parse_console_failure(const std::string& name, const std::string& text){ + const JsonValue value = parse_json(text); + const JsonObject& json = value.to_object_throw(); + ConsoleFailure out; + out.name = name; + out.checkpoint = json.get_string_throw("checkpoint"); + if (const JsonObject* start = json.get_object("start")){ + out.start = ReplayTile{start->get_string_throw("map"), tile_of(name, start->get_array_throw("tile"))}; + out.start_facing = direction_of(name, start->get_string_throw("facing")); + } + for (const auto& item : json.get_array_throw("faults")){ + out.faults.push_back(fault_of(name, item.to_object_throw())); + } + const JsonObject& failed = json.get_object_throw("failed"); + out.failed_step = failed.get_string_throw("step"); + out.failed_error = failed.get_string_throw("error"); + out.frame = json.get_string_throw("frame"); + if (const JsonArray* evidence = json.get_array("evidence")){ + for (const auto& line : *evidence){ + out.evidence.push_back(line.to_string_throw()); + } + } + return out; +} + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegReplay.h b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegReplay.h new file mode 100644 index 0000000000..4f699912c1 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegReplay.h @@ -0,0 +1,107 @@ +/* Leg Replay + * + * From: https://github.com/PokemonAutomation/ + * + * A console failure as a small fault file (the files of a game's ConsoleFailures folder beside + * a game's simulation tests), made from a console log's [Leg] records and + * raw walk facts by devtools/maps/leg_replay.py. The file keeps only what + * the simulation cannot work out from its map: where the run started if + * not on its checkpoint's start, the detections it missed, the walks that + * went further than counted, and the people the map lacks. Trainers, NPCs + * and walls come from the map. Where the failure happened and the frame it + * was checked against ride along for the reader. + * + * {"checkpoint": "01.02", + * "faults": [{"kind": "hold_through_warp", "step": "past_curtis", + * "direction": "east", "fade": "unseen", "far_pose_of": "into_oreburgh"}], + * "failed": {"step": "into_oreburgh", "error": "no_banner"}, +// code-health: allow header_symbols -- the fault file model and its reader are one unit, a split would scatter it + */ + +#ifndef PokemonAutomation_Pokemon_LegReplay_H +#define PokemonAutomation_Pokemon_LegReplay_H + +#include +#include +#include +#include +#include "Pokemon_Direction.h" +#include "Pokemon_Grid.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// A tile of a named map. +struct ReplayTile{ + std::string map; + TilePos tile; +}; + +// What went differently on the console than the program counted on. +enum class ReplayFaultKind{ + FADE_MISSED, // the `index`th fade of the run goes unseen + BANNER_MISSED, // the `index`th map banner of the run goes unseen + MOVED_FURTHER, // the step's counted taps went `tiles` further (onto a warp, `far_pose_of`) + HOLD_THROUGH_WARP, // the step's hold toward `direction` ran on through a warp (`far_pose_of`) + SCENE_ENDS_FURTHER, // the player stood `tiles` toward `direction` past where the scene step leaves them + PERSON_STANDS, // `name`, a person of the map (who wanders or stays put), stood on `tile` + MOVED_LESS, // the `index`th hold of `step`'s walk (from 1) moved `tiles` fewer than it was held for, with + // nothing on screen: the walker took the move as seen +}; +// "fade_missed", "banner_missed", "moved_further", "hold_through_warp", "scene_ends_further", "person_stands", +// "moved_less". +std::string replay_fault_kind_name(ReplayFaultKind kind); + +// How the walk saw the fade of a warp it went through. +enum class ReplayFade{ + UNSEEN, + BLACK, + DIM, + WHITE, +}; + +// One fault; the fields of its kind (above) are the ones that count. +struct ReplayFault{ + ReplayFaultKind kind = ReplayFaultKind::FADE_MISSED; + // FADE_MISSED, BANNER_MISSED: counted within the checkpoint, from 0. + // MOVED_LESS: counted within the step's walk, from 1. + size_t index = 0; + // MOVED_FURTHER, HOLD_THROUGH_WARP, SCENE_ENDS_FURTHER, MOVED_LESS. + std::string step; + Direction direction = Direction::EAST; + size_t tiles = 1; + ReplayFade fade = ReplayFade::UNSEEN; + bool banner_up = false; + // The step of the checkpoint whose pose after it is where the warp lands. + std::string far_pose_of; + // PERSON_STANDS. + std::string name; + TilePos tile; +}; + +// A console failure: the checkpoint it ran, its faults and what it failed at. +struct ConsoleFailure{ + std::string name; + std::string checkpoint; + // Where the run starts if not on the checkpoint's start. + std::optional start; + std::optional start_facing; + std::vector faults; + // What the console did: the step and the start of its error. + std::string failed_step; + std::string failed_error; + // The frame the failure was checked against, a file name in the lab's + // failure screenshots. + std::string frame; + // The log lines the faults were worked out from. + std::vector evidence; +}; + +// Reads a fault file; throws on a missing or unknown field. +ConsoleFailure parse_console_failure(const std::string& name, const std::string& text); + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace.cpp b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace.cpp new file mode 100644 index 0000000000..8043c6c8f4 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace.cpp @@ -0,0 +1,267 @@ +/* Leg Trace + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "Common/Cpp/Json/JsonArray.h" +#include "Common/Cpp/Json/JsonObject.h" +#include "Common/Cpp/Json/JsonValue.h" +#include "Pokemon_LegTrace.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +std::string_view leg_end_kind_name(LegEndKind kind){ + switch (kind){ + case LegEndKind::OPEN: + return "open"; + case LegEndKind::WALL: + return "wall"; + case LegEndKind::TILE: + return "tile"; + } + return "?"; +} + +std::string_view leg_end_known_name(LegEndKnown known){ + switch (known){ + case LegEndKnown::WALL: + return "wall"; + case LegEndKnown::FADE: + return "fade"; + case LegEndKnown::DIALOGUE: + return "dialogue"; + case LegEndKnown::COUNTED: + return "counted"; + case LegEndKnown::DEAD_RECKONED: + return "dead_reckoned"; + case LegEndKnown::SEEN: + return "seen"; + case LegEndKnown::UNKNOWN: + return "unknown"; + } + return "?"; +} + +std::string_view leg_interrupt_name(LegInterrupt interrupt){ + switch (interrupt){ + case LegInterrupt::WILD_BATTLE: + return "wild_battle"; + case LegInterrupt::TRAINER: + return "trainer"; + case LegInterrupt::SCENE: + return "scene"; + case LegInterrupt::DIALOGUE: + return "dialogue"; + case LegInterrupt::BLOCKED: + return "blocked"; + } + return "?"; +} + + +namespace{ + +// [["Route 202", 23, 25], ...] +JsonArray tiles_json(const std::vector& tiles){ + JsonArray out; + for (const RouteTile& tile : tiles){ + JsonArray entry; + entry.push_back(tile.map); + entry.push_back((int64_t)tile.pos.x); + entry.push_back((int64_t)tile.pos.y); + out.push_back(std::move(entry)); + } + return out; +} + +} // namespace + +std::string leg_trace_json(const LegRecord& record){ + JsonObject out; + out["checkpoint"] = record.checkpoint; + out["event"] = record.event; + out["leg"] = (int64_t)record.leg; + out["dir"] = direction_name(record.plan.direction); + out["tiles"] = (int64_t)record.plan.tiles; + out["end_kind"] = std::string(leg_end_kind_name(record.plan.end_kind)); + out["planned_end"] = tiles_json(record.plan.ends); + if (record.moved){ + out["moved"] = (int64_t)*record.moved; + } + out["chunks"] = (int64_t)record.chunks; + out["ms"] = record.walked_ms; + out["end"] = tiles_json(record.end); + out["known"] = std::string(leg_end_known_name(record.known)); + JsonArray interrupts; + for (LegInterrupt interrupt : record.interrupts){ + interrupts.push_back(std::string(leg_interrupt_name(interrupt))); + } + out["interrupts"] = std::move(interrupts); + if (!record.note.empty()){ + out["note"] = record.note; + } + return out.dump(-1); +} + +std::string leg_trace_line(const LegRecord& record){ + return std::string(LEG_TRACE_TAG) + leg_trace_json(record); +} + + +namespace{ + +std::optional parse_int(std::string_view text){ + int value = 0; + const auto result = std::from_chars(text.data(), text.data() + text.size(), value); + if (result.ec != std::errc() || result.ptr != text.data() + text.size()){ + return std::nullopt; + } + return value; +} + +// "Route 202 (23, 25)". +std::optional parse_route_tile(std::string_view text){ + const size_t open = text.rfind(" ("); + const size_t comma = text.rfind(", "); + if (open == std::string_view::npos || comma == std::string_view::npos || comma < open || !text.ends_with(")")){ + return std::nullopt; + } + const std::optional x = parse_int(text.substr(open + 2, comma - open - 2)); + const std::optional y = parse_int(text.substr(comma + 2, text.size() - comma - 3)); + if (open == 0 || !x || !y){ + return std::nullopt; + } + return RouteTile{std::string(text.substr(0, open)), TilePos{*x, *y}}; +} + +std::optional parse_direction(std::string_view text){ + for (Direction direction : ALL_DIRECTIONS){ + if (text == direction_name(direction)){ + return direction; + } + } + return std::nullopt; +} + +} // namespace + +std::vector parse_route_tiles(std::string_view text){ + std::vector out; + while (!text.empty()){ + // Each tile ends at its ")"; ", " separates them. + const size_t close = text.find(')'); + if (close == std::string_view::npos){ + return {}; + } + const std::optional tile = parse_route_tile(text.substr(0, close + 1)); + if (!tile){ + return {}; + } + out.push_back(*tile); + text.remove_prefix(close + 1); + if (text.starts_with(", ")){ + text.remove_prefix(2); + }else if (!text.empty()){ + return {}; + } + } + return out; +} + +std::optional parse_route_leg_line(std::string_view text){ + // "Leg /: () to " + if (!text.starts_with("Leg ")){ + return std::nullopt; + } + const size_t colon = text.find(": "); + if (colon == std::string_view::npos){ + return std::nullopt; + } + text.remove_prefix(colon + 2); + const size_t space = text.find(' '); + const size_t open = text.find(" ("); + const size_t close = text.find(") to "); + if (space == std::string_view::npos || open == std::string_view::npos || close == std::string_view::npos + || !(space <= open && open < close)){ + return std::nullopt; + } + const std::optional direction = parse_direction(text.substr(0, space)); + const std::optional tiles = parse_int(text.substr(space + 1, open - space - 1)); + const std::string_view end = text.substr(open + 2, close - open - 2); + if (!direction || !tiles || *tiles < 0){ + return std::nullopt; + } + LegPlan plan{.direction = *direction, .tiles = (size_t)*tiles}; + if (end == "to wall"){ + plan.end_kind = LegEndKind::WALL; + }else if (end == "warp"){ + plan.end_kind = LegEndKind::TILE; + }else if (end == "open"){ + plan.end_kind = LegEndKind::OPEN; + }else{ + return std::nullopt; + } + plan.ends = parse_route_tiles(text.substr(close + 5)); + return plan; +} + + +void LegTracer::begin_event(const std::string& checkpoint, const std::string& event){ + end_event(); + m_checkpoint = checkpoint; + m_event = event; + m_legs = 0; +} + +void LegTracer::end_event(){ + if (m_leg){ + end_leg({}, LegEndKnown::UNKNOWN); + } +} + +void LegTracer::begin_leg(LegPlan plan){ + end_event(); + m_leg = LegRecord{.checkpoint = m_checkpoint, .event = m_event, .leg = ++m_legs, .plan = std::move(plan)}; +} + +void LegTracer::walked(const LegWalk& walk){ + if (!m_leg){ + return; + } + m_leg->chunks += walk.chunks; + m_leg->walked_ms += walk.walked_ms; + for (size_t i = 0; i < walk.wild_battles; i++){ + m_leg->interrupts.push_back(LegInterrupt::WILD_BATTLE); + } + if (walk.tapped){ + m_leg->moved = m_leg->moved.value_or(0) + *walk.tapped; + } +} + +void LegTracer::interrupted(LegInterrupt interrupt){ + if (m_leg){ + m_leg->interrupts.push_back(interrupt); + } +} + +void LegTracer::end_leg(std::vector end, LegEndKnown known, std::string note){ + if (!m_leg){ + return; + } + LegRecord record = std::move(*m_leg); + m_leg.reset(); + record.end = std::move(end); + record.known = known; + record.note = std::move(note); + m_emit(leg_trace_line(record)); +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace.h b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace.h new file mode 100644 index 0000000000..d177c6e09b --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace.h @@ -0,0 +1,167 @@ +/* Leg Trace + * + * From: https://github.com/PokemonAutomation/ + * + * One structured record per walked leg, shared by the games' walkers and + * their simulations, so a console run's route can be compared leg by leg + * with the simulation's (devtools/maps/leg_trace.py). Each leg is one log + * line beside the free-text route lines: + * + * [Leg] {"checkpoint":"00.04","event":"to_the_signs_row","leg":1, + * "dir":"north","tiles":1,"end_kind":"tile","moved":1,...} + * + * A record holds what was planned (direction, tiles, how the leg ends and + * the tiles it may end on), what was measured (taps that moved, the + * walk's moved chunks and held time), where the leg ended and how that is + * known, and what interrupted it. + * +// code-health: allow header_symbols -- leg trace diagnostic models and route execution logging types + */ + +#ifndef PokemonAutomation_Pokemon_LegTrace_H +#define PokemonAutomation_Pokemon_LegTrace_H + +#include +#include +#include +#include +#include +#include +#include +#include "Pokemon_Grid.h" +#include "Pokemon_RouteLog.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// Prefix tag of a leg record's log line. +inline constexpr std::string_view LEG_TRACE_TAG = "[Leg] "; + +// How a leg is planned to end. +enum class LegEndKind{ + OPEN, // timed or tapped into the open + WALL, // held until a wall stops it + TILE, // counted taps onto an exact tile +}; +// "open", "wall", "tile". +std::string_view leg_end_kind_name(LegEndKind kind); + +// How the end of a leg is known. +enum class LegEndKnown{ + WALL, // a wall stopped the walk + FADE, // a warp's fade or banner ended it + DIALOGUE, // a dialogue stopped it + COUNTED, // taps that moved, counted against the map + DEAD_RECKONED, // taken to have ended as planned + SEEN, // stopped short, where the walker found itself + UNKNOWN, // not known: the leg was cut off +}; +// "wall", "fade", "dialogue", "counted", "dead_reckoned", "seen", "unknown". +std::string_view leg_end_known_name(LegEndKnown known); + +// What stopped or delayed a leg on the way. +enum class LegInterrupt{ + WILD_BATTLE, + TRAINER, + SCENE, + DIALOGUE, + BLOCKED, // someone the map lacks stood in the way +}; +// "wild_battle", "trainer", "scene", "dialogue", "blocked". +std::string_view leg_interrupt_name(LegInterrupt interrupt); + +// What a leg is planned to do. +struct LegPlan{ + Direction direction = Direction::NORTH; + size_t tiles = 0; + LegEndKind end_kind = LegEndKind::OPEN; + // Every tile the leg may end on; empty where the plan does not say. + std::vector ends; +}; + +// One walked leg. +struct LegRecord{ + std::string checkpoint; + std::string event; + // 1-based, counted through the event, replans included. + size_t leg = 0; + LegPlan plan; + // The taps that moved, where the leg counts them. + std::optional moved; + // What the walks of the leg report: chunks of screen movement, and the + // time the direction was held. + size_t chunks = 0; + int64_t walked_ms = 0; + // The tiles the leg ended on, as the walker knows them. + std::vector end; + LegEndKnown known = LegEndKnown::UNKNOWN; + std::vector interrupts; + // Why the leg ended other than as planned ("replan", "warp"), or "". + std::string note; +}; + +// The record as one line of compact JSON, keys sorted. +std::string leg_trace_json(const LegRecord& record); +// LEG_TRACE_TAG and the record's JSON. +std::string leg_trace_line(const LegRecord& record); + + +// The route log's text read back: "Route 202 (23, 25), Route 202 (30, 26)". +// Nothing for text that is not a list of tiles. +std::vector parse_route_tiles(std::string_view text); +// A route_leg_line() read back: "Leg 1/2: east 15 (to wall) to ". +// A warp leg ends at its tiles, as a tile leg. +std::optional parse_route_leg_line(std::string_view text); + + +// What one walk of a leg reports. +struct LegWalk{ + // Chunks of screen movement, and the time the direction was held. + size_t chunks = 0; + int64_t walked_ms = 0; + size_t wild_battles = 0; + // The taps that moved, for a walk of taps. + std::optional tapped; +}; + +// Builds a walker's leg records from what it does, and hands each, as its +// log line, to `emit` once the leg ends. A leg still open when the next +// starts, or when its event ends, is emitted with its end unknown. +class LegTracer{ +public: + explicit LegTracer(std::function emit) + : m_emit(std::move(emit)) + {} + + // The legs from here on are `event`'s of `checkpoint`. + void begin_event(const std::string& checkpoint, const std::string& event); + // Emits the leg still open, if one is. + void end_event(); + + // A leg starts. + void begin_leg(LegPlan plan); + // The open leg's plan; null if no leg is open. + const LegPlan* open_leg() const{ + return m_leg ? &m_leg->plan : nullptr; + } + // A walk of the open leg ended. A leg can take several (a walk on to + // the wall after a battle); they add up. + void walked(const LegWalk& walk); + void interrupted(LegInterrupt interrupt); + // The open leg ended on `end`, known by `known`; emits it. + void end_leg(std::vector end, LegEndKnown known, std::string note = ""); + +private: + std::function m_emit; + std::string m_checkpoint; + std::string m_event; + size_t m_legs = 0; + std::optional m_leg; +}; + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace_Tests.cpp b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace_Tests.cpp new file mode 100644 index 0000000000..dc2c889309 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace_Tests.cpp @@ -0,0 +1,167 @@ +/* Leg Trace Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The leg record's line word for word, the route log's leg lines read + * back, and the tracer's records for a leg that ends, one cut off by the + * next, and one with a wild battle on the way. + * + */ + +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "Pokemon_LegTrace.h" +#include "Pokemon_LegTrace_Tests.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + +struct LegTraceInput{ + std::string name; + std::function actual; +}; +struct LegTraceExpectation{ + std::string expected; +}; +struct LegTraceTestCase{ + LegTraceInput input; + LegTraceExpectation expectation; +}; + +class Test_LegTrace : public UnitTest{ +public: + explicit Test_LegTrace(LegTraceTestCase scenario) + : UnitTest("Pokemon::LegTrace - " + scenario.input.name) + , m_scenario(std::move(scenario)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::string actual = m_scenario.input.actual(); + if (actual == m_scenario.expectation.expected){ + return true; + } + return "expected:\n" + m_scenario.expectation.expected + "\ngot:\n" + actual; + } + +private: + LegTraceTestCase m_scenario; +}; + +std::string tiles_text(const std::vector& tiles){ + return tiles.empty() ? "none" : describe(tiles); +} + +std::string plan_text(const std::optional& plan){ + if (!plan){ + return "none"; + } + return std::string(direction_name(plan->direction)) + " " + std::to_string(plan->tiles) + " " + + std::string(leg_end_kind_name(plan->end_kind)) + " -> " + tiles_text(plan->ends); +} + +// The lines a tracer emits for `script`, one a line. +std::string traced(const std::function& script){ + std::string out; + LegTracer tracer([&](const std::string& line){ out += line + "\n"; }); + script(tracer); + return out; +} + +const LegTraceTestCase CASES[] = { + { + .input = {.name = "a tap leg's line", .actual = []{ + LegRecord record{ + .checkpoint = "00.04", .event = "to_the_signs_row", .leg = 1, + .plan = {.direction = Direction::NORTH, .tiles = 1, .end_kind = LegEndKind::TILE, + .ends = {{"Route 201", {57, 4}}}}, + .moved = 1, .chunks = 1, .walked_ms = 800, + .end = {{"Route 201", {57, 4}}}, .known = LegEndKnown::COUNTED, + }; + return leg_trace_line(record); + }}, + .expectation = {.expected = + "[Leg] {\"checkpoint\":\"00.04\",\"chunks\":1,\"dir\":\"north\",\"end\":[[\"Route 201\",57,4]]," + "\"end_kind\":\"tile\",\"event\":\"to_the_signs_row\",\"interrupts\":[],\"known\":\"counted\"," + "\"leg\":1,\"moved\":1,\"ms\":800,\"planned_end\":[[\"Route 201\",57,4]],\"tiles\":1}"}, + }, + { + .input = {.name = "tiles read back", .actual = []{ + return tiles_text(parse_route_tiles("Route 202 (23, 25), Mom's House (30, -1)")); + }}, + .expectation = {.expected = "Route 202 (23, 25), Mom's House (30, -1)"}, + }, + { + .input = {.name = "no tiles in other text", .actual = []{ + return tiles_text(parse_route_tiles("an unknown tile")); + }}, + .expectation = {.expected = "none"}, + }, + { + .input = {.name = "a wall leg line read back", .actual = []{ + return plan_text(parse_route_leg_line("Leg 1/2: east 15 (to wall) to Route 202 (23, 25), Route 202 (30, 26)")); + }}, + .expectation = {.expected = "east 15 wall -> Route 202 (23, 25), Route 202 (30, 26)"}, + }, + { + .input = {.name = "an open leg line read back", .actual = []{ + return plan_text(parse_route_leg_line("Leg 3/3: west 1 (open) to Twinleaf Town (7, 9)")); + }}, + .expectation = {.expected = "west 1 open -> Twinleaf Town (7, 9)"}, + }, + { + .input = {.name = "not a leg line", .actual = []{ + return plan_text(parse_route_leg_line("[Route] Turning west.")); + }}, + .expectation = {.expected = "none"}, + }, + { + .input = {.name = "a leg cut off by the next, and by its event's end", .actual = []{ + return traced([](LegTracer& tracer){ + tracer.begin_event("00.04", "beside_mom"); + tracer.begin_leg({.direction = Direction::WEST, .tiles = 2, .end_kind = LegEndKind::OPEN}); + tracer.begin_leg({.direction = Direction::NORTH, .tiles = 1, .end_kind = LegEndKind::WALL}); + tracer.walked({.chunks = 3, .walked_ms = 640}); + tracer.begin_event("00.04", "mom"); + }); + }}, + .expectation = {.expected = + "[Leg] {\"checkpoint\":\"00.04\",\"chunks\":0,\"dir\":\"west\",\"end\":[],\"end_kind\":\"open\"," + "\"event\":\"beside_mom\",\"interrupts\":[],\"known\":\"unknown\",\"leg\":1,\"ms\":0,\"planned_end\":[],\"tiles\":2}\n" + "[Leg] {\"checkpoint\":\"00.04\",\"chunks\":3,\"dir\":\"north\",\"end\":[],\"end_kind\":\"wall\"," + "\"event\":\"beside_mom\",\"interrupts\":[],\"known\":\"unknown\",\"leg\":2,\"ms\":640,\"planned_end\":[],\"tiles\":1}\n"}, + }, + { + .input = {.name = "a wild battle on the way, two walks to the wall", .actual = []{ + return traced([](LegTracer& tracer){ + tracer.begin_event("00.04b", "to_catch_grass"); + tracer.begin_leg({.direction = Direction::SOUTH, .tiles = 4, .end_kind = LegEndKind::WALL}); + tracer.walked({.chunks = 2, .walked_ms = 500, .wild_battles = 1}); + tracer.walked({.chunks = 2, .walked_ms = 400}); + tracer.end_leg({{"Route 202", {20, 27}}}, LegEndKnown::WALL); + tracer.end_event(); + }); + }}, + .expectation = {.expected = + "[Leg] {\"checkpoint\":\"00.04b\",\"chunks\":4,\"dir\":\"south\",\"end\":[[\"Route 202\",20,27]],\"end_kind\":\"wall\"," + "\"event\":\"to_catch_grass\",\"interrupts\":[\"wild_battle\"],\"known\":\"wall\",\"leg\":1,\"ms\":900,\"planned_end\":[],\"tiles\":4}\n"}, + }, +}; + +} // namespace + + +void add_tests_LegTrace(UnitTestDatabase& database){ + for (const auto& scenario : CASES){ + database.add(scenario); + } +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace_Tests.h b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace_Tests.h new file mode 100644 index 0000000000..a81561fa7f --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegTrace_Tests.h @@ -0,0 +1,23 @@ +/* Leg Trace Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_Pokemon_LegTrace_Tests_H +#define PokemonAutomation_Pokemon_LegTrace_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// Registers leg trace record and tracer unit tests. +void add_tests_LegTrace(UnitTestDatabase& database); + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_RouteLog_Tests.cpp b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_RouteLog_Tests.cpp index 5715211f01..a024bcaa41 100644 --- a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_RouteLog_Tests.cpp +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_RouteLog_Tests.cpp @@ -13,6 +13,7 @@ #include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/TestRunners/UnitTest.h" +#include "Pokemon_LegTrace_Tests.h" #include "Pokemon_RouteLog.h" #include "Pokemon_RouteLog_Tests.h" @@ -198,6 +199,8 @@ void add_tests_RouteLog(UnitTestDatabase& database){ for (const auto& scenario : CASES){ database.add(scenario); } + // The leg records read the route lines back. + add_tests_LegTrace(database); } diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.cpp b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.cpp new file mode 100644 index 0000000000..e01bbe4e07 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.cpp @@ -0,0 +1,54 @@ +/* Known Failing + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Pokemon_KnownFailing.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +KnownFailing::KnownFailing(std::map fixes) + : m_fixes(std::make_move_iterator(fixes.begin()), std::make_move_iterator(fixes.end())) +{} + +UnitTestResult KnownFailing::run(Logger& logger, std::string_view name, const std::function& body) const{ + UnitTestResult result; + try{ + result = body(); + }catch (Exception& e){ + result = std::format("{}: {}", e.name(), e.message()); + } + const auto known = m_fixes.find(name); + if (known == m_fixes.end()){ + return result; + } + if (result.result == UnitTestResult::PASSED){ + return std::format("on KNOWN_FAILING ({}) but passes now: take it off the list.", known->second); + } + logger.log(std::format("Known failing until {}: {}", known->second, result.message)); + return true; +} + +std::vector KnownFailing::stale(const std::set& names) const{ + std::vector out; + for (const auto& [name, fix] : m_fixes){ + if (!names.contains(name)){ + out.push_back(name); + } + } + return out; +} + + + +} // namespace Pokemon + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.h b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.h new file mode 100644 index 0000000000..b6684a4ccb --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.h @@ -0,0 +1,58 @@ +/* Known Failing + * + * From: https://github.com/PokemonAutomation/ + * + * The list of sim tests the program fails today, each with the change + * likely to fix it, shared by the games' fault sweeps and console-failure + * cases. A test on the list passes while it fails, so the suite stays + * green with the failure logged, and fails once it passes, so the entry + * comes off. stale() finds the entries that name no test: a renamed test + * would otherwise drop off the list silently and fail the suite. + * + */ + +#ifndef PokemonAutomation_Pokemon_KnownFailing_H +#define PokemonAutomation_Pokemon_KnownFailing_H + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "Common/Cpp/TestRunners/UnitTest.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// The tests known to fail, by name, with their likely fixes. +class KnownFailing{ +public: + // `fixes`: by test name (or case name, as the game keys it), the + // change likely to fix it. + explicit KnownFailing(std::map fixes); + + // Runs test `name`'s `body`, an Exception it throws being its failure, + // and gives its result as the list sees it: as is if `name` is not + // listed; a pass that logs the failure if it is and fails; a failure + // "on KNOWN_FAILING () but passes now: take it off the list." if + // it is and passes. + UnitTestResult run(Logger& logger, std::string_view name, const std::function& body) const; + + // The listed names not in `names` (the tests that exist), in order. + std::vector stale(const std::set& names) const; + +private: + std::map> m_fixes; +}; + + + +} // namespace Pokemon + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.cpp b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.cpp new file mode 100644 index 0000000000..ff7dfbd7b6 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.cpp @@ -0,0 +1,209 @@ +/* Known Failing Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The known-failing list on fake tests: one on the list passes while it + * fails and fails once it passes, one off the list keeps its result, an + * exception is a failure, and entries naming no test are stale. + * + */ + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "Pokemon_KnownFailing.h" +#include "Pokemon_KnownFailing_Tests.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + +class LinesLogger : public Logger{ +public: + using Logger::log; + virtual void log(const std::string& msg, Color) override{ + lines.push_back(msg); + } + + std::vector lines; +}; + +const KnownFailing& known_failing(){ + static const KnownFailing list({ + {"listed", "the planner's goal kinds"}, + {"listed and gone", "a fix for a test since renamed"}, + }); + return list; +} + +// How the fake test's body ends. +enum class BodyOutcome{ + PASSES, + FAILS, + THROWS, +}; + +struct KnownFailingCase{ + struct Input{ + std::string test; + BodyOutcome body; + }; + struct Expectation{ + bool passes; + std::string message; + std::vector log; + }; + + std::string name; + Input input; + Expectation expectation; +}; + +const KnownFailingCase KNOWN_FAILING_CASES[] = { + { + .name = "a listed test passes while it fails", + .input = {.test = "listed", .body = BodyOutcome::FAILS}, + .expectation = {.passes = true, .message = "", .log = {"Known failing until the planner's goal kinds: stopped at the gate."}}, + }, + { + .name = "a listed test fails once it passes", + .input = {.test = "listed", .body = BodyOutcome::PASSES}, + .expectation = { + .passes = false, + .message = "on KNOWN_FAILING (the planner's goal kinds) but passes now: take it off the list.", + .log = {}, + }, + }, + { + .name = "a listed test's exception is its known failure", + .input = {.test = "listed", .body = BodyOutcome::THROWS}, + .expectation = {.passes = true, .message = "", .log = {"Known failing until the planner's goal kinds: ParseException: no row 99.99"}}, + }, + { + .name = "an unlisted test keeps its failure", + .input = {.test = "unlisted", .body = BodyOutcome::FAILS}, + .expectation = {.passes = false, .message = "stopped at the gate.", .log = {}}, + }, + { + .name = "an unlisted test keeps its pass", + .input = {.test = "unlisted", .body = BodyOutcome::PASSES}, + .expectation = {.passes = true, .message = "", .log = {}}, + }, + { + .name = "an unlisted test's exception is its failure", + .input = {.test = "unlisted", .body = BodyOutcome::THROWS}, + .expectation = {.passes = false, .message = "ParseException: no row 99.99", .log = {}}, + }, +}; + +UnitTestResult fake_body(BodyOutcome outcome){ + switch (outcome){ + case BodyOutcome::PASSES: + return true; + case BodyOutcome::FAILS: + return "stopped at the gate."; + case BodyOutcome::THROWS: + throw ParseException("no row 99.99"); + } + return true; +} + +class Test_KnownFailing : public UnitTest{ +public: + explicit Test_KnownFailing(KnownFailingCase test_case) + : UnitTest(std::format("Pokemon::Navigation - Known failing: {}", test_case.name)) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + LinesLogger logger; + const UnitTestResult result = known_failing().run(logger, m_case.input.test, [this]{ + return fake_body(m_case.input.body); + }); + const KnownFailingCase::Expectation& expected = m_case.expectation; + const bool passed = result.result == UnitTestResult::PASSED; + if (passed != expected.passes){ + return std::format("expected {}, got \"{}\".", expected.passes ? "a pass" : "a failure", result.message); + } + if (result.message != expected.message){ + return std::format("message: expected \"{}\", got \"{}\".", expected.message, result.message); + } + if (logger.lines != expected.log){ + return std::format("log: expected {} lines, got {}: \"{}\".", expected.log.size(), logger.lines.size(), logger.lines.empty() ? "" : logger.lines[0]); + } + return true; + } + +private: + KnownFailingCase m_case; +}; + + +struct StaleCase{ + struct Input{ + std::set tests; + }; + struct Expectation{ + std::vector stale; + }; + + std::string name; + Input input; + Expectation expectation; +}; + +const StaleCase STALE_CASES[] = { + { + .name = "an entry naming no test is stale", + .input = {.tests = {"listed", "unlisted"}}, + .expectation = {.stale = {"listed and gone"}}, + }, + { + .name = "entries naming tests are not", + .input = {.tests = {"listed", "listed and gone"}}, + .expectation = {.stale = {}}, + }, +}; + +class Test_KnownFailingStale : public UnitTest{ +public: + explicit Test_KnownFailingStale(StaleCase test_case) + : UnitTest(std::format("Pokemon::Navigation - Known failing: {}", test_case.name)) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::vector stale = known_failing().stale(m_case.input.tests); + if (stale != m_case.expectation.stale){ + return std::format("expected {} stale entries, got {}.", m_case.expectation.stale.size(), stale.size()); + } + return true; + } + +private: + StaleCase m_case; +}; + +} // namespace + + +void add_tests_KnownFailing(UnitTestDatabase& database){ + for (const KnownFailingCase& item : KNOWN_FAILING_CASES){ + database.add(item); + } + for (const StaleCase& item : STALE_CASES){ + database.add(item); + } +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.h b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.h new file mode 100644 index 0000000000..49b7baa538 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.h @@ -0,0 +1,23 @@ +/* Known Failing Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_Pokemon_KnownFailing_Tests_H +#define PokemonAutomation_Pokemon_KnownFailing_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// Registers the known-failing list tests. +void add_tests_KnownFailing(UnitTestDatabase& database); + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.cpp b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.cpp new file mode 100644 index 0000000000..a56b6e9e25 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.cpp @@ -0,0 +1,63 @@ +/* Sim Seeds + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "Pokemon_SimSeeds.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + +// The mean and worst (largest) of `values`, which has one at least. +SeedStats stats_of(const std::vector& values){ + return { + .mean = std::accumulate(values.begin(), values.end(), 0.) / static_cast(values.size()), + .worst = *std::max_element(values.begin(), values.end()), + }; +} + +} // namespace + + +SeedsResult run_seeds( + Logger& logger, std::string_view label, const std::function& run_one, + uint32_t seeds +){ + SeedsResult out; + out.seeds = seeds; + std::map> passed; + for (uint32_t seed = 1; seed <= seeds; seed++){ + SeedRun run = run_one(seed); + if (run.failure){ + out.failures.push_back({"", std::format("seed {}", seed), std::move(*run.failure)}); + continue; + } + for (const auto& [measure, value] : run.measures){ + passed[measure].push_back(value); + } + } + std::string line = std::format("Seeds {}: passed {}/{}", label, seeds - out.failures.size(), seeds); + for (const auto& [measure, values] : passed){ + const SeedStats stats = stats_of(values); + line += std::format("; {} mean {:.1f}, worst {:.1f}", measure, stats.mean, stats.worst); + out.stats.emplace(measure, stats); + } + logger.log(line); + return out; +} + + + +} // namespace Pokemon + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.h b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.h new file mode 100644 index 0000000000..2fc7088181 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.h @@ -0,0 +1,71 @@ +/* Sim Seeds + * + * From: https://github.com/PokemonAutomation/ + * + * The games' Monte Carlo rows, shared by BDSP's and FRLG's: a seeded + * row (wild battles at the game's odds, wandering people) runs once per + * seed, each seed feeding the sim's std::mt19937. Its failures are a + * fault sweep's (sweep_result()), a variant per seed, and it logs the + * mean and worst of what each seed measured. + * + */ + +#ifndef PokemonAutomation_Pokemon_SimSeeds_H +#define PokemonAutomation_Pokemon_SimSeeds_H + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "Pokemon_SimSweep.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// How many seeds a seeded row runs unless it asks for others. +inline constexpr uint32_t DEFAULT_SWEEP_SEEDS = 30; + +// One seed's run of a row: why it failed, or nothing; and what it measured +// by name ("s", "stops", ...), only what applies to the seed. +struct SeedRun{ + std::optional failure = std::nullopt; + std::map measures = {}; +}; + +// A measure over the seeds that passed and measured it. +struct SeedStats{ + double mean = 0; + double worst = 0; +}; + +// The seeds a row ran, those that failed, and what the rest measured. +struct SeedsResult{ + uint32_t seeds = 0; + // A failure per failed seed, its variant "seed N". + std::vector failures = {}; + // By measure, over the passed seeds; a measure no passed seed has is + // left out. + std::map stats = {}; +}; + +// Runs `run_one` from seeds 1 to `seeds`; logs "Seeds