diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEngine.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEngine.cpp index b874795b8a..c1acee52b6 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEngine.cpp +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEngine.cpp @@ -11,6 +11,7 @@ #include #include "Pokemon/AutoStory/Pokemon_CheckpointAttempts.h" #include "Pokemon/AutoStory/Pokemon_StoryEvents.h" +#include "Pokemon/AutoStory/Pokemon_StoryOutcomeClass.h" #include "Pokemon_StoryEngine.h" namespace PokemonAutomation{ @@ -289,7 +290,7 @@ SegmentReport StoryEngine::run_segment( report.failed_checkpoint = checkpoint.id; report.failed_event = result.run.failed_name; report.failure = run_failure(result); - m_backend.log(std::format("[Segment] {}: failed at {}: {}", id, checkpoint.id, to_string(report.failure))); + m_backend.log(std::format("[Segment] {}: failed at {}: {}", id, checkpoint.id, to_string_with_class(report.failure))); return report; } start_saved = result.saved; @@ -329,7 +330,7 @@ VerifyReport StoryEngine::verify_checkpoint(const StoryCheckpoint& che report.saved = result.saved; m_backend.log(std::format( "[Verify] {}: run {}/{} {}", checkpoint.id, run, policy.runs, - passed ? std::string("passed") : std::format("failed: {}", to_string(run_failure(result))) + passed ? std::string("passed") : std::format("failed: {}", to_string_with_class(run_failure(result))) )); if (repeats_on_reload(result.run.failure)){ report.failure = result.run.failure; diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEngine_Tests.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEngine_Tests.cpp index e06c533cc9..a61824e09d 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEngine_Tests.cpp +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEngine_Tests.cpp @@ -242,7 +242,7 @@ const RunnerCase RUNNER_CASES[] = { {.name = "declared end pose", .input = {.step = R"({"id": "s", "kind": "scene", "declared_after": {"map": "Route1", "tiles": [[1, 2]]}})"}, .expectation = {.line = "[Step] s MASH/default: ok", .pose_map = "Route1"}}, {.name = "talk to nobody fails", .input = {.step = R"({"id": "s", "kind": "talk"})"}, - .expectation = {.failure = "talk_no_text", .line = "[Step] s TALK/default: failed: talk_no_text: A opened no text box"}}, + .expectation = {.failure = "talk_no_text", .line = "[Step] s TALK/default: failed: talk_no_text: A opened no text box (class=screen_unread)"}}, }; std::string check_runner(const RunnerCase& test, CancellableScope& scope){ diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryStepRunner.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryStepRunner.cpp index c372203458..300ec4f33c 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryStepRunner.cpp +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryStepRunner.cpp @@ -6,6 +6,7 @@ #include #include +#include "Pokemon/AutoStory/Pokemon_StoryOutcomeClass.h" #include "Pokemon/AutoStory/Pokemon_StoryStepExecutor.h" #include "Pokemon_StoryStepRunner.h" @@ -53,7 +54,7 @@ Outcome StoryStepRunner::act(const StoryStep& step, RunState& state){ } m_backend.log(std::format( "[Step] {} {}: {}", step.name, step_label(step), - outcome.succeeded() ? "ok" : std::format("failed: {}", to_string(outcome)) + outcome.succeeded() ? "ok" : std::format("failed: {}", to_string_with_class(outcome)) )); return outcome; } diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h index 9d9a1e62d3..b45fc3e1aa 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h @@ -25,6 +25,8 @@ #include "Pokemon_SceneJudges_Tests.h" #include "Pokemon_StoryActions_Tests.h" #include "Pokemon_StoryEvents_Tests.h" +#include "Pokemon_StoryFailureClasses_Tests.h" +#include "Pokemon_StoryFaultSweep_Tests.h" #include "Pokemon_StoryInterrupts_Tests.h" #include "Pokemon_StoryLog_Tests.h" #include "Pokemon_StoryRngHandoff_Tests.h" @@ -47,6 +49,8 @@ inline void add_tests_AutoStory(UnitTestDatabase& database){ add_tests_StoryRunCounts(database); add_tests_CheckpointRetry(database); add_tests_StoryLog(database); + add_tests_StoryFailureClasses(database); + add_tests_StoryFaultSweep(database); add_tests_StoryRngHandoff(database); add_tests_StoryRangeOptions(database); add_tests_StoryEvents(database); diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryEvents.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryEvents.cpp index c2cb6142df..1dd6975bff 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryEvents.cpp +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryEvents.cpp @@ -8,7 +8,9 @@ #include #include #include "Common/Cpp/Exceptions.h" +#include "CommonFramework/Exceptions/OperationFailedException.h" #include "Pokemon_StoryEvents.h" +#include "Pokemon_StoryOutcomeClass.h" namespace PokemonAutomation{ namespace Pokemon{ @@ -80,10 +82,11 @@ Outcome outcome_unchanged(){ return outcome; } -Outcome outcome_failed(std::string failure, std::string detail){ +Outcome outcome_failed(std::string failure, std::string detail, std::optional failure_class){ Outcome outcome; outcome.failure = std::move(failure); outcome.detail = std::move(detail); + outcome.failure_class = failure_class; return outcome; } std::string to_string(const Outcome& outcome){ @@ -350,16 +353,19 @@ Outcome run_event(EventRun& run){ } outcome = std::move(*stop); } - log_status(std::format("failed: {}", to_string(outcome)), run); + log_status(std::format("failed: {}", to_string_with_class(outcome)), run); return outcome; } } -// What the exception being handled says, for a log line. +// What the exception being handled says, for a log line, with its class +// when the engine knows it. std::string current_exception_text(){ try{ throw; + }catch (OperationFailedException& e){ + return std::format("{}: {} ({})", e.name(), e.message(), story_failure_class_tag(StoryFailureClass::OPERATION_FAILED)); }catch (Exception& e){ return std::format("{}: {}", e.name(), e.message()); }catch (std::exception& e){ @@ -387,7 +393,7 @@ CheckpointReport stop_checkpoint(EventLogger& log, const std::string& id, Outcom std::string where = report.failed_event == 0 ? "" : std::format(" at {} {}", report.failed_event, report.failed_name); - log(checkpoint_log_line(id, std::format("failed{}: {}", where, to_string(failure)))); + log(checkpoint_log_line(id, std::format("failed{}: {}", where, to_string_with_class(failure)))); report.failure = std::move(failure); return report; } diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryEvents_Tests.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryEvents_Tests.cpp index cafb66cd54..78b3d5796c 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryEvents_Tests.cpp +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryEvents_Tests.cpp @@ -15,8 +15,10 @@ #include #include #include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/Exceptions/OperationFailedException.h" #include "Pokemon_StoryEvents.h" #include "Pokemon_StoryEvents_Tests.h" +#include "Pokemon_StoryOutcomeClass.h" #include "Pokemon_TilePose.h" namespace PokemonAutomation{ @@ -248,7 +250,8 @@ UnitTestResult test_stop_names_failed_event(){ return "Nothing after the failed event should run."; } if (!lines.has("[Event] 00.04/2 talk_mom: recover: stop (expect_failed): four rows cover the mat to the TV") - || lines.lines.back() != "[Checkpoint] 00.04: failed at 2 talk_mom: expect_failed: dialogue opened" + || !lines.has("[Event] 00.04/2 talk_mom: failed: expect_failed: dialogue opened (class=state_check_failed)") + || lines.lines.back() != "[Checkpoint] 00.04: failed at 2 talk_mom: expect_failed: dialogue opened (class=state_check_failed)" ){ return std::format("Unexpected log:{}", lines.all()); } @@ -276,6 +279,49 @@ UnitTestResult test_unmatched_failure_stops(){ return true; } +// A game's own failure name with the class the game gives it. +UnitTestResult test_failure_names_its_class(){ + Checkpoint checkpoint = make_checkpoint(); + checkpoint.events.push_back(make_event("probe_for_mom", [](RunState&){ + return outcome_failed("probe_exhausted", "4 rows", StoryFailureClass::POSE_LOST); + })); + RunState state; + Lines lines; + EventLogger log = lines.logger(); + const CheckpointReport report = run_checkpoint(state, checkpoint, log); + if (failure_class_of(report.failure) != StoryFailureClass::POSE_LOST){ + return std::format("Expected the report's failure of class pose_lost:{}", lines.all()); + } + if (!lines.has("[Event] 00.04/1 probe_for_mom: failed: probe_exhausted: 4 rows (class=pose_lost)") + || lines.lines.back() != "[Checkpoint] 00.04: failed at 1 probe_for_mom: probe_exhausted: 4 rows (class=pose_lost)" + ){ + return std::format("Unexpected log:{}", lines.all()); + } + return true; +} + +// A game's action that throws OperationFailedException: the exception goes +// on, the log names its class. +UnitTestResult test_throw_names_its_class(){ + Checkpoint checkpoint = make_checkpoint(); + checkpoint.events.push_back(make_event("talk_mom", [](RunState&) -> Outcome{ + throw OperationFailedException(ErrorReportMode::NO_ERROR_REPORT, "no dialog"); + })); + RunState state; + Lines lines; + EventLogger log = lines.logger(); + try{ + run_checkpoint(state, checkpoint, log); + return "Expected the exception to go on."; + }catch (OperationFailedException&){} + if (!lines.has("[Event] 00.04/1 talk_mom: failed: OperationFailedException: no dialog (class=operation_failed)") + || lines.lines.back() != "[Checkpoint] 00.04: failed at 1 talk_mom: OperationFailedException: no dialog (class=operation_failed)" + ){ + return std::format("Unexpected log:{}", lines.all()); + } + return true; +} + Interrupt wild_battle(ResumePolicy resume, World& world){ Interrupt interrupt; interrupt.name = "wild_battle"; @@ -664,6 +710,14 @@ const StoryEventTestCase STORY_EVENT_CASES[] = { .input = {.name = "an unanswered failure stops", .test = test_unmatched_failure_stops}, .expectation = {.description = "stops checkpoint execution when failure has no handler"}, }, + { + .input = {.name = "a failure names its class", .test = test_failure_names_its_class}, + .expectation = {.description = "logs the class a game gives its own failure name"}, + }, + { + .input = {.name = "a throw names its class", .test = test_throw_names_its_class}, + .expectation = {.description = "logs operation_failed for an OperationFailedException that goes on"}, + }, { .input = {.name = "an interrupt mid-event", .test = test_interrupt_mid_event}, .expectation = {.description = "handles asynchronous interruption during event execution"}, diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFailureClasses.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFailureClasses.cpp new file mode 100644 index 0000000000..ec7b5f582d --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFailureClasses.cpp @@ -0,0 +1,257 @@ +/* Story Failure Classes + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "Pokemon_StoryFailureClasses.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + +constexpr std::string_view TAG_PREFIX = "class="; + +using C = StoryFailureClass; +using R = StoryFailureClassInfo::Retry; + +const std::vector CLASSES{ + // The run and the console. + { + C::GAME_NOT_RUNNING, "game_not_running", + "the Switch HOME menu shows instead of the game", + "the HOME menu read at a step's start or after a failure", + {}, {"game_not_running"}, R::STOP, + }, + { + C::LAUNCH_FAILED, "launch_failed", + "getting from HOME to the game's overworld failed", + "the launch driver: no game tile, a screen that ignores its press, no overworld in time", + {}, {"launch_failed"}, R::STOP, + }, + { + C::WRONG_START, "wrong_start", + "the game is not at the checkpoint's start: another save, an older one, another place", + "the checkpoint's start checks and the preflight", + {}, {"start_failed", "wrong_save", "precondition_failed"}, R::STOP, + }, + { + C::RELOAD_FAILED, "reload_failed", + "reloading the checkpoint's save failed", + "the game's reload hook", + {}, {"reload_failed"}, R::STOP, + }, + { + C::SAVE_FAILED, "save_failed", + "saving at the checkpoint's end failed", + "the game's save hook", + {}, {"save_failed"}, R::RELOAD, + }, + { + C::STORY_ERROR, "story_error", + "the story's data or code is wrong: a step, range or action the program cannot run", + "the engine's lint and lookups, before or as a step starts", + {}, + { + "unsupported_step", "unknown_checkpoint", "bad_range", "empty_segment", + "bad_verify_policy", "lint_failed", "no_save_hook", "console_only", + }, + R::STOP, + }, + + // Walking. + { + C::LOST_TAP, "lost_tap", + "a tap the game drops", + "the walker reads the player where the tap started", + {FaultClass::LOST_TAP}, {}, R::RETRY_STEP, + }, + { + C::UNSEEN_STEP, "unseen_step", + "a step tap that moved the player, read as not moved", + "the walker finds the player a tile past where it read", + {FaultClass::UNSEEN_STEP}, {}, R::RETRY_STEP, + }, + { + C::WALK_SHORT, "walk_short", + "a walk or hold ended short of its tile", + "the walker's position check at a wall or landmark", + {}, {}, R::RETRY_STEP, + }, + { + C::BLOCKED, "blocked", + "something on the way stops the walk: a person, a wall the map does not have", + "the walker's progress check: no tile gained", + {}, {}, R::RETRY_STEP, + }, + { + C::MOUNT_LOST, "mount_lost", + "off the Bicycle (or another mount) at a leg that rides", + "the walker's tiles per tap", + {FaultClass::MOUNT_LOST}, {}, R::RETRY_STEP, + }, + { + C::POSE_LOST, "pose_lost", + "where the player is or faces is unknown or not what the map says", + "the localizer: no pose, another map, another facing", + {}, {}, R::RELOAD, + }, + + // Warps and fades. + { + C::WARP_NO_OP, "warp_no_op", + "a step onto a warp shows the fade's black but does not warp", + "the map or tile after the fade is the one before it", + {FaultClass::WARP_NO_OP}, {}, R::RETRY_STEP, + }, + { + C::LATE_FADE, "late_fade", + "a warp's fade shows late or is missed", + "the fade watcher: the warp unconfirmed, the player on a map the walk did not plan", + {FaultClass::LATE_FADE}, {}, R::RETRY_STEP, + }, + { + C::SLOW_FADE_IN, "slow_fade_in", + "a warp's fade-in runs long: a move right after it goes nowhere", + "the first move after a fade does not move the player", + {FaultClass::SLOW_FADE_IN}, {}, R::RETRY_STEP, + }, + { + C::DARK_MAP, "dark_map", + "a dark map (a cave without Flash) the readers cannot see", + "the floor reads dim; the walker's and the scene's reads fail there", + {FaultClass::DARK_CAVE}, {}, R::STOP, + }, + + // Scenes, talks and battles. + { + C::SCENE_END_POSE, "scene_end_pose", + "a step whose end the game sets leaves the player on a tile next to the expected one", + "the next walk starts or ends a tile off", + {FaultClass::SCENE_END_POSE}, {}, R::RETRY_STEP, + }, + { + C::SCENE_EARLY, "scene_early", + "a scene starts on the last tiles of the walk before it", + "the walk is interrupted by the scene's dialogue", + {FaultClass::SCENE_EARLY}, {}, R::RETRY_STEP, + }, + { + C::BATTLE_OPENING, "battle_opening", + "a battle's opening is still playing when a step starts", + "the step's first read is not the overworld or battle menu it expects", + {FaultClass::BATTLE_OPENING}, {}, R::RETRY_STEP, + }, + { + C::WILD_BATTLES, "wild_battles", + "more wild battles than a clean run meets", + "the interrupt handler clears each; too many in a step fail it", + {FaultClass::WILD_BATTLES}, {"too_many_interrupts"}, R::RETRY_STEP, + }, + { + C::INTERRUPT_FAILED, "interrupt_failed", + "something interrupted a step and was not cleared", + "the interrupt handler: no handler, a failed clear, the screen not settled", + {}, {"unhandled_interrupt", "interrupt_failed", "interrupt_unsettled", "interrupted"}, R::RELOAD, + }, + { + C::SCREEN_UNREAD, "screen_unread", + "a screen the step waits for (dialogue, banner, overworld, prompt) is not seen", + "the step's detector times out", + {}, {"talk_no_text", "mash_no_overworld"}, R::RELOAD, + }, + { + C::MENU_FAILED, "menu_failed", + "a menu action (mart, PC, item, TM, Fly) did not do what it was to", + "the menu's readers: the item, count or choice not as planned", + {}, {}, R::RELOAD, + }, + { + C::BATTLE_LOST, "battle_lost", + "a battle lost or not won: a blackout", + "the battle's end screen or the blackout's heal spot", + {}, {}, R::RELOAD, + }, + { + C::STATE_CHECK_FAILED, "state_check_failed", + "a check of the game's state failed: an event's expects, the checkpoint's end", + "the event's expects and requires, the checkpoint's end and postconditions", + {}, {"expect_failed", "require_failed", "end_failed", "postconditions_failed"}, R::RELOAD, + }, + { + C::OPERATION_FAILED, "operation_failed", + "a game's action threw, with no class of its own", + "an OperationFailedException out of a step", + {}, {"operation_failed"}, R::RELOAD, + }, + { + C::SIM_MISMATCH, "sim_mismatch", + "a simulated run left the story: the sim and the table disagree", + "the story sim's end checks (sim runs only)", + {}, {}, R::STOP, + }, +}; + +// The class whose id is `id`; nothing for an unknown id. +std::optional find_story_failure_class(std::string_view id){ + for (const StoryFailureClassInfo& info : CLASSES){ + if (info.id == id){ + return info.failure_class; + } + } + return std::nullopt; +} + +} // namespace + + +const std::vector& story_failure_classes(){ + return CLASSES; +} + +const StoryFailureClassInfo& story_failure_class_info(StoryFailureClass failure_class){ + return CLASSES[static_cast(failure_class)]; +} + +std::optional story_failure_class_of(FaultClass fault){ + for (const StoryFailureClassInfo& info : CLASSES){ + if (std::ranges::find(info.injected_by, fault) != info.injected_by.end()){ + return info.failure_class; + } + } + return std::nullopt; +} + +std::optional story_failure_class_of_outcome(std::string_view failure){ + for (const StoryFailureClassInfo& info : CLASSES){ + if (std::ranges::find(info.outcomes, failure) != info.outcomes.end()){ + return info.failure_class; + } + } + return std::nullopt; +} + + +std::string story_failure_class_tag(StoryFailureClass failure_class){ + return std::format("{}{}", TAG_PREFIX, story_failure_class_info(failure_class).id); +} + +std::optional read_story_failure_class_tag(std::string_view text){ + const size_t start = text.find(TAG_PREFIX); + if (start == std::string_view::npos){ + return std::nullopt; + } + const std::string_view rest = text.substr(start + TAG_PREFIX.size()); + const size_t end = rest.find_first_not_of("abcdefghijklmnopqrstuvwxyz_"); + return find_story_failure_class(rest.substr(0, end)); +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFailureClasses.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFailureClasses.h new file mode 100644 index 0000000000..4cf5f9003d --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFailureClasses.h @@ -0,0 +1,113 @@ +/* Story Failure Classes + * + * From: https://github.com/PokemonAutomation/ + * + * One list of the ways an AutoStory step fails, for every game: what the + * run log and run records name a failure by, and what the fault sweeps + * inject. Each class is a row of data: its id, what it means, how a run + * sees it, which sim faults inject it, and what a retry should do. + * + * An Outcome's failure name ("expect_failed") is finer than a class: a + * class covers the names that mean the same to a retry. The engine's names + * are mapped here; a game's own names are mapped when it adopts the list. + * + */ + +#ifndef PokemonAutomation_Pokemon_StoryFailureClasses_H +#define PokemonAutomation_Pokemon_StoryFailureClasses_H + +#include +#include +#include +#include +#include "Pokemon/Navigation/Simulation/Pokemon_SimFaults.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// A class of story failure, in story_failure_classes()'s order. +enum class StoryFailureClass{ + GAME_NOT_RUNNING, + LAUNCH_FAILED, + WRONG_START, + RELOAD_FAILED, + SAVE_FAILED, + STORY_ERROR, + LOST_TAP, + UNSEEN_STEP, + WALK_SHORT, + BLOCKED, + MOUNT_LOST, + POSE_LOST, + WARP_NO_OP, + LATE_FADE, + SLOW_FADE_IN, + DARK_MAP, + SCENE_END_POSE, + SCENE_EARLY, + BATTLE_OPENING, + WILD_BATTLES, + INTERRUPT_FAILED, + SCREEN_UNREAD, + MENU_FAILED, + BATTLE_LOST, + STATE_CHECK_FAILED, + OPERATION_FAILED, + SIM_MISMATCH, +}; + +// One class of failure. +struct StoryFailureClassInfo{ + // What a retry should do after a failure of the class: a hint, the + // program decides. + enum class Retry{ + // Find where the player is and run the step again. + RETRY_STEP, + // Reload the checkpoint's save and run it again. + RELOAD, + // Stop the run: no retry fixes it, a person or a code change must. + STOP, + }; + + StoryFailureClass failure_class; + // The class in logs and run records: "warp_no_op". + std::string_view id; + // What went wrong, in one line. + std::string_view meaning; + // How a run sees it. + std::string_view detected_by; + // The sim faults that inject it; none when no sim can yet. + std::vector injected_by; + // The engine's Outcome failure names of this class. + std::vector outcomes; + Retry retry; +}; + +// Every class, in the enum's order. +const std::vector& story_failure_classes(); + +// The row of `failure_class`. +const StoryFailureClassInfo& story_failure_class_info(StoryFailureClass failure_class); + +// The class a sim fault injects. Every FaultClass has one (the tests +// check); nothing only for one added since without a row. +std::optional story_failure_class_of(FaultClass fault); + +// The class of an engine Outcome failure name ("expect_failed"); nothing +// for a name no class lists yet, like a game's own. +std::optional story_failure_class_of_outcome(std::string_view failure); + + +// The class as a log line or run record writes it: "class=warp_no_op". +std::string story_failure_class_tag(StoryFailureClass failure_class); + +// The class tagged in `text`, a log line or run record field that holds a +// story_failure_class_tag(); nothing when it holds none or an unknown id. +std::optional read_story_failure_class_tag(std::string_view text); + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFailureClasses_Tests.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFailureClasses_Tests.cpp new file mode 100644 index 0000000000..df17c2aefd --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFailureClasses_Tests.cpp @@ -0,0 +1,262 @@ +/* Story Failure Classes Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The list's own checks: every sim fault and every engine failure name + * maps to one class, and every class id reads back from the text a log + * line or run record writes. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "Pokemon_StoryFailureClasses.h" +#include "Pokemon_StoryFailureClasses_Tests.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + +// The class id of `failure_class`, or "none" for nothing: for messages. +std::string_view id_or_none(std::optional failure_class){ + return failure_class ? story_failure_class_info(*failure_class).id : "none"; +} + + +// An engine Outcome failure name and the class it maps to. +struct OutcomeClassTestCase{ + struct Input{ + std::string_view failure; + }; + + struct Expectation{ + std::optional failure_class; + }; + + std::string name; + Input input; + Expectation expectation; +}; + +class Test_OutcomeClass : public UnitTest{ +public: + explicit Test_OutcomeClass(OutcomeClassTestCase test_case) + : UnitTest(std::format("Pokemon::StoryFailureClasses - outcome: {}", test_case.name)) + , m_case(std::move(test_case)) + {} + + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::optional got = story_failure_class_of_outcome(m_case.input.failure); + if (got != m_case.expectation.failure_class){ + return std::format( + "\"{}\": expected {}, got {}", + m_case.input.failure, id_or_none(m_case.expectation.failure_class), id_or_none(got) + ); + } + return true; + } + +private: + const OutcomeClassTestCase m_case; +}; + +using C = StoryFailureClass; + +// Every failure name the engine's code returns (autostory/engine), and +// #477's launch_failed. +const OutcomeClassTestCase OUTCOME_CASES[] = { + {.name = "HOME menu", .input = {"game_not_running"}, .expectation = {C::GAME_NOT_RUNNING}}, + {.name = "launch", .input = {"launch_failed"}, .expectation = {C::LAUNCH_FAILED}}, + {.name = "start checks", .input = {"start_failed"}, .expectation = {C::WRONG_START}}, + {.name = "another save", .input = {"wrong_save"}, .expectation = {C::WRONG_START}}, + {.name = "preflight", .input = {"precondition_failed"}, .expectation = {C::WRONG_START}}, + {.name = "reload", .input = {"reload_failed"}, .expectation = {C::RELOAD_FAILED}}, + {.name = "save", .input = {"save_failed"}, .expectation = {C::SAVE_FAILED}}, + {.name = "unsupported step", .input = {"unsupported_step"}, .expectation = {C::STORY_ERROR}}, + {.name = "unknown checkpoint", .input = {"unknown_checkpoint"}, .expectation = {C::STORY_ERROR}}, + {.name = "bad range", .input = {"bad_range"}, .expectation = {C::STORY_ERROR}}, + {.name = "empty segment", .input = {"empty_segment"}, .expectation = {C::STORY_ERROR}}, + {.name = "bad verify policy", .input = {"bad_verify_policy"}, .expectation = {C::STORY_ERROR}}, + {.name = "lint", .input = {"lint_failed"}, .expectation = {C::STORY_ERROR}}, + {.name = "no save hook", .input = {"no_save_hook"}, .expectation = {C::STORY_ERROR}}, + {.name = "console only", .input = {"console_only"}, .expectation = {C::STORY_ERROR}}, + {.name = "too many interrupts", .input = {"too_many_interrupts"}, .expectation = {C::WILD_BATTLES}}, + {.name = "unhandled interrupt", .input = {"unhandled_interrupt"}, .expectation = {C::INTERRUPT_FAILED}}, + {.name = "interrupt not cleared", .input = {"interrupt_failed"}, .expectation = {C::INTERRUPT_FAILED}}, + {.name = "interrupt unsettled", .input = {"interrupt_unsettled"}, .expectation = {C::INTERRUPT_FAILED}}, + {.name = "interrupted", .input = {"interrupted"}, .expectation = {C::INTERRUPT_FAILED}}, + {.name = "talk without text", .input = {"talk_no_text"}, .expectation = {C::SCREEN_UNREAD}}, + {.name = "mash without overworld", .input = {"mash_no_overworld"}, .expectation = {C::SCREEN_UNREAD}}, + {.name = "expect", .input = {"expect_failed"}, .expectation = {C::STATE_CHECK_FAILED}}, + {.name = "require", .input = {"require_failed"}, .expectation = {C::STATE_CHECK_FAILED}}, + {.name = "checkpoint end", .input = {"end_failed"}, .expectation = {C::STATE_CHECK_FAILED}}, + {.name = "postconditions", .input = {"postconditions_failed"}, .expectation = {C::STATE_CHECK_FAILED}}, + {.name = "operation", .input = {"operation_failed"}, .expectation = {C::OPERATION_FAILED}}, + {.name = "a game's own name", .input = {"no_banner"}, .expectation = {std::nullopt}}, + {.name = "empty", .input = {""}, .expectation = {std::nullopt}}, +}; + + +// A check over the whole list. +struct ListCheckTestCase{ + // The check, run over story_failure_classes(). + struct Input{ + UnitTestResult (*check)(); + }; + + // What holds when the check passes. + struct Expectation{ + std::string_view holds; + }; + + std::string name; + Input input; + Expectation expectation; +}; + +class Test_ListCheck : public UnitTest{ +public: + explicit Test_ListCheck(ListCheckTestCase test_case) + : UnitTest(std::format("Pokemon::StoryFailureClasses - {}", test_case.name)) + , m_case(std::move(test_case)) + {} + + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return m_case.input.check(); + } + +private: + const ListCheckTestCase m_case; +}; + +UnitTestResult rows_in_enum_order(){ + const std::vector& classes = story_failure_classes(); + for (size_t c = 0; c < classes.size(); c++){ + if (static_cast(classes[c].failure_class) != c){ + return std::format("row {} ({}) is out of the enum's order", c, classes[c].id); + } + } + return true; +} + +UnitTestResult every_sim_fault_maps_to_one_class(){ + for (FaultClass fault : fault_classes()){ + size_t rows = 0; + for (const StoryFailureClassInfo& info : story_failure_classes()){ + rows += std::ranges::count(info.injected_by, fault); + } + if (rows != 1){ + return std::format("sim fault \"{}\" is in {} rows, not 1", fault_class_name(fault), rows); + } + if (!story_failure_class_of(fault)){ + return std::format("sim fault \"{}\" maps to no class", fault_class_name(fault)); + } + } + return true; +} + +UnitTestResult no_outcome_in_two_classes(){ + std::map seen; + for (const StoryFailureClassInfo& info : story_failure_classes()){ + for (std::string_view failure : info.outcomes){ + const auto [iter, added] = seen.emplace(failure, info.id); + if (!added){ + return std::format("\"{}\" is in {} and {}", failure, iter->second, info.id); + } + } + } + return true; +} + +UnitTestResult ids_round_trip(){ + std::set ids; + for (const StoryFailureClassInfo& info : story_failure_classes()){ + if (!ids.insert(info.id).second){ + return std::format("id \"{}\" is in two rows", info.id); + } + const std::optional got = read_story_failure_class_tag(story_failure_class_tag(info.failure_class)); + if (got != info.failure_class){ + return std::format("id \"{}\" reads back as {}", info.id, id_or_none(got)); + } + } + return true; +} + +UnitTestResult tags_round_trip_through_a_log_line(){ + for (const StoryFailureClassInfo& info : story_failure_classes()){ + const std::string line = std::format( + "[Checkpoint] 00.04: failed at 2 talk_mom: expect_failed: has_parcel ({})", + story_failure_class_tag(info.failure_class) + ); + const std::optional got = read_story_failure_class_tag(line); + if (got != info.failure_class){ + return std::format("\"{}\" reads back as {}", line, id_or_none(got)); + } + } + return true; +} + +UnitTestResult tags_round_trip_through_a_run_record(){ + for (const StoryFailureClassInfo& info : story_failure_classes()){ + const std::string field = std::format("\"failure\": \"{}\"", story_failure_class_tag(info.failure_class)); + const std::optional got = read_story_failure_class_tag(field); + if (got != info.failure_class){ + return std::format("{} reads back as {}", field, id_or_none(got)); + } + } + return true; +} + +UnitTestResult no_class_without_a_tag(){ + const std::string_view lines[] = { + "[Checkpoint] 00.04: failed at 2 talk_mom: expect_failed: has_parcel", + "[Checkpoint] 00.04: failed (class=no_such_class)", + "class=", + }; + for (std::string_view line : lines){ + const std::optional got = read_story_failure_class_tag(line); + if (got){ + return std::format("\"{}\" reads as {}", line, id_or_none(got)); + } + } + return true; +} + +const ListCheckTestCase LIST_CHECKS[] = { + {.name = "rows in the enum's order", .input = {rows_in_enum_order}, .expectation = {"story_failure_class_info() indexes by the enum"}}, + {.name = "every sim fault maps to one class", .input = {every_sim_fault_maps_to_one_class}, .expectation = {"the sweeps' faults all have a class"}}, + {.name = "no outcome name in two classes", .input = {no_outcome_in_two_classes}, .expectation = {"a failure name has one class"}}, + {.name = "ids round-trip", .input = {ids_round_trip}, .expectation = {"each id is unique and its tag alone reads back"}}, + {.name = "tags round-trip through a log line", .input = {tags_round_trip_through_a_log_line}, .expectation = {"the run log names the class"}}, + {.name = "tags round-trip through a run record", .input = {tags_round_trip_through_a_run_record}, .expectation = {"a run record names the class"}}, + {.name = "no class without a tag", .input = {no_class_without_a_tag}, .expectation = {"untagged or unknown text has no class"}}, +}; + +} // namespace + + +void add_tests_StoryFailureClasses(UnitTestDatabase& database){ + for (const OutcomeClassTestCase& test_case : OUTCOME_CASES){ + database.add(test_case); + } + for (const ListCheckTestCase& test_case : LIST_CHECKS){ + database.add(test_case); + } +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFailureClasses_Tests.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFailureClasses_Tests.h new file mode 100644 index 0000000000..c14ff5bca4 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFailureClasses_Tests.h @@ -0,0 +1,23 @@ +/* Story Failure Classes Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_Pokemon_StoryFailureClasses_Tests_H +#define PokemonAutomation_Pokemon_StoryFailureClasses_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// Registers unit tests for the shared story failure classes. +void add_tests_StoryFailureClasses(UnitTestDatabase& database); + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFaultSweep.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFaultSweep.cpp new file mode 100644 index 0000000000..3ee4d68bc0 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFaultSweep.cpp @@ -0,0 +1,159 @@ +/* Story Fault Sweep + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "Pokemon_StoryFaultSweep.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + +// Whether `cell` passes with `result`: logs a table row per failing +// variant, with the class's tag. +UnitTestResult judge_cell(Logger& logger, const FaultSweepCell& cell, const FaultSweepResult& result){ + const std::string tag = story_failure_class_tag(cell.failure_class); + if (result.not_swept){ + logger.log(std::format("Fault sweep {} ({}): not swept, {}", cell.row, tag, *result.not_swept)); + return true; + } + logger.log(std::format("Fault sweep {} ({}): {} variants, {} fail.", cell.row, tag, result.variants, result.failures.size())); + if (result.failures.empty()){ + return true; + } + std::string message = std::format("{} of {} variants fail {} ({}):", result.failures.size(), result.variants, cell.row, tag); + for (const FaultSweepResult::Failure& failure : result.failures){ + logger.log(std::format("| {} | {} | {} | {} | {} |", cell.row, failure.step, tag, failure.variant, failure.stopped)); + message += std::format("\n {} {}: {}", failure.step, failure.variant, failure.stopped); + } + return message; +} + +// `result` turned over when `name` is on the known-failing list. +UnitTestResult judge_known_failing(Logger& logger, const StoryFaultSweep& sweep, const std::string& name, UnitTestResult result){ + const auto known = sweep.known_failing.find(name); + if (known == sweep.known_failing.end()){ + return result; + } + if (result.result == UnitTestResult::PASSED){ + return std::format("on KNOWN_FAILING ({}) but passes now: take it off the list.", known->second); + } + logger.log(std::format("Known failing until {}: {}", known->second, result.message)); + return true; +} + +class Test_FaultSweepCell : public UnitTest{ +public: + Test_FaultSweepCell(std::shared_ptr sweep, FaultSweepCell cell) + : UnitTest(fault_sweep_test_name(sweep->test_prefix, cell)) + , m_sweep(std::move(sweep)) + , m_cell(std::move(cell)) + {} + + virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ + UnitTestResult result; + try{ + const FaultSweepResult swept = m_sweep->injectors.at(m_cell.fault)(logger, m_cell.row); + result = judge_cell(logger, m_cell, swept); + }catch (Exception& e){ + result = std::format("{}: {}", e.name(), e.message()); + } + return judge_known_failing(logger, *m_sweep, name(), std::move(result)); + } + +private: + const std::shared_ptr m_sweep; + const FaultSweepCell m_cell; +}; + +class Test_KnownFailingNames : public UnitTest{ +public: + explicit Test_KnownFailingNames(std::shared_ptr sweep) + : UnitTest(std::format("{} - Fault sweep: known failing names its tests", sweep->test_prefix)) + , m_sweep(std::move(sweep)) + {} + + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::vector stale = known_failing_without_a_test(*m_sweep); + if (stale.empty()){ + return true; + } + std::string message = "KNOWN_FAILING names no test (renamed?):"; + for (const std::string& name : stale){ + message += std::format("\n {}", name); + } + return message; + } + +private: + const std::shared_ptr m_sweep; +}; + +} // namespace + + +std::vector fault_sweep_cells(const StoryFaultSweep& sweep){ + std::vector cells; + for (const StoryFailureClassInfo& info : story_failure_classes()){ + for (FaultClass fault : info.injected_by){ + if (!sweep.injectors.contains(fault)){ + continue; + } + for (const std::string& row : sweep.rows){ + cells.push_back(FaultSweepCell{.failure_class = info.failure_class, .fault = fault, .row = row}); + } + } + } + return cells; +} + +std::string fault_sweep_test_name(std::string_view test_prefix, const FaultSweepCell& cell){ + return std::format("{} - Fault sweep: {}, {}", test_prefix, cell.row, story_failure_class_info(cell.failure_class).id); +} + +std::vector classes_not_swept(const StoryFaultSweep& sweep){ + std::vector missed; + for (const StoryFailureClassInfo& info : story_failure_classes()){ + for (FaultClass fault : info.injected_by){ + if (!sweep.injectors.contains(fault)){ + missed.push_back(info.failure_class); + break; + } + } + } + return missed; +} + +std::vector known_failing_without_a_test(const StoryFaultSweep& sweep){ + std::set names; + for (const FaultSweepCell& cell : fault_sweep_cells(sweep)){ + names.insert(fault_sweep_test_name(sweep.test_prefix, cell)); + } + std::vector stale; + for (const auto& item : sweep.known_failing){ + if (!names.contains(item.first)){ + stale.push_back(item.first); + } + } + return stale; +} + +void add_fault_sweep_tests(UnitTestDatabase& database, std::shared_ptr sweep){ + for (FaultSweepCell& cell : fault_sweep_cells(*sweep)){ + database.add(sweep, std::move(cell)); + } + database.add(std::move(sweep)); +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFaultSweep.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFaultSweep.h new file mode 100644 index 0000000000..9b85141408 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFaultSweep.h @@ -0,0 +1,98 @@ +/* Story Fault Sweep + * + * From: https://github.com/PokemonAutomation/ + * + * The fault sweeps every game runs on its sim: each class of failure the + * consoles showed (Pokemon_StoryFailureClasses.h) injected at every row + * of a game's story, not only where it was seen. A game gives its sweep as + * data: its rows, an injector per sim fault its sim can inject, and the + * sweeps it fails today. This makes one test per class and row, named by + * the class's id, and judges it the same way for every game. + * + */ + +#ifndef PokemonAutomation_Pokemon_StoryFaultSweep_H +#define PokemonAutomation_Pokemon_StoryFaultSweep_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "Pokemon/Navigation/Simulation/Pokemon_SimFaults.h" +#include "Pokemon_StoryFailureClasses.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// What injecting one fault over one row found. +struct FaultSweepResult{ + // One injected variant that failed its row. + struct Failure{ + // The row's step the fault was injected at. + std::string step; + // Which variant of the fault: "warp 1", "seed 16". + std::string variant; + // How the row stopped or where it ended instead. + std::string stopped; + }; + + size_t variants = 0; + std::vector failures = {}; + // Why the row was not swept (it fails clean, the fault has no chance + // in it); nothing when it was. + std::optional not_swept = std::nullopt; +}; + +// A game's fault sweep, as data. +struct StoryFaultSweep{ + // A game's injector of one sim fault: runs every variant of it over + // the row named `row`. + using Injector = std::function; + + // The start of every test's name: "PokemonFRLG::AutoStory". + std::string test_prefix; + // The story's rows to sweep, by id, in story order. + std::vector rows; + // An injector per sim fault the game's sim can inject. + std::map injectors; + // The sweeps the program fails today, by test name, with the likely + // fix: such a test passes while it fails, and fails once it passes. + std::map known_failing = {}; +}; + +// One test of a sweep: one sim fault over one row. +struct FaultSweepCell{ + StoryFailureClass failure_class; + FaultClass fault; + std::string row; +}; + +// Every test of `sweep`: the classes in the shared list's order, each +// over every row. +std::vector fault_sweep_cells(const StoryFaultSweep& sweep); +// " - Fault sweep: 11.01, warp_no_op". +std::string fault_sweep_test_name(std::string_view test_prefix, const FaultSweepCell& cell); + +// The classes the shared list injects by a sim fault that `sweep` has no +// injector for, in the list's order: what the game cannot sweep yet. +std::vector classes_not_swept(const StoryFaultSweep& sweep); +// The names on `sweep`'s known-failing list that no test has, as after a +// rename. +std::vector known_failing_without_a_test(const StoryFaultSweep& sweep); + +// Adds a test per cell of `sweep`, and one that its known-failing list +// names only its tests. +void add_fault_sweep_tests(UnitTestDatabase& database, std::shared_ptr sweep); + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFaultSweep_Tests.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFaultSweep_Tests.cpp new file mode 100644 index 0000000000..00a62c9817 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFaultSweep_Tests.cpp @@ -0,0 +1,329 @@ +/* Story Fault Sweep Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The shared sweep against a fake game: two rows, injectors that report + * what a test sets, and a known-failing list. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "Pokemon_StoryFaultSweep.h" +#include "Pokemon_StoryFaultSweep_Tests.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + +constexpr std::string_view PREFIX = "Fake::AutoStory"; + +// An injector that reports `result` for every row. +StoryFaultSweep::Injector reporting(FaultSweepResult result){ + return [result = std::move(result)](Logger&, const std::string&){ + return result; + }; +} + +FaultSweepResult clean_result(){ + return FaultSweepResult{.variants = 3, .failures = {}, .not_swept = std::nullopt}; +} + +FaultSweepResult failing_result(){ + return FaultSweepResult{ + .variants = 3, + .failures = {{.step = "fly_celadon", .variant = "warp 1", .stopped = "ended in Lavender Town"}}, + .not_swept = std::nullopt, + }; +} + +// A fake game's sweep: rows 01.01 and 01.02, the given injectors. +StoryFaultSweep fake_sweep(std::map injectors, std::map known_failing = {}){ + return StoryFaultSweep{ + .test_prefix = std::string(PREFIX), + .rows = {"01.01", "01.02"}, + .injectors = std::move(injectors), + .known_failing = std::move(known_failing), + }; +} + + +// The tests add_fault_sweep_tests() makes, by name. +struct SweepNamesTestCase{ + struct Input{ + std::vector injected; + }; + + struct Expectation{ + std::vector names; + }; + + std::string name; + Input input; + Expectation expectation; +}; + +class Test_SweepNames : public UnitTest{ +public: + explicit Test_SweepNames(SweepNamesTestCase test_case) + : UnitTest(std::format("Pokemon::StoryFaultSweep - tests: {}", test_case.name)) + , m_case(std::move(test_case)) + {} + + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + std::map injectors; + for (FaultClass fault : m_case.input.injected){ + injectors.emplace(fault, reporting(clean_result())); + } + std::vector names; + for (const FaultSweepCell& cell : fault_sweep_cells(fake_sweep(std::move(injectors)))){ + names.emplace_back(fault_sweep_test_name(PREFIX, cell)); + } + if (names != m_case.expectation.names){ + std::string got; + for (const std::string& name : names){ + got += std::format("\n {}", name); + } + return std::format("Unexpected tests:{}", got); + } + return true; + } + +private: + const SweepNamesTestCase m_case; +}; + +const SweepNamesTestCase SWEEP_NAMES_CASES[] = { + { + .name = "classes in the list's order, then rows", + .input = {.injected = {FaultClass::LATE_FADE, FaultClass::WARP_NO_OP}}, + .expectation = {.names = { + "Fake::AutoStory - Fault sweep: 01.01, warp_no_op", + "Fake::AutoStory - Fault sweep: 01.02, warp_no_op", + "Fake::AutoStory - Fault sweep: 01.01, late_fade", + "Fake::AutoStory - Fault sweep: 01.02, late_fade", + }}, + }, + { + .name = "a sim fault named by its story class", + .input = {.injected = {FaultClass::DARK_CAVE}}, + .expectation = {.names = { + "Fake::AutoStory - Fault sweep: 01.01, dark_map", + "Fake::AutoStory - Fault sweep: 01.02, dark_map", + }}, + }, + { + .name = "no injectors", + .input = {.injected = {}}, + .expectation = {.names = {}}, + }, +}; + + +// One sweep test made by add_fault_sweep_tests(), run. +struct SweepRunTestCase{ + struct Input{ + FaultSweepResult result; + // The test's known-failing fix, if it is on the list. + std::optional known_failing; + }; + + struct Expectation{ + bool passes; + // What the failure message starts with, when it fails. + std::string message; + }; + + std::string name; + Input input; + Expectation expectation; +}; + +class Test_SweepRun : public UnitTest{ +public: + explicit Test_SweepRun(SweepRunTestCase test_case) + : UnitTest(std::format("Pokemon::StoryFaultSweep - run: {}", test_case.name)) + , m_case(std::move(test_case)) + {} + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + const std::string test_name = "Fake::AutoStory - Fault sweep: 01.02, warp_no_op"; + std::map known_failing; + if (m_case.input.known_failing){ + known_failing.emplace(test_name, *m_case.input.known_failing); + } + UnitTestDatabase database; + add_fault_sweep_tests(database, std::make_shared(fake_sweep( + {{FaultClass::WARP_NO_OP, reporting(m_case.input.result)}}, std::move(known_failing) + ))); + const auto test = database.find(test_name); + if (test == database.end()){ + return std::format("No test \"{}\".", test_name); + } + const UnitTestResult result = test->second->run(logger, scope); + const bool passed = result.result == UnitTestResult::PASSED; + if (passed != m_case.expectation.passes){ + return std::format("Expected {}, got: {}", m_case.expectation.passes ? "a pass" : "a failure", result.message); + } + if (!passed && !result.message.starts_with(m_case.expectation.message)){ + return std::format("Expected a message starting \"{}\", got \"{}\"", m_case.expectation.message, result.message); + } + return true; + } + +private: + const SweepRunTestCase m_case; +}; + +const SweepRunTestCase SWEEP_RUN_CASES[] = { + { + .name = "every variant passes", + .input = {.result = clean_result(), .known_failing = std::nullopt}, + .expectation = {.passes = true, .message = ""}, + }, + { + .name = "a variant fails", + .input = {.result = failing_result(), .known_failing = std::nullopt}, + .expectation = { + .passes = false, + .message = "1 of 3 variants fail 01.02 (class=warp_no_op):\n fly_celadon warp 1: ended in Lavender Town", + }, + }, + { + .name = "a row not swept", + .input = {.result = FaultSweepResult{.variants = 0, .failures = {}, .not_swept = "the row fails clean"}, .known_failing = std::nullopt}, + .expectation = {.passes = true, .message = ""}, + }, + { + .name = "a known failure", + .input = {.result = failing_result(), .known_failing = "check the fly lands"}, + .expectation = {.passes = true, .message = ""}, + }, + { + .name = "a known failure that passes now", + .input = {.result = clean_result(), .known_failing = "check the fly lands"}, + .expectation = {.passes = false, .message = "on KNOWN_FAILING (check the fly lands) but passes now: take it off the list."}, + }, +}; + + +// The sweep's list checks against the shared classes. +struct SweepListTestCase{ + struct Input{ + std::vector injected; + std::vector known_failing; + }; + + struct Expectation{ + std::vector missed; + std::vector stale_known_failing; + }; + + std::string name; + Input input; + Expectation expectation; +}; + +class Test_SweepList : public UnitTest{ +public: + explicit Test_SweepList(SweepListTestCase test_case) + : UnitTest(std::format("Pokemon::StoryFaultSweep - list: {}", test_case.name)) + , m_case(std::move(test_case)) + {} + + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + std::map injectors; + for (FaultClass fault : m_case.input.injected){ + injectors.emplace(fault, reporting(clean_result())); + } + std::map known_failing; + for (const std::string& name : m_case.input.known_failing){ + known_failing.emplace(name, "a fix"); + } + const StoryFaultSweep sweep = fake_sweep(std::move(injectors), std::move(known_failing)); + if (classes_not_swept(sweep) != m_case.expectation.missed){ + return std::format("Unexpected classes not swept: {}", classes_not_swept(sweep).size()); + } + if (known_failing_without_a_test(sweep) != m_case.expectation.stale_known_failing){ + return std::format("Unexpected stale known-failing names: {}", known_failing_without_a_test(sweep).size()); + } + return true; + } + +private: + const SweepListTestCase m_case; +}; + +using C = StoryFailureClass; + +const SweepListTestCase SWEEP_LIST_CASES[] = { + { + .name = "a game that injects all but one", + .input = { + .injected = { + FaultClass::SCENE_END_POSE, FaultClass::SCENE_EARLY, FaultClass::BATTLE_OPENING, FaultClass::WILD_BATTLES, + FaultClass::LATE_FADE, FaultClass::SLOW_FADE_IN, FaultClass::WARP_NO_OP, FaultClass::DARK_CAVE, + FaultClass::LOST_TAP, FaultClass::UNSEEN_STEP, + }, + .known_failing = {}, + }, + .expectation = {.missed = {C::MOUNT_LOST}, .stale_known_failing = {}}, + }, + { + .name = "a game that injects none", + .input = {.injected = {}, .known_failing = {}}, + .expectation = { + .missed = { + C::LOST_TAP, C::UNSEEN_STEP, C::MOUNT_LOST, C::WARP_NO_OP, C::LATE_FADE, C::SLOW_FADE_IN, + C::DARK_MAP, C::SCENE_END_POSE, C::SCENE_EARLY, C::BATTLE_OPENING, C::WILD_BATTLES, + }, + .stale_known_failing = {}, + }, + }, + { + .name = "a known failure under an old name", + .input = { + .injected = {FaultClass::WARP_NO_OP}, + .known_failing = { + "Fake::AutoStory - Fault sweep: 01.01, warp_no_op", + "Fake::AutoStory - Fault sweep: 01.01, warp no-op", + }, + }, + .expectation = { + .missed = { + C::LOST_TAP, C::UNSEEN_STEP, C::MOUNT_LOST, C::LATE_FADE, C::SLOW_FADE_IN, + C::DARK_MAP, C::SCENE_END_POSE, C::SCENE_EARLY, C::BATTLE_OPENING, C::WILD_BATTLES, + }, + .stale_known_failing = {"Fake::AutoStory - Fault sweep: 01.01, warp no-op"}, + }, + }, +}; + +} // namespace + + +void add_tests_StoryFaultSweep(UnitTestDatabase& database){ + for (const SweepNamesTestCase& test_case : SWEEP_NAMES_CASES){ + database.add(test_case); + } + for (const SweepRunTestCase& test_case : SWEEP_RUN_CASES){ + database.add(test_case); + } + for (const SweepListTestCase& test_case : SWEEP_LIST_CASES){ + database.add(test_case); + } +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFaultSweep_Tests.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFaultSweep_Tests.h new file mode 100644 index 0000000000..41d4443e73 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryFaultSweep_Tests.h @@ -0,0 +1,23 @@ +/* Story Fault Sweep Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_Pokemon_StoryFaultSweep_Tests_H +#define PokemonAutomation_Pokemon_StoryFaultSweep_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// Registers unit tests for the shared fault sweep. +void add_tests_StoryFaultSweep(UnitTestDatabase& database); + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryOutcome.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryOutcome.h index a752df6972..66a1e1a083 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryOutcome.h +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryOutcome.h @@ -17,6 +17,7 @@ #include #include #include "Common/Cpp/MoveOnlyFunction.h" +#include "Pokemon_StoryFailureClasses.h" #include "Pokemon_StoryLog.h" #include "Pokemon_StoryPose.h" @@ -36,6 +37,10 @@ struct Outcome{ // check, a TM the lead already knows. A reload of the same save undoes // nothing it did. bool changed_nothing = false; + // The failure's class, when the action knows it: a game's own failure + // name ("probe_exhausted") has none in the shared list until the game + // gives it one here. Nothing: the class the list gives `failure`. + std::optional failure_class = std::nullopt; bool succeeded() const; }; @@ -44,7 +49,12 @@ Outcome outcome_ok(std::optional pose = std::nullopt); // outcome_ok() with `changed_nothing`. Outcome outcome_unchanged(); // Constructs a failed event outcome with reason and optional diagnostic detail. -Outcome outcome_failed(std::string failure, std::string detail = ""); +// `failure_class` is for a name the shared list does not class (a game's +// own); without it the list's class for `failure` holds. +Outcome outcome_failed( + std::string failure, std::string detail = "", + std::optional failure_class = std::nullopt +); // "probe_exhausted: 4 rows": the failure and its detail, if any. std::string to_string(const Outcome& outcome); diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryOutcomeClass.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryOutcomeClass.cpp new file mode 100644 index 0000000000..064bb87ee4 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryOutcomeClass.cpp @@ -0,0 +1,35 @@ +/* Story Outcome Class + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "Pokemon_StoryOutcomeClass.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +std::optional failure_class_of(const Outcome& outcome){ + if (outcome.succeeded()){ + return std::nullopt; + } + if (outcome.failure_class){ + return outcome.failure_class; + } + return story_failure_class_of_outcome(outcome.failure); +} + +std::string to_string_with_class(const Outcome& outcome){ + const std::optional failure_class = failure_class_of(outcome); + if (!failure_class){ + return to_string(outcome); + } + return std::format("{} ({})", to_string(outcome), story_failure_class_tag(*failure_class)); +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryOutcomeClass.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryOutcomeClass.h new file mode 100644 index 0000000000..ec692db6ab --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryOutcomeClass.h @@ -0,0 +1,35 @@ +/* Story Outcome Class + * + * From: https://github.com/PokemonAutomation/ + * + * The class (Pokemon_StoryFailureClasses.h) of a failed Outcome, and the + * text a failure's log line ends with. + * + */ + +#ifndef PokemonAutomation_Pokemon_StoryOutcomeClass_H +#define PokemonAutomation_Pokemon_StoryOutcomeClass_H + +#include +#include +#include "Pokemon_StoryFailureClasses.h" +#include "Pokemon_StoryOutcome.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// The class of a failed `outcome`: its own, else the shared list's for its +// name. Nothing for a success or a name no class lists. +std::optional failure_class_of(const Outcome& outcome); + +// to_string() and the class, as a failure's log line ends: +// "expect_failed: has_parcel (class=state_check_failed)"; without a class, +// to_string() alone. +std::string to_string_with_class(const Outcome& outcome); + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySegments.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySegments.cpp index c46ab2764e..2ff6960ed0 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySegments.cpp +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySegments.cpp @@ -8,6 +8,7 @@ #include #include "Pokemon_CheckpointAttempts.h" #include "Pokemon_StoryLog.h" +#include "Pokemon_StoryOutcomeClass.h" #include "Pokemon_StorySegments.h" namespace PokemonAutomation{ @@ -220,7 +221,7 @@ SegmentReport stop_segment(const std::string& id, Outcome failure, SegmentHooks& where += std::format(" {}", report.failed_event); } } - say(std::format("[Segment] {}: failed{}: {}", id, where, to_string(failure)), hooks); + say(std::format("[Segment] {}: failed{}: {}", id, where, to_string_with_class(failure)), hooks); report.failure = std::move(failure); return report; } @@ -286,7 +287,7 @@ bool verify_run( "[Verify] {}: run {}/{} failed{}: {}", spec.id, run, policy.runs, result.failed_name.empty() ? "" : std::format(" at {}", result.failed_name), - to_string(result.failure) + to_string_with_class(result.failure) ), hooks); return false; } diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimFaults.cpp b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimFaults.cpp index 6d0bf64fb4..cac34ff2fe 100644 --- a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimFaults.cpp +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimFaults.cpp @@ -23,6 +23,7 @@ const std::vector& fault_classes(){ FaultClass::DARK_CAVE, FaultClass::LOST_TAP, FaultClass::UNSEEN_STEP, + FaultClass::MOUNT_LOST, }; return classes; } @@ -49,6 +50,8 @@ std::string_view fault_class_name(FaultClass fault){ return "lost tap"; case FaultClass::UNSEEN_STEP: return "unseen step"; + case FaultClass::MOUNT_LOST: + return "mount lost"; } return "?"; } diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimFaults.h b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimFaults.h index f58b09b854..806e5b6696 100644 --- a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimFaults.h +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimFaults.h @@ -49,6 +49,9 @@ enum class FaultClass{ LOST_TAP, // A step tap that moved the player, read as not moved. UNSEEN_STEP, + // A leg that rides (the Bicycle) starts with the player off the mount, + // walking at foot pace. + MOUNT_LOST, }; // Every fault class, in the enum's order. diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index c64cc92e1d..4ae6751191 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1638,6 +1638,14 @@ file(GLOB LIBRARY_SOURCES Source/Pokemon/AutoStory/Pokemon_StoryEvents.h Source/Pokemon/AutoStory/Pokemon_StoryEvents_Tests.cpp Source/Pokemon/AutoStory/Pokemon_StoryEvents_Tests.h + Source/Pokemon/AutoStory/Pokemon_StoryFailureClasses.cpp + Source/Pokemon/AutoStory/Pokemon_StoryFailureClasses.h + Source/Pokemon/AutoStory/Pokemon_StoryFailureClasses_Tests.cpp + Source/Pokemon/AutoStory/Pokemon_StoryFailureClasses_Tests.h + Source/Pokemon/AutoStory/Pokemon_StoryFaultSweep.cpp + Source/Pokemon/AutoStory/Pokemon_StoryFaultSweep.h + Source/Pokemon/AutoStory/Pokemon_StoryFaultSweep_Tests.cpp + Source/Pokemon/AutoStory/Pokemon_StoryFaultSweep_Tests.h Source/Pokemon/AutoStory/Pokemon_StoryInterrupts.cpp Source/Pokemon/AutoStory/Pokemon_StoryInterrupts.h Source/Pokemon/AutoStory/Pokemon_StoryInterrupts_Tests.cpp @@ -1647,6 +1655,8 @@ file(GLOB LIBRARY_SOURCES Source/Pokemon/AutoStory/Pokemon_StoryLog_Tests.cpp Source/Pokemon/AutoStory/Pokemon_StoryLog_Tests.h Source/Pokemon/AutoStory/Pokemon_StoryOutcome.h + Source/Pokemon/AutoStory/Pokemon_StoryOutcomeClass.cpp + Source/Pokemon/AutoStory/Pokemon_StoryOutcomeClass.h Source/Pokemon/AutoStory/Pokemon_StoryPose.h Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.cpp Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.h