diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.h b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.h new file mode 100644 index 0000000000..0b036eb5bf --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.h @@ -0,0 +1,46 @@ +/* Story Engine Report + * + * From: https://github.com/PokemonAutomation/ + * + * What an engine run of one checkpoint reports. + * + */ + +#ifndef PokemonAutomation_Pokemon_StoryEnginePolicy_H +#define PokemonAutomation_Pokemon_StoryEnginePolicy_H + +#include +#include "Pokemon/AutoStory/Pokemon_StoryCheckpointTypes.h" +#include "Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// How a checkpoint ended: passed (and its postconditions, where checked, +// did not fail) or not. +enum class CheckpointVerdict{ + PASSED, + FAILED, +}; + +// An engine run of one checkpoint. +struct StoryEngineReport{ + // The steps' run. A checkpoint that failed its lint has none. + CheckpointReport run = {}; + // PASSED when the steps completed and no postcondition failed. + CheckpointVerdict verdict = CheckpointVerdict::FAILED; + // Checked after steps that completed. + std::optional postconditions = std::nullopt; + // The game saved at the checkpoint's end. + bool saved = false; + // A SAVE step among its steps saved: a reload now returns there, not + // to the save the checkpoint started from. + bool saved_part_way = false; +}; + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h index c0bb0d4bea..7a48248160 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h @@ -20,6 +20,7 @@ #include "Pokemon_StoryEvents_Tests.h" #include "Pokemon_StoryFailureClasses_Tests.h" #include "Pokemon_StoryLog_Tests.h" +#include "Pokemon_StorySavePolicy_Tests.h" #include "Pokemon_StoryStepExecutor_Tests.h" #include "Types/Pokemon_StoryActionArg_Tests.h" #include "Types/Pokemon_StoryRunTypes_Tests.h" @@ -38,6 +39,7 @@ inline void add_tests_AutoStory(UnitTestDatabase& database){ add_tests_StoryStepJson(database); add_tests_AuthoredCheckpointJson(database); add_tests_StoryEngine(database); + add_tests_StorySavePolicy(database); add_tests_SceneJudges(database); add_tests_StoryFiles(database); add_tests_StoryRunTypes(database); diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.cpp new file mode 100644 index 0000000000..5b0e617f3b --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.cpp @@ -0,0 +1,227 @@ +/* Story Postcondition Check + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "Pokemon_StoryPostconditionCheck.h" +#include "Pokemon_StoryStepExecutor.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + +std::string verdict_name(PostconditionVerdict verdict){ + switch (verdict){ + case PostconditionVerdict::PASSED: + return "passed"; + case PostconditionVerdict::FAILED: + return "failed"; + case PostconditionVerdict::NOT_CHECKED: + return "not checked"; + } + return "?"; +} + +PostconditionResult result(std::string what, PostconditionVerdict verdict, std::string detail){ + return PostconditionResult{.what = std::move(what), .verdict = verdict, .detail = std::move(detail)}; +} + +PostconditionResult not_read(std::string what){ + return result(std::move(what), PostconditionVerdict::NOT_CHECKED, "the game can't read it"); +} + +// The first pose among a step's args (a route's target). +std::optional pose_arg(const StoryStep& step){ + for (const auto& [unused, value] : step.args){ + if (const GridPose* pose = std::get_if(&value)){ + return *pose; + } + } + return std::nullopt; +} + +std::string tiles_text(const std::vector& tiles){ + std::string text; + for (const TilePos& tile : tiles){ + text += (text.empty() ? "" : " or ") + to_string(tile); + } + return text; +} + +void check_map(const GridPose& end, const ObservedState& observed, std::vector& out){ + if (!observed.map){ + out.push_back(not_read("map")); + return; + } + const bool same = *observed.map == end.map; + out.push_back(result( + "map", same ? PostconditionVerdict::PASSED : PostconditionVerdict::FAILED, + same ? end.map : std::format("expected {}, read {}", end.map, *observed.map) + )); +} + +// The tile and facing mean nothing on another map: not checked then. +void check_tile(const GridPose& end, const ObservedState& observed, bool map_failed, std::vector& out){ + if (end.tiles.empty()){ + return; + } + if (map_failed){ + out.push_back(result("tile", PostconditionVerdict::NOT_CHECKED, "on another map")); + return; + } + if (!observed.tile){ + out.push_back(not_read("tile")); + return; + } + const bool on = std::find(end.tiles.begin(), end.tiles.end(), *observed.tile) != end.tiles.end(); + out.push_back(result( + "tile", on ? PostconditionVerdict::PASSED : PostconditionVerdict::FAILED, + on ? to_string(*observed.tile) : std::format("expected {}, read {}", tiles_text(end.tiles), to_string(*observed.tile)) + )); +} + +void check_facing(const GridPose& end, const ObservedState& observed, bool map_failed, std::vector& out){ + if (!end.facing){ + return; + } + if (map_failed){ + out.push_back(result("facing", PostconditionVerdict::NOT_CHECKED, "on another map")); + return; + } + if (!observed.facing){ + out.push_back(not_read("facing")); + return; + } + const bool same = *observed.facing == *end.facing; + out.push_back(result( + "facing", same ? PostconditionVerdict::PASSED : PostconditionVerdict::FAILED, + same ? direction_name(*end.facing) + : std::format("expected {}, read {}", direction_name(*end.facing), direction_name(*observed.facing)) + )); +} + +void check_position(const StoryCheckpoint& checkpoint, const ObservedState& observed, std::vector& out){ + const std::optional end = checkpoint_end_pose(checkpoint); + if (!end){ + return; + } + check_map(*end, observed, out); + const bool map_failed = out.back().verdict == PostconditionVerdict::FAILED; + check_tile(*end, observed, map_failed, out); + check_facing(*end, observed, map_failed, out); +} + +void check_items(const StoryPostconditions& expected, const ObservedState& observed, std::vector& out){ + for (const StoryItem& item : expected.items){ + const std::string what = std::format("item {}", item.item); + const auto read = observed.items.find(item.item); + if (read == observed.items.end()){ + out.push_back(not_read(what)); + continue; + } + const bool same = read->second == item.quantity; + out.push_back(result( + what, same ? PostconditionVerdict::PASSED : PostconditionVerdict::FAILED, + same ? std::format("{}", item.quantity) : std::format("expected {}, read {}", item.quantity, read->second) + )); + } +} + +void check_flags(const StoryPostconditions& expected, const ObservedState& observed, std::vector& out){ + for (const std::string& flag : expected.flags){ + const std::string what = std::format("flag {}", flag); + const auto read = observed.flags.find(flag); + if (read == observed.flags.end()){ + out.push_back(not_read(what)); + continue; + } + out.push_back(result( + what, read->second ? PostconditionVerdict::PASSED : PostconditionVerdict::FAILED, + read->second ? "set" : "expected set, read clear" + )); + } +} + +std::string summary_verdict(const PostconditionReport& report){ + if (report.results.empty()){ + return "nothing to check"; + } + if (report.count(PostconditionVerdict::FAILED) > 0){ + return "failed"; + } + return report.all_passed() ? "passed" : "incomplete"; +} + +} // namespace + + +size_t PostconditionReport::count(PostconditionVerdict verdict) const{ + size_t total = 0; + for (const PostconditionResult& item : results){ + total += item.verdict == verdict ? 1 : 0; + } + return total; +} + +bool PostconditionReport::all_passed() const{ + return !results.empty() && count(PostconditionVerdict::PASSED) == results.size(); +} + + +std::optional checkpoint_end_pose(const StoryCheckpoint& checkpoint){ + if (checkpoint.postconditions.position){ + return checkpoint.postconditions.position; + } + for (auto step = checkpoint.steps.rbegin(); step != checkpoint.steps.rend(); ++step){ + if (step->pose_after){ + return grid_pose_of(*step->pose_after); + } + if (std::optional target = pose_arg(*step)){ + return target; + } + // Taps and single steps move the player without saying where to. + if (step->kind == StoryStepKind::LEGS || step->kind == StoryStepKind::STEP){ + return std::nullopt; + } + } + return std::nullopt; +} + +PostconditionReport check_postconditions(const StoryCheckpoint& checkpoint, const ObservedState& observed){ + PostconditionReport report; + report.checkpoint = checkpoint.id; + check_position(checkpoint, observed, report.results); + check_items(checkpoint.postconditions, observed, report.results); + check_flags(checkpoint.postconditions, observed, report.results); + return report; +} + +std::vector postcondition_log_lines(const PostconditionReport& report){ + std::vector lines; + for (const PostconditionResult& item : report.results){ + lines.push_back(std::format( + "[Postcondition] {} {}: {}: {}", report.checkpoint, item.what, verdict_name(item.verdict), item.detail + )); + } + lines.push_back(std::format( + "[Postcondition] {}: {} passed, {} failed, {} not checked: {}", + report.checkpoint, + report.count(PostconditionVerdict::PASSED), + report.count(PostconditionVerdict::FAILED), + report.count(PostconditionVerdict::NOT_CHECKED), + summary_verdict(report) + )); + return lines; +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.h new file mode 100644 index 0000000000..c04c0e8e7e --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.h @@ -0,0 +1,89 @@ +/* Story Postcondition Check + * + * From: https://github.com/PokemonAutomation/ + * + * A checkpoint's postconditions (Types/Pokemon_StoryPostconditions.h) + * checked against what the game read once the run was over: a step of its + * own, not the run's verdict, which can be wrong. A postcondition the game + * can't read yet is reported as not checked, never as passed. + * + */ + +#ifndef PokemonAutomation_Pokemon_StoryPostconditionCheck_H +#define PokemonAutomation_Pokemon_StoryPostconditionCheck_H + +#include +#include +#include +#include +#include +#include +#include "Pokemon/AutoStory/Types/Pokemon_GridPose.h" +#include "Pokemon/AutoStory/Types/Pokemon_StoryCheckpoint.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// What the game read after the run. Each part is unset (or, for items and +// flags, absent from the map) when the game can't read it: the matching +// postconditions are then not checked. +struct ObservedState{ + std::optional map; + std::optional tile; + std::optional facing; + // The Bag's count of each item read, 0 for one read and not there. + std::map> items; + // Each story flag read, and whether it is set. + std::map> flags; +}; + +// The result of one postcondition. +enum class PostconditionVerdict{ + PASSED, + FAILED, + NOT_CHECKED, +}; + +// One postcondition checked: "map", "tile", "facing", "item LIFT_KEY", +// "flag ...", its verdict and what was expected and read. +struct PostconditionResult{ + std::string what; + PostconditionVerdict verdict = PostconditionVerdict::NOT_CHECKED; + std::string detail; +}; + +// Every postcondition of one checkpoint, checked. +struct PostconditionReport{ + std::string checkpoint; + std::vector results; + + size_t count(PostconditionVerdict verdict) const; + // Every postcondition was read and held: none failed, none unchecked, + // and there was at least one. + bool all_passed() const; +}; + +// Where `checkpoint` ends: its postconditions' position, else the last +// step's declared pose or target. Unset when a step after that moves the +// player in a way the plan does not place (taps of legs, a single step). +std::optional checkpoint_end_pose(const StoryCheckpoint& checkpoint); + +// Checks the end position (map, tile among the pose's tiles, facing), +// then each item count and flag, against `observed`. +PostconditionReport check_postconditions(const StoryCheckpoint& checkpoint, const ObservedState& observed); + +// The report as log lines, one per postcondition and a summary: +// [Postcondition] 00.31 map: passed: RocketHideout_B3F +// [Postcondition] 00.31 item RARE_CANDY: failed: expected 2, read 1 +// [Postcondition] 00.31 flag hideout_lift_key_grunt: not checked: the game can't read it +// [Postcondition] 00.31: 3 passed, 1 failed, 1 not checked: failed +// The summary is "passed" only when every postcondition passed; +// "incomplete" when none failed and some were not checked. +std::vector postcondition_log_lines(const PostconditionReport& report); + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.cpp new file mode 100644 index 0000000000..7ac81dcb16 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.cpp @@ -0,0 +1,192 @@ +/* Story Save Policy Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The postcondition check (Pokemon_StoryPostconditionCheck.h) on a + * checkpoint's end state. + * + */ + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Json/JsonValue.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "Data/Pokemon_StoryStepJson.h" +#include "Pokemon_StoryPostconditionCheck.h" +#include "Pokemon_StorySavePolicy_Tests.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + + +// A checkpoint that ends beside B3F's Rare Candy, with the LIFT KEY and +// both Rare Candies. +inline constexpr std::string_view CHECKPOINT_0031 = R"({ + "id": "00.31", + "title": "Rocket Hideout to the LIFT KEY", + "start": {"map": "CeladonCity_PokemonCenter_1F", "tiles": [[7, 4]], "facing": "north"}, + "events": [ + {"id": "beside_b3f_rare_candy", "kind": "route", "args": {"target": {"map": "RocketHideout_B3F", "tiles": [[12, 13]], "facing": "north"}}}, + {"id": "b3f_rare_candy", "kind": "talk"} + ], + "metadata": {"phase": "testing", "postconditions": { + "position": {"map": "RocketHideout_B3F", "tiles": [[12, 13]], "facing": "north"}, + "items": [{"item": "LIFT_KEY", "quantity": 1}, {"item": "RARE_CANDY", "quantity": 2}] + }} +})"; + +StoryCheckpoint checkpoint_0031(){ + return read_story_checkpoint(parse_json(std::string(CHECKPOINT_0031)), "00.31"); +} + +ObservedState read_0031(std::string map, int candies){ + ObservedState observed; + observed.map = std::move(map); + observed.tile = TilePos{12, 13}; + observed.facing = Direction::NORTH; + observed.items = {{"LIFT_KEY", 1}, {"RARE_CANDY", (size_t)candies}}; + return observed; +} + +UnitTestResult test_postconditions_json(){ + const StoryCheckpoint checkpoint = checkpoint_0031(); + const StoryPostconditions& post = checkpoint.postconditions; + if (!post.position || post.position->map != "RocketHideout_B3F" || post.items.size() != 2 + || post.items[1] != StoryItem{.item = "RARE_CANDY", .quantity = 2} + ){ + return "Unexpected postconditions read."; + } + if (post.flags.size() != 0 || post.position->tiles != std::vector{TilePos{12, 13}}){ + return "Unexpected postcondition position or flags read."; + } + return true; +} + +UnitTestResult test_postconditions_pass(){ + const PostconditionReport report = check_postconditions(checkpoint_0031(), read_0031("RocketHideout_B3F", 2)); + const std::vector lines = postcondition_log_lines(report); + if (!report.all_passed() || lines.back() != "[Postcondition] 00.31: 5 passed, 0 failed, 0 not checked: passed"){ + return std::format("Expected every postcondition passed: {}", lines.back()); + } + return true; +} + +// A run that reports success while still on B4F, the Rare Candy not +// picked up. +UnitTestResult test_postconditions_fail(){ + const PostconditionReport report = check_postconditions(checkpoint_0031(), read_0031("RocketHideout_B4F", 1)); + const std::vector lines = postcondition_log_lines(report); + const std::vector expected = { + "[Postcondition] 00.31 map: failed: expected RocketHideout_B3F, read RocketHideout_B4F", + "[Postcondition] 00.31 tile: not checked: on another map", + "[Postcondition] 00.31 facing: not checked: on another map", + "[Postcondition] 00.31 item LIFT_KEY: passed: 1", + "[Postcondition] 00.31 item RARE_CANDY: failed: expected 2, read 1", + "[Postcondition] 00.31: 1 passed, 2 failed, 2 not checked: failed", + }; + if (lines != expected){ + std::string text; + for (const std::string& line : lines){ + text += std::format("\n{}", line); + } + return std::format("Unexpected lines:{}", text); + } + return true; +} + +UnitTestResult test_postconditions_not_checked(){ + StoryCheckpoint checkpoint = checkpoint_0031(); + checkpoint.postconditions.flags = {"hideout_lift_key_grunt"}; + ObservedState observed; + observed.map = "RocketHideout_B3F"; + const PostconditionReport report = check_postconditions(checkpoint, observed); + const std::vector lines = postcondition_log_lines(report); + if (report.all_passed() || report.count(PostconditionVerdict::NOT_CHECKED) != 5 + || lines.back() != "[Postcondition] 00.31: 1 passed, 0 failed, 5 not checked: incomplete" + ){ + return std::format("What isn't read is not checked, never passed: {}", lines.back()); + } + observed.flags["hideout_lift_key_grunt"] = false; + if (check_postconditions(checkpoint, observed).count(PostconditionVerdict::FAILED) != 1){ + return "A flag read clear should fail."; + } + return true; +} + +UnitTestResult test_end_pose_from_the_plan(){ + StoryCheckpoint checkpoint = checkpoint_0031(); + checkpoint.postconditions.position.reset(); + const std::optional end = checkpoint_end_pose(checkpoint); + if (!end || end->map != "RocketHideout_B3F" || end->tiles != std::vector{TilePos{12, 13}}){ + return "Expected the last route's target as the end."; + } + StoryCheckpoint stepped = checkpoint; + StoryStep step; + step.kind = StoryStepKind::STEP; + step.name = "one more"; + stepped.steps.push_back(std::move(step)); + if (checkpoint_end_pose(stepped)){ + return "A step after the last target leaves the end unknown."; + } + StoryStep legs; + legs.kind = StoryStepKind::LEGS; + legs.name = "taps"; + checkpoint.steps.push_back(std::move(legs)); + if (checkpoint_end_pose(checkpoint)){ + return "Legs after the last target leave the end unknown."; + } + if (!check_postconditions(checkpoint, read_0031("RocketHideout_B3F", 2)).all_passed()){ + return "Without an end pose, the items alone should be checked."; + } + return true; +} + + +// A named test; it returns its own failure. +struct SavePolicyTest{ + std::string name; + UnitTestResult (*test)(); +}; + +class Test_StorySavePolicy : public UnitTest{ +public: + explicit Test_StorySavePolicy(SavePolicyTest test) + : UnitTest(std::format("Pokemon::StorySavePolicy - {}", test.name)) + , m_test(std::move(test)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return m_test.test(); + } + +private: + SavePolicyTest m_test; +}; + +const SavePolicyTest SAVE_POLICY_TESTS[] = { + {.name = "postconditions: json", .test = test_postconditions_json}, + {.name = "postconditions: pass", .test = test_postconditions_pass}, + {.name = "postconditions: fail", .test = test_postconditions_fail}, + {.name = "postconditions: not checked", .test = test_postconditions_not_checked}, + {.name = "postconditions: end pose from the plan", .test = test_end_pose_from_the_plan}, +}; + +} // namespace + + +void add_tests_StorySavePolicy(UnitTestDatabase& database){ + for (const SavePolicyTest& test : SAVE_POLICY_TESTS){ + database.add(test); + } +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.h new file mode 100644 index 0000000000..4e95d5e81a --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.h @@ -0,0 +1,23 @@ +/* Story Save Policy Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_Pokemon_StorySavePolicy_Tests_H +#define PokemonAutomation_Pokemon_StorySavePolicy_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// Registers unit tests for when a run saves (save modes) and the postcondition check. +void add_tests_StorySavePolicy(UnitTestDatabase& database); + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index c02d4b6862..901de1bf89 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1478,6 +1478,7 @@ file(GLOB LIBRARY_SOURCES Source/Pokemon/AutoStory/Engine/Pokemon_StoryBuiltinActions.h Source/Pokemon/AutoStory/Engine/Pokemon_StoryEngine_Tests.cpp Source/Pokemon/AutoStory/Engine/Pokemon_StoryEngine_Tests.h + Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.h Source/Pokemon/AutoStory/Engine/Pokemon_StoryFakeBackend.cpp Source/Pokemon/AutoStory/Engine/Pokemon_StoryFakeBackend.h Source/Pokemon/AutoStory/Engine/Pokemon_StoryGame.h @@ -1519,6 +1520,10 @@ file(GLOB LIBRARY_SOURCES Source/Pokemon/AutoStory/Pokemon_StoryOutcomeClass.h Source/Pokemon/AutoStory/Pokemon_StoryPose.cpp Source/Pokemon/AutoStory/Pokemon_StoryPose.h + Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.cpp + Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.h + Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.cpp + Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.h Source/Pokemon/AutoStory/Pokemon_StoryStepExecutor.cpp Source/Pokemon/AutoStory/Pokemon_StoryStepExecutor.h Source/Pokemon/AutoStory/Pokemon_StoryStepExecutor_Tests.cpp