From 1f0cd80bb7abc1139f7d749de5d33df167cc4a3f Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 15 Nov 2026 00:15:16 -0800 Subject: [PATCH 1/3] AutoStory: postcondition checks and the engine save policy --- .../Engine/Pokemon_StoryEnginePolicy.cpp | 65 +++ .../Engine/Pokemon_StoryEnginePolicy.h | 64 +++ .../AutoStory/Pokemon_AutoStory_Tests.h | 2 + .../Pokemon_StoryPostconditionCheck.cpp | 227 ++++++++ .../Pokemon_StoryPostconditionCheck.h | 89 +++ .../Pokemon_StorySavePolicy_Tests.cpp | 508 ++++++++++++++++++ .../AutoStory/Pokemon_StorySavePolicy_Tests.h | 23 + SerialPrograms/cmake/SourceFiles.cmake | 6 + 8 files changed, 984 insertions(+) create mode 100644 SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.cpp create mode 100644 SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.h create mode 100644 SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.cpp create mode 100644 SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.h create mode 100644 SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.cpp create mode 100644 SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.h diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.cpp new file mode 100644 index 0000000000..4b0a6902a1 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.cpp @@ -0,0 +1,65 @@ +/* Story Engine Policy + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "Common/Cpp/Cryptography/SHA256.h" +#include "Pokemon_StoryEnginePolicy.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + + +StorySaveMode mode_of(StoryPhase phase){ + switch (phase){ + case StoryPhase::TESTING: + return StorySaveMode::NONE; + case StoryPhase::VERIFICATION: + return StorySaveMode::AFTER_VERIFICATION; + case StoryPhase::REPRODUCTION: + return StorySaveMode::PER_CHECKPOINT; + } + return StorySaveMode::NONE; +} + + +} // namespace + + +std::string story_steps_hash(const StoryCheckpoint& checkpoint){ + const std::string& text = checkpoint.steps_json; + SHA256 hash; + hash.push(text.data(), text.size()); + hash.finish(); + return std::format("sha256:{}", hash.get_hash_hex()); +} + +StorySaveDecision decide_story_save(const StoryCheckpoint& checkpoint, std::string_view steps_hash){ + const StoryCheckpointMetadata& metadata = checkpoint.metadata; + const std::string phase = story_phase_name(metadata.phase); + if (metadata.phase == StoryPhase::TESTING){ + return {.mode = StorySaveMode::NONE, .reason = std::format("[Save] {}: phase testing: save mode none", checkpoint.id)}; + } + if (!metadata.verified || metadata.verified->steps_hash != steps_hash){ + return { + .mode = StorySaveMode::NONE, + .reason = std::format( + "[Save] {}: phase {} but steps_hash mismatch (recorded {}, now {}): treated as testing, save mode none", + checkpoint.id, phase, metadata.verified ? metadata.verified->steps_hash : "none", steps_hash + ), + }; + } + const StorySaveMode mode = mode_of(metadata.phase); + return {.mode = mode, .reason = std::format("[Save] {}: phase {}: save mode {}", checkpoint.id, phase, story_save_mode_name(mode))}; +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.h b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.h new file mode 100644 index 0000000000..197fce0023 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.h @@ -0,0 +1,64 @@ +/* Story Engine Policy + * + * From: https://github.com/PokemonAutomation/ + * + * When an engine run of a checkpoint may save, from the checkpoint's + * metadata (autostory/SAVE_POLICY.md on the notes branch): a testing + * checkpoint never saves, a verification one saves once after its runs + * passed, a reproduction one at its end. A checkpoint whose steps no + * longer hash to the ones it was verified with is treated as testing: its + * verification was of other steps. + * + */ + +#ifndef PokemonAutomation_Pokemon_StoryEnginePolicy_H +#define PokemonAutomation_Pokemon_StoryEnginePolicy_H + +#include +#include +#include +#include +#include "Pokemon/AutoStory/Pokemon_StoryCheckpointTypes.h" +#include "Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.h" +#include "Pokemon/AutoStory/Types/Pokemon_StoryActionSpec.h" +#include "Pokemon/AutoStory/Types/Pokemon_StoryCheckpoint.h" +#include "Pokemon/AutoStory/Types/Pokemon_StorySaveMode.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// "sha256:" of `checkpoint`'s steps JSON as read from its file +// (StoryCheckpoint::steps_json): any edit to the steps, including a why +// or a default written out, changes it. +std::string story_steps_hash(const StoryCheckpoint& checkpoint); + +// A run's save mode and the log line that says why. +struct StorySaveDecision{ + StorySaveMode mode = StorySaveMode::NONE; + std::string reason = {}; +}; + +// The save mode of a run of `checkpoint`, whose steps hash to `steps_hash`. +StorySaveDecision decide_story_save(const StoryCheckpoint& checkpoint, std::string_view steps_hash); + +// An engine run of one checkpoint. +struct StoryEngineReport{ + // The steps' run. A checkpoint that failed its lint has none. + CheckpointReport run = {}; + StorySaveDecision save = {}; + // PASSED when the steps completed and no postcondition failed. + CheckpointVerdict verdict = CheckpointVerdict::FAILED; + // Checked after steps that completed. + std::optional postconditions = std::nullopt; + bool saved = false; + // A SAVE step among its steps saved: a reload now returns there, not + // to the save the checkpoint started from. + bool saved_part_way = false; +}; + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h index b549581b3a..b32b414303 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h @@ -22,6 +22,7 @@ #include "Pokemon_StoryEvents_Tests.h" #include "Pokemon_StoryFailureClasses_Tests.h" #include "Pokemon_StoryLog_Tests.h" +#include "Pokemon_StorySavePolicy_Tests.h" #include "Pokemon_StorySegments_Tests.h" #include "Pokemon_StoryStepExecutor_Tests.h" #include "Types/Pokemon_StoryActionArg_Tests.h" @@ -43,6 +44,7 @@ inline void add_tests_AutoStory(UnitTestDatabase& database){ add_tests_FrameCoverage(database); add_tests_StoryEngine(database); add_tests_StorySegments(database); + add_tests_StorySavePolicy(database); add_tests_SceneJudges(database); add_tests_StoryFiles(database); } diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.cpp new file mode 100644 index 0000000000..5b0e617f3b --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.cpp @@ -0,0 +1,227 @@ +/* Story Postcondition Check + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "Pokemon_StoryPostconditionCheck.h" +#include "Pokemon_StoryStepExecutor.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + +std::string verdict_name(PostconditionVerdict verdict){ + switch (verdict){ + case PostconditionVerdict::PASSED: + return "passed"; + case PostconditionVerdict::FAILED: + return "failed"; + case PostconditionVerdict::NOT_CHECKED: + return "not checked"; + } + return "?"; +} + +PostconditionResult result(std::string what, PostconditionVerdict verdict, std::string detail){ + return PostconditionResult{.what = std::move(what), .verdict = verdict, .detail = std::move(detail)}; +} + +PostconditionResult not_read(std::string what){ + return result(std::move(what), PostconditionVerdict::NOT_CHECKED, "the game can't read it"); +} + +// The first pose among a step's args (a route's target). +std::optional pose_arg(const StoryStep& step){ + for (const auto& [unused, value] : step.args){ + if (const GridPose* pose = std::get_if(&value)){ + return *pose; + } + } + return std::nullopt; +} + +std::string tiles_text(const std::vector& tiles){ + std::string text; + for (const TilePos& tile : tiles){ + text += (text.empty() ? "" : " or ") + to_string(tile); + } + return text; +} + +void check_map(const GridPose& end, const ObservedState& observed, std::vector& out){ + if (!observed.map){ + out.push_back(not_read("map")); + return; + } + const bool same = *observed.map == end.map; + out.push_back(result( + "map", same ? PostconditionVerdict::PASSED : PostconditionVerdict::FAILED, + same ? end.map : std::format("expected {}, read {}", end.map, *observed.map) + )); +} + +// The tile and facing mean nothing on another map: not checked then. +void check_tile(const GridPose& end, const ObservedState& observed, bool map_failed, std::vector& out){ + if (end.tiles.empty()){ + return; + } + if (map_failed){ + out.push_back(result("tile", PostconditionVerdict::NOT_CHECKED, "on another map")); + return; + } + if (!observed.tile){ + out.push_back(not_read("tile")); + return; + } + const bool on = std::find(end.tiles.begin(), end.tiles.end(), *observed.tile) != end.tiles.end(); + out.push_back(result( + "tile", on ? PostconditionVerdict::PASSED : PostconditionVerdict::FAILED, + on ? to_string(*observed.tile) : std::format("expected {}, read {}", tiles_text(end.tiles), to_string(*observed.tile)) + )); +} + +void check_facing(const GridPose& end, const ObservedState& observed, bool map_failed, std::vector& out){ + if (!end.facing){ + return; + } + if (map_failed){ + out.push_back(result("facing", PostconditionVerdict::NOT_CHECKED, "on another map")); + return; + } + if (!observed.facing){ + out.push_back(not_read("facing")); + return; + } + const bool same = *observed.facing == *end.facing; + out.push_back(result( + "facing", same ? PostconditionVerdict::PASSED : PostconditionVerdict::FAILED, + same ? direction_name(*end.facing) + : std::format("expected {}, read {}", direction_name(*end.facing), direction_name(*observed.facing)) + )); +} + +void check_position(const StoryCheckpoint& checkpoint, const ObservedState& observed, std::vector& out){ + const std::optional end = checkpoint_end_pose(checkpoint); + if (!end){ + return; + } + check_map(*end, observed, out); + const bool map_failed = out.back().verdict == PostconditionVerdict::FAILED; + check_tile(*end, observed, map_failed, out); + check_facing(*end, observed, map_failed, out); +} + +void check_items(const StoryPostconditions& expected, const ObservedState& observed, std::vector& out){ + for (const StoryItem& item : expected.items){ + const std::string what = std::format("item {}", item.item); + const auto read = observed.items.find(item.item); + if (read == observed.items.end()){ + out.push_back(not_read(what)); + continue; + } + const bool same = read->second == item.quantity; + out.push_back(result( + what, same ? PostconditionVerdict::PASSED : PostconditionVerdict::FAILED, + same ? std::format("{}", item.quantity) : std::format("expected {}, read {}", item.quantity, read->second) + )); + } +} + +void check_flags(const StoryPostconditions& expected, const ObservedState& observed, std::vector& out){ + for (const std::string& flag : expected.flags){ + const std::string what = std::format("flag {}", flag); + const auto read = observed.flags.find(flag); + if (read == observed.flags.end()){ + out.push_back(not_read(what)); + continue; + } + out.push_back(result( + what, read->second ? PostconditionVerdict::PASSED : PostconditionVerdict::FAILED, + read->second ? "set" : "expected set, read clear" + )); + } +} + +std::string summary_verdict(const PostconditionReport& report){ + if (report.results.empty()){ + return "nothing to check"; + } + if (report.count(PostconditionVerdict::FAILED) > 0){ + return "failed"; + } + return report.all_passed() ? "passed" : "incomplete"; +} + +} // namespace + + +size_t PostconditionReport::count(PostconditionVerdict verdict) const{ + size_t total = 0; + for (const PostconditionResult& item : results){ + total += item.verdict == verdict ? 1 : 0; + } + return total; +} + +bool PostconditionReport::all_passed() const{ + return !results.empty() && count(PostconditionVerdict::PASSED) == results.size(); +} + + +std::optional checkpoint_end_pose(const StoryCheckpoint& checkpoint){ + if (checkpoint.postconditions.position){ + return checkpoint.postconditions.position; + } + for (auto step = checkpoint.steps.rbegin(); step != checkpoint.steps.rend(); ++step){ + if (step->pose_after){ + return grid_pose_of(*step->pose_after); + } + if (std::optional target = pose_arg(*step)){ + return target; + } + // Taps and single steps move the player without saying where to. + if (step->kind == StoryStepKind::LEGS || step->kind == StoryStepKind::STEP){ + return std::nullopt; + } + } + return std::nullopt; +} + +PostconditionReport check_postconditions(const StoryCheckpoint& checkpoint, const ObservedState& observed){ + PostconditionReport report; + report.checkpoint = checkpoint.id; + check_position(checkpoint, observed, report.results); + check_items(checkpoint.postconditions, observed, report.results); + check_flags(checkpoint.postconditions, observed, report.results); + return report; +} + +std::vector postcondition_log_lines(const PostconditionReport& report){ + std::vector lines; + for (const PostconditionResult& item : report.results){ + lines.push_back(std::format( + "[Postcondition] {} {}: {}: {}", report.checkpoint, item.what, verdict_name(item.verdict), item.detail + )); + } + lines.push_back(std::format( + "[Postcondition] {}: {} passed, {} failed, {} not checked: {}", + report.checkpoint, + report.count(PostconditionVerdict::PASSED), + report.count(PostconditionVerdict::FAILED), + report.count(PostconditionVerdict::NOT_CHECKED), + summary_verdict(report) + )); + return lines; +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.h new file mode 100644 index 0000000000..c04c0e8e7e --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.h @@ -0,0 +1,89 @@ +/* Story Postcondition Check + * + * From: https://github.com/PokemonAutomation/ + * + * A checkpoint's postconditions (Types/Pokemon_StoryPostconditions.h) + * checked against what the game read once the run was over: a step of its + * own, not the run's verdict, which can be wrong. A postcondition the game + * can't read yet is reported as not checked, never as passed. + * + */ + +#ifndef PokemonAutomation_Pokemon_StoryPostconditionCheck_H +#define PokemonAutomation_Pokemon_StoryPostconditionCheck_H + +#include +#include +#include +#include +#include +#include +#include "Pokemon/AutoStory/Types/Pokemon_GridPose.h" +#include "Pokemon/AutoStory/Types/Pokemon_StoryCheckpoint.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// What the game read after the run. Each part is unset (or, for items and +// flags, absent from the map) when the game can't read it: the matching +// postconditions are then not checked. +struct ObservedState{ + std::optional map; + std::optional tile; + std::optional facing; + // The Bag's count of each item read, 0 for one read and not there. + std::map> items; + // Each story flag read, and whether it is set. + std::map> flags; +}; + +// The result of one postcondition. +enum class PostconditionVerdict{ + PASSED, + FAILED, + NOT_CHECKED, +}; + +// One postcondition checked: "map", "tile", "facing", "item LIFT_KEY", +// "flag ...", its verdict and what was expected and read. +struct PostconditionResult{ + std::string what; + PostconditionVerdict verdict = PostconditionVerdict::NOT_CHECKED; + std::string detail; +}; + +// Every postcondition of one checkpoint, checked. +struct PostconditionReport{ + std::string checkpoint; + std::vector results; + + size_t count(PostconditionVerdict verdict) const; + // Every postcondition was read and held: none failed, none unchecked, + // and there was at least one. + bool all_passed() const; +}; + +// Where `checkpoint` ends: its postconditions' position, else the last +// step's declared pose or target. Unset when a step after that moves the +// player in a way the plan does not place (taps of legs, a single step). +std::optional checkpoint_end_pose(const StoryCheckpoint& checkpoint); + +// Checks the end position (map, tile among the pose's tiles, facing), +// then each item count and flag, against `observed`. +PostconditionReport check_postconditions(const StoryCheckpoint& checkpoint, const ObservedState& observed); + +// The report as log lines, one per postcondition and a summary: +// [Postcondition] 00.31 map: passed: RocketHideout_B3F +// [Postcondition] 00.31 item RARE_CANDY: failed: expected 2, read 1 +// [Postcondition] 00.31 flag hideout_lift_key_grunt: not checked: the game can't read it +// [Postcondition] 00.31: 3 passed, 1 failed, 1 not checked: failed +// The summary is "passed" only when every postcondition passed; +// "incomplete" when none failed and some were not checked. +std::vector postcondition_log_lines(const PostconditionReport& report); + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.cpp new file mode 100644 index 0000000000..d2bc64502d --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.cpp @@ -0,0 +1,508 @@ +/* Story Save Policy Tests + * + * From: https://github.com/PokemonAutomation/ + * + * When a run saves (Types/Pokemon_StorySaveMode.h): the decision table, + * then the segment runner and verify mode in each mode against a world + * that only records what happened. Then the postcondition check + * (Pokemon_StoryPostconditionCheck.h) on FRLG 00.31's end state. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Json/JsonValue.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "Data/Pokemon_StoryStepJson.h" +#include "Pokemon_StoryPostconditionCheck.h" +#include "Pokemon_StorySavePolicy_Tests.h" +#include "Pokemon_StorySegments.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + + +// One row of the decision table. +struct SaveDecision{ + StorySaveMode mode; + CheckpointVerdict verdict; + SavePoint point; + bool saves; +}; + +constexpr SaveDecision SAVE_DECISIONS[] = { + {StorySaveMode::NONE, CheckpointVerdict::FAILED, SavePoint::CHECKPOINT_END, false}, + {StorySaveMode::NONE, CheckpointVerdict::FAILED, SavePoint::RUN_END, false}, + {StorySaveMode::NONE, CheckpointVerdict::PASSED, SavePoint::CHECKPOINT_END, false}, + {StorySaveMode::NONE, CheckpointVerdict::PASSED, SavePoint::RUN_END, false}, + {StorySaveMode::AFTER_VERIFICATION, CheckpointVerdict::FAILED, SavePoint::CHECKPOINT_END, false}, + {StorySaveMode::AFTER_VERIFICATION, CheckpointVerdict::FAILED, SavePoint::RUN_END, false}, + {StorySaveMode::AFTER_VERIFICATION, CheckpointVerdict::PASSED, SavePoint::CHECKPOINT_END, false}, + {StorySaveMode::AFTER_VERIFICATION, CheckpointVerdict::PASSED, SavePoint::RUN_END, true}, + {StorySaveMode::PER_CHECKPOINT, CheckpointVerdict::FAILED, SavePoint::CHECKPOINT_END, false}, + {StorySaveMode::PER_CHECKPOINT, CheckpointVerdict::FAILED, SavePoint::RUN_END, false}, + {StorySaveMode::PER_CHECKPOINT, CheckpointVerdict::PASSED, SavePoint::CHECKPOINT_END, true}, + {StorySaveMode::PER_CHECKPOINT, CheckpointVerdict::PASSED, SavePoint::RUN_END, true}, +}; + +UnitTestResult test_decision_table(){ + for (const SaveDecision& row : SAVE_DECISIONS){ + if (story_save_allowed(row.mode, row.verdict, row.point) != row.saves){ + return std::format( + "Wrong decision for {}{}{}", story_save_mode_name(row.mode), + row.verdict == CheckpointVerdict::PASSED ? ", passed" : ", failed", + row.point == SavePoint::RUN_END ? ", run end" : ", mid-run" + ); + } + } + return true; +} + + +// A world of three checkpoints that only records what happened: each +// checkpoint's walk, each save and each reload. +struct Lab{ + std::vector trace; + std::vector lines; + // Checkpoints whose walk fails this many more times. + std::map failures; + // The game's postcondition check: none when unset. + std::optional postconditions; + + SegmentHooks hooks(){ + SegmentHooks hooks; + hooks.save = [this](RunState&){ + trace.push_back("save"); + return outcome_ok(); + }; + hooks.reload = [this](RunState&){ + trace.push_back("reload"); + return outcome_ok(); + }; + hooks.log = [this](const std::string& line){ lines.push_back(line); }; + if (postconditions){ + hooks.check_postconditions = [this](const std::string& id, RunState&){ + trace.push_back(std::format("check {}", id)); + return *postconditions; + }; + } + return hooks; + } + bool has(const std::string& line) const{ + return std::find(lines.begin(), lines.end(), line) != lines.end(); + } + std::string all() const{ + std::string text; + for (const std::string& line : lines){ + text += std::format("\n{}", line); + } + return text; + } +}; + +CheckpointSpec walk_row(std::string id, Lab& lab){ + CheckpointSpec spec; + spec.id = id; + spec.saves = true; + spec.build = [&lab, id]{ + Checkpoint checkpoint; + checkpoint.id = id; + Event walk; + walk.name = "walk"; + walk.act = [&lab, id](RunState&){ + lab.trace.push_back(id); + int& left = lab.failures[id]; + if (left > 0){ + left--; + return outcome_failed("blocked", "a wanderer on the path"); + } + return outcome_ok(); + }; + Recovery stop; + stop.kind = RecoveryKind::STOP; + walk.recoveries.push_back(std::move(stop)); + checkpoint.events.push_back(std::move(walk)); + return checkpoint; + }; + return spec; +} + +Segment three_rows(Lab& lab){ + Segment segment; + segment.id = "00"; + for (std::string_view id : {"00.01", "00.02", "00.03"}){ + segment.checkpoints.push_back(walk_row(std::string(id), lab)); + } + return segment; +} + +SegmentReport run_in(StorySaveMode mode, Lab& lab){ + Segment segment = three_rows(lab); + SegmentHooks hooks = lab.hooks(); + RunState state; + return run_segment(SegmentRange{}, SegmentPolicy{.save_mode = mode}, segment, hooks, state); +} + +UnitTestResult test_none_never_saves(){ + Lab lab; + const SegmentReport report = run_in(StorySaveMode::NONE, lab); + const std::vector expected = {"00.01", "00.02", "00.03"}; + if (!report.completed || lab.trace != expected){ + return std::format("Expected three checkpoints and no save:{}", lab.all()); + } + if (!lab.has("[Segment] 00: save mode none") || !lab.has("[Checkpoint] 00.02: not saving (save mode none)")){ + return std::format("Expected the skipped saves logged:{}", lab.all()); + } + return true; +} + +UnitTestResult test_per_checkpoint_saves_each(){ + Lab lab; + const SegmentReport report = run_in(StorySaveMode::PER_CHECKPOINT, lab); + const std::vector expected = {"00.01", "save", "00.02", "save", "00.03", "save"}; + if (!report.completed || lab.trace != expected){ + return std::format("Expected a save after each checkpoint:{}", lab.all()); + } + return true; +} + +UnitTestResult test_after_verification_saves_at_the_end(){ + Lab lab; + const SegmentReport report = run_in(StorySaveMode::AFTER_VERIFICATION, lab); + const std::vector expected = {"00.01", "00.02", "00.03", "save"}; + if (!report.completed || lab.trace != expected){ + return std::format("Expected one save, at the end:{}", lab.all()); + } + if (!lab.has("[Postcondition] 00.03: not checked: the game has no postcondition check")){ + return std::format("Expected the missing check logged as not checked:{}", lab.all()); + } + return true; +} + +UnitTestResult test_after_verification_checks_before_saving(){ + Lab lab; + lab.postconditions = outcome_ok(); + const SegmentReport report = run_in(StorySaveMode::AFTER_VERIFICATION, lab); + const std::vector expected = {"00.01", "00.02", "00.03", "check 00.03", "save"}; + if (!report.completed || lab.trace != expected){ + return std::format("Expected the check, then the save:{}", lab.all()); + } + return true; +} + +// FRLG 00.31, 2026-11-10: the run said it passed on the wrong floor. +UnitTestResult test_failed_postconditions_never_save(){ + Lab lab; + lab.postconditions = outcome_failed("postconditions", "map: expected RocketHideout_B3F, read RocketHideout_B4F"); + const SegmentReport report = run_in(StorySaveMode::AFTER_VERIFICATION, lab); + if (report.completed || report.failed_checkpoint != "00.03"){ + return std::format("Expected the run failed at its last checkpoint:{}", lab.all()); + } + if (std::find(lab.trace.begin(), lab.trace.end(), "save") != lab.trace.end()){ + return std::format("A run whose postconditions failed must not save:{}", lab.all()); + } + if (!lab.has("[Checkpoint] 00.03: postconditions failed, not saving")){ + return std::format("Expected the refused save logged:{}", lab.all()); + } + return true; +} + +// Without per-checkpoint saves, a reload would go back to the run's start. +UnitTestResult test_no_reload_after_the_first(){ + Lab lab; + lab.failures["00.02"] = 1; + const SegmentReport report = run_in(StorySaveMode::NONE, lab); + const std::vector expected = {"00.01", "00.02"}; + if (report.completed || report.reloads != 0 || lab.trace != expected){ + return std::format("Expected 00.02 to fail without a reload:{}", lab.all()); + } + if (!lab.has("[Segment] 00: 00.02 failed; save mode none left no save at its start, so no reload")){ + return std::format("Expected why it didn't reload:{}", lab.all()); + } + return true; +} + +UnitTestResult test_first_row_still_reloads(){ + Lab lab; + lab.failures["00.01"] = 1; + const SegmentReport report = run_in(StorySaveMode::NONE, lab); + const std::vector expected = {"00.01", "reload", "00.01", "00.02", "00.03"}; + if (!report.completed || report.reloads != 1 || lab.trace != expected){ + return std::format("Expected the first checkpoint retried from the start save:{}", lab.all()); + } + return true; +} + +VerifyReport verify_in(StorySaveMode mode, Lab& lab){ + Segment segment = three_rows(lab); + SegmentHooks hooks = lab.hooks(); + RunState state; + const VerifyPolicy policy{.runs = 3, .passes_needed = 2, .save_mode = mode}; + return verify_checkpoint("00.02", policy, segment, hooks, state); +} + +UnitTestResult test_verify_saves_once_every_run_passed(){ + Lab lab; + const VerifyReport report = verify_in(StorySaveMode::AFTER_VERIFICATION, lab); + const std::vector expected = {"00.02", "reload", "00.02", "reload", "00.02", "save"}; + if (!report.verified || !report.saved || lab.trace != expected){ + return std::format("Expected one save, after the last run:{}", lab.all()); + } + if (!lab.has("[Verify] 00.02: every run passed: saved at the end state")){ + return std::format("Expected the save logged:{}", lab.all()); + } + return true; +} + +UnitTestResult test_verify_no_save_after_a_failed_run(){ + Lab lab; + lab.failures["00.02"] = 1; + const VerifyReport report = verify_in(StorySaveMode::AFTER_VERIFICATION, lab); + if (!report.verified || report.saved || std::find(lab.trace.begin(), lab.trace.end(), "save") != lab.trace.end()){ + return std::format("Verified 2/3, but a failed run means no save:{}", lab.all()); + } + if (!lab.has("[Verify] 00.02: not every run passed: not saving")){ + return std::format("Expected why it didn't save:{}", lab.all()); + } + return true; +} + + +// FRLG 00.31's end, in the shared format: beside B3F's Rare Candy, with +// the LIFT KEY and both Rare Candies. +inline constexpr std::string_view CHECKPOINT_0031 = R"({ + "id": "00.31", + "title": "Rocket Hideout to the LIFT KEY", + "start": {"map": "CeladonCity_PokemonCenter_1F", "tiles": [[7, 4]], "facing": "north"}, + "events": [ + {"id": "beside_b3f_rare_candy", "kind": "route", "args": {"target": {"map": "RocketHideout_B3F", "tiles": [[12, 13]], "facing": "north"}}}, + {"id": "b3f_rare_candy", "kind": "talk"} + ], + "metadata": {"phase": "testing", "postconditions": { + "position": {"map": "RocketHideout_B3F", "tiles": [[12, 13]], "facing": "north"}, + "items": [{"item": "LIFT_KEY", "quantity": 1}, {"item": "RARE_CANDY", "quantity": 2}] + }} +})"; + +StoryCheckpoint checkpoint_0031(){ + return read_story_checkpoint(parse_json(std::string(CHECKPOINT_0031)), "00.31"); +} + +ObservedState read_0031(std::string map, int candies){ + ObservedState observed; + observed.map = std::move(map); + observed.tile = TilePos{12, 13}; + observed.facing = Direction::NORTH; + observed.items = {{"LIFT_KEY", 1}, {"RARE_CANDY", (size_t)candies}}; + return observed; +} + +UnitTestResult test_postconditions_json(){ + const StoryCheckpoint checkpoint = checkpoint_0031(); + const StoryPostconditions& post = checkpoint.postconditions; + if (!post.position || post.position->map != "RocketHideout_B3F" || post.items.size() != 2 + || post.items[1] != StoryItem{.item = "RARE_CANDY", .quantity = 2} + ){ + return "Unexpected postconditions read."; + } + if (post.flags.size() != 0 || post.position->tiles != std::vector{TilePos{12, 13}}){ + return "Unexpected postcondition position or flags read."; + } + return true; +} + +UnitTestResult test_postconditions_pass(){ + const PostconditionReport report = check_postconditions(checkpoint_0031(), read_0031("RocketHideout_B3F", 2)); + const std::vector lines = postcondition_log_lines(report); + if (!report.all_passed() || lines.back() != "[Postcondition] 00.31: 5 passed, 0 failed, 0 not checked: passed"){ + return std::format("Expected every postcondition passed: {}", lines.back()); + } + return true; +} + +// The 2026-11-10 false pass: still on B4F, the Rare Candy not picked up. +UnitTestResult test_postconditions_fail(){ + const PostconditionReport report = check_postconditions(checkpoint_0031(), read_0031("RocketHideout_B4F", 1)); + const std::vector lines = postcondition_log_lines(report); + const std::vector expected = { + "[Postcondition] 00.31 map: failed: expected RocketHideout_B3F, read RocketHideout_B4F", + "[Postcondition] 00.31 tile: not checked: on another map", + "[Postcondition] 00.31 facing: not checked: on another map", + "[Postcondition] 00.31 item LIFT_KEY: passed: 1", + "[Postcondition] 00.31 item RARE_CANDY: failed: expected 2, read 1", + "[Postcondition] 00.31: 1 passed, 2 failed, 2 not checked: failed", + }; + if (lines != expected){ + std::string text; + for (const std::string& line : lines){ + text += std::format("\n{}", line); + } + return std::format("Unexpected lines:{}", text); + } + return true; +} + +UnitTestResult test_postconditions_not_checked(){ + StoryCheckpoint checkpoint = checkpoint_0031(); + checkpoint.postconditions.flags = {"hideout_lift_key_grunt"}; + ObservedState observed; + observed.map = "RocketHideout_B3F"; + const PostconditionReport report = check_postconditions(checkpoint, observed); + const std::vector lines = postcondition_log_lines(report); + if (report.all_passed() || report.count(PostconditionVerdict::NOT_CHECKED) != 5 + || lines.back() != "[Postcondition] 00.31: 1 passed, 0 failed, 5 not checked: incomplete" + ){ + return std::format("What isn't read is not checked, never passed: {}", lines.back()); + } + observed.flags["hideout_lift_key_grunt"] = false; + if (check_postconditions(checkpoint, observed).count(PostconditionVerdict::FAILED) != 1){ + return "A flag read clear should fail."; + } + return true; +} + +UnitTestResult test_end_pose_from_the_plan(){ + StoryCheckpoint checkpoint = checkpoint_0031(); + checkpoint.postconditions.position.reset(); + const std::optional end = checkpoint_end_pose(checkpoint); + if (!end || end->map != "RocketHideout_B3F" || end->tiles != std::vector{TilePos{12, 13}}){ + return "Expected the last route's target as the end."; + } + StoryCheckpoint stepped = checkpoint; + StoryStep step; + step.kind = StoryStepKind::STEP; + step.name = "one more"; + stepped.steps.push_back(std::move(step)); + if (checkpoint_end_pose(stepped)){ + return "A step after the last target leaves the end unknown."; + } + StoryStep legs; + legs.kind = StoryStepKind::LEGS; + legs.name = "taps"; + checkpoint.steps.push_back(std::move(legs)); + if (checkpoint_end_pose(checkpoint)){ + return "Legs after the last target leave the end unknown."; + } + if (!check_postconditions(checkpoint, read_0031("RocketHideout_B3F", 2)).all_passed()){ + return "Without an end pose, the items alone should be checked."; + } + return true; +} + + +// A save-policy scenario and its expected result. +struct SavePolicyCase{ + // The scenario to run. + std::string name; + struct Input{ + UnitTestResult (*test)() = nullptr; + }; + + // What the scenario checks; it returns its own failure. + struct Expectation{ + std::string_view description; + }; + + Input input; + Expectation expectation; +}; + +class Test_StorySavePolicy : public UnitTest{ +public: + explicit Test_StorySavePolicy(SavePolicyCase test_case) + : UnitTest(std::format("Pokemon::StorySavePolicy - {}", test_case.name)) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return m_case.input.test(); + } + +private: + SavePolicyCase m_case; +}; + +const SavePolicyCase SAVE_POLICY_CASES[] = { + { + .name = "decision table", .input = {.test = test_decision_table}, + .expectation = {.description = "mode x verdict x save point, all twelve rows"}, + }, + { + .name = "none never saves", .input = {.test = test_none_never_saves}, + .expectation = {.description = "three passing checkpoints, no save, each skip logged"}, + }, + { + .name = "per checkpoint saves each", .input = {.test = test_per_checkpoint_saves_each}, + .expectation = {.description = "a save after every passing checkpoint"}, + }, + { + .name = "after verification saves at the end", .input = {.test = test_after_verification_saves_at_the_end}, + .expectation = {.description = "one save after the last checkpoint; no check logged as not checked"}, + }, + { + .name = "after verification checks before saving", .input = {.test = test_after_verification_checks_before_saving}, + .expectation = {.description = "the game's postcondition check runs before the save"}, + }, + { + .name = "failed postconditions never save", .input = {.test = test_failed_postconditions_never_save}, + .expectation = {.description = "a failed check fails the run and refuses the save"}, + }, + { + .name = "no reload after the first checkpoint", .input = {.test = test_no_reload_after_the_first}, + .expectation = {.description = "a later checkpoint has no save to reload, so it fails at once"}, + }, + { + .name = "the first checkpoint still reloads", .input = {.test = test_first_row_still_reloads}, + .expectation = {.description = "the first checkpoint reloads the run's start save"}, + }, + { + .name = "verify saves once every run passed", .input = {.test = test_verify_saves_once_every_run_passed}, + .expectation = {.description = "3/3 passed: one save at the end of run 3"}, + }, + { + .name = "verify: no save after a failed run", .input = {.test = test_verify_no_save_after_a_failed_run}, + .expectation = {.description = "2/3 verified, but no save"}, + }, + { + .name = "postconditions: json", .input = {.test = test_postconditions_json}, + .expectation = {.description = "00.31's postconditions read and written back"}, + }, + { + .name = "postconditions: pass", .input = {.test = test_postconditions_pass}, + .expectation = {.description = "B3F (12, 13) north, LIFT_KEY 1, RARE_CANDY 2: passed"}, + }, + { + .name = "postconditions: fail", .input = {.test = test_postconditions_fail}, + .expectation = {.description = "still on B4F with 1 Rare Candy: failed"}, + }, + { + .name = "postconditions: not checked", .input = {.test = test_postconditions_not_checked}, + .expectation = {.description = "unread parts are not checked, never passed"}, + }, + { + .name = "postconditions: end pose from the plan", .input = {.test = test_end_pose_from_the_plan}, + .expectation = {.description = "the last route's target, or unknown after legs"}, + }, +}; + +} // namespace + + +void add_tests_StorySavePolicy(UnitTestDatabase& database){ + for (const SavePolicyCase& test_case : SAVE_POLICY_CASES){ + database.add(test_case); + } +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.h b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.h new file mode 100644 index 0000000000..4e95d5e81a --- /dev/null +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.h @@ -0,0 +1,23 @@ +/* Story Save Policy Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_Pokemon_StorySavePolicy_Tests_H +#define PokemonAutomation_Pokemon_StorySavePolicy_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// Registers unit tests for when a run saves (save modes) and the postcondition check. +void add_tests_StorySavePolicy(UnitTestDatabase& database); + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 50abf05d48..63f1370f5f 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1486,6 +1486,8 @@ file(GLOB LIBRARY_SOURCES Source/Pokemon/AutoStory/Engine/Pokemon_StoryBuiltinActions.h Source/Pokemon/AutoStory/Engine/Pokemon_StoryEngine_Tests.cpp Source/Pokemon/AutoStory/Engine/Pokemon_StoryEngine_Tests.h + Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.cpp + Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.h Source/Pokemon/AutoStory/Engine/Pokemon_StoryFakeBackend.cpp Source/Pokemon/AutoStory/Engine/Pokemon_StoryFakeBackend.h Source/Pokemon/AutoStory/Engine/Pokemon_StoryGame.h @@ -1523,6 +1525,10 @@ file(GLOB LIBRARY_SOURCES Source/Pokemon/AutoStory/Pokemon_StoryOutcomeClass.cpp Source/Pokemon/AutoStory/Pokemon_StoryOutcomeClass.h Source/Pokemon/AutoStory/Pokemon_StoryPose.h + Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.cpp + Source/Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.h + Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.cpp + Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.h Source/Pokemon/AutoStory/Pokemon_StorySegments.cpp Source/Pokemon/AutoStory/Pokemon_StorySegments.h Source/Pokemon/AutoStory/Pokemon_StorySegments_Tests.cpp From 561b4995822133b587735b7a6ffba0dedab50aff Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 15 Nov 2026 01:01:01 -0800 Subject: [PATCH 2/3] AutoStory: the save policy suite tests this PR's policy The decision-table and segment save tests now live with the code they test; this suite gains the steps hash cases and a save decision table (phase by recorded hash). A checkpoint that was never verified gets its own reason instead of a hash mismatch against "none". Comments describe the faults, not our runs or notes. --- .../Engine/Pokemon_StoryEnginePolicy.cpp | 10 +- .../Engine/Pokemon_StoryEnginePolicy.h | 2 +- .../Pokemon_StorySavePolicy_Tests.cpp | 451 ++++++------------ 3 files changed, 146 insertions(+), 317 deletions(-) diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.cpp index 4b0a6902a1..5276973de5 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.cpp +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.cpp @@ -46,12 +46,18 @@ StorySaveDecision decide_story_save(const StoryCheckpoint& checkpoint, std::stri if (metadata.phase == StoryPhase::TESTING){ return {.mode = StorySaveMode::NONE, .reason = std::format("[Save] {}: phase testing: save mode none", checkpoint.id)}; } - if (!metadata.verified || metadata.verified->steps_hash != steps_hash){ + if (!metadata.verified){ + return { + .mode = StorySaveMode::NONE, + .reason = std::format("[Save] {}: phase {} but never verified: treated as testing, save mode none", checkpoint.id, phase), + }; + } + if (metadata.verified->steps_hash != steps_hash){ return { .mode = StorySaveMode::NONE, .reason = std::format( "[Save] {}: phase {} but steps_hash mismatch (recorded {}, now {}): treated as testing, save mode none", - checkpoint.id, phase, metadata.verified ? metadata.verified->steps_hash : "none", steps_hash + checkpoint.id, phase, metadata.verified->steps_hash, steps_hash ), }; } diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.h b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.h index 197fce0023..ac71834570 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.h +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.h @@ -3,7 +3,7 @@ * From: https://github.com/PokemonAutomation/ * * When an engine run of a checkpoint may save, from the checkpoint's - * metadata (autostory/SAVE_POLICY.md on the notes branch): a testing + * metadata: a testing * checkpoint never saves, a verification one saves once after its runs * passed, a reproduction one at its end. A checkpoint whose steps no * longer hash to the ones it was verified with is treated as testing: its diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.cpp index d2bc64502d..f9ca4c269d 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.cpp +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.cpp @@ -2,16 +2,14 @@ * * From: https://github.com/PokemonAutomation/ * - * When a run saves (Types/Pokemon_StorySaveMode.h): the decision table, - * then the segment runner and verify mode in each mode against a world - * that only records what happened. Then the postcondition check - * (Pokemon_StoryPostconditionCheck.h) on FRLG 00.31's end state. + * The engine's save policy (Engine/Pokemon_StoryEnginePolicy.h): the + * steps hash and the save mode a checkpoint's phase and recorded hash + * give. Then the postcondition check (Pokemon_StoryPostconditionCheck.h) + * on a checkpoint's end state. * */ -#include #include -#include #include #include #include @@ -20,9 +18,9 @@ #include "Common/Cpp/Json/JsonValue.h" #include "Common/Cpp/TestRunners/UnitTest.h" #include "Data/Pokemon_StoryStepJson.h" +#include "Engine/Pokemon_StoryEnginePolicy.h" #include "Pokemon_StoryPostconditionCheck.h" #include "Pokemon_StorySavePolicy_Tests.h" -#include "Pokemon_StorySegments.h" namespace PokemonAutomation{ namespace Pokemon{ @@ -31,254 +29,138 @@ namespace Pokemon{ namespace{ -// One row of the decision table. -struct SaveDecision{ - StorySaveMode mode; - CheckpointVerdict verdict; - SavePoint point; - bool saves; -}; - -constexpr SaveDecision SAVE_DECISIONS[] = { - {StorySaveMode::NONE, CheckpointVerdict::FAILED, SavePoint::CHECKPOINT_END, false}, - {StorySaveMode::NONE, CheckpointVerdict::FAILED, SavePoint::RUN_END, false}, - {StorySaveMode::NONE, CheckpointVerdict::PASSED, SavePoint::CHECKPOINT_END, false}, - {StorySaveMode::NONE, CheckpointVerdict::PASSED, SavePoint::RUN_END, false}, - {StorySaveMode::AFTER_VERIFICATION, CheckpointVerdict::FAILED, SavePoint::CHECKPOINT_END, false}, - {StorySaveMode::AFTER_VERIFICATION, CheckpointVerdict::FAILED, SavePoint::RUN_END, false}, - {StorySaveMode::AFTER_VERIFICATION, CheckpointVerdict::PASSED, SavePoint::CHECKPOINT_END, false}, - {StorySaveMode::AFTER_VERIFICATION, CheckpointVerdict::PASSED, SavePoint::RUN_END, true}, - {StorySaveMode::PER_CHECKPOINT, CheckpointVerdict::FAILED, SavePoint::CHECKPOINT_END, false}, - {StorySaveMode::PER_CHECKPOINT, CheckpointVerdict::FAILED, SavePoint::RUN_END, false}, - {StorySaveMode::PER_CHECKPOINT, CheckpointVerdict::PASSED, SavePoint::CHECKPOINT_END, true}, - {StorySaveMode::PER_CHECKPOINT, CheckpointVerdict::PASSED, SavePoint::RUN_END, true}, -}; +struct HashCase{ + // Two checkpoints' events, hashed with the built-in actions. + struct Input{ + std::string_view a; + std::string_view b; + }; -UnitTestResult test_decision_table(){ - for (const SaveDecision& row : SAVE_DECISIONS){ - if (story_save_allowed(row.mode, row.verdict, row.point) != row.saves){ - return std::format( - "Wrong decision for {}{}{}", story_save_mode_name(row.mode), - row.verdict == CheckpointVerdict::PASSED ? ", passed" : ", failed", - row.point == SavePoint::RUN_END ? ", run end" : ", mid-run" - ); - } - } - return true; -} + // Whether they hash the same, and `a`'s hash when pinned: a change to it + // means every recorded verification goes stale. + struct Expectation{ + bool same = true; + std::optional hash_a = std::nullopt; + }; + std::string_view name; + Input input; + Expectation expectation; +}; -// A world of three checkpoints that only records what happened: each -// checkpoint's walk, each save and each reload. -struct Lab{ - std::vector trace; - std::vector lines; - // Checkpoints whose walk fails this many more times. - std::map failures; - // The game's postcondition check: none when unset. - std::optional postconditions; - - SegmentHooks hooks(){ - SegmentHooks hooks; - hooks.save = [this](RunState&){ - trace.push_back("save"); - return outcome_ok(); - }; - hooks.reload = [this](RunState&){ - trace.push_back("reload"); - return outcome_ok(); - }; - hooks.log = [this](const std::string& line){ lines.push_back(line); }; - if (postconditions){ - hooks.check_postconditions = [this](const std::string& id, RunState&){ - trace.push_back(std::format("check {}", id)); - return *postconditions; - }; - } - return hooks; - } - bool has(const std::string& line) const{ - return std::find(lines.begin(), lines.end(), line) != lines.end(); - } - std::string all() const{ - std::string text; - for (const std::string& line : lines){ - text += std::format("\n{}", line); - } - return text; - } +inline constexpr std::string_view ROUTE_B3F = R"([{"id": "go", "kind": "route", "args": {"target": {"map": "RocketHideout_B3F", "tiles": [[12, 13]]}}}])"; + +const HashCase HASH_CASES[] = { + {.name = "pinned", .input = {.a = ROUTE_B3F, .b = ROUTE_B3F}, + .expectation = {.same = true, .hash_a = "sha256:13bc877c960a15cd0ed2f37248fa8336308b048a865623dde7654d41ac7147b8"}}, + {.name = "whitespace is not hashed", .input = {.a = R"([{"id": "t", "kind": "talk"}])", .b = R"([ { "id":"t", "kind":"talk" } ])"}, + .expectation = {.same = true}}, + {.name = "why differs", .input = {.a = R"([{"id": "t", "kind": "talk", "why": "a"}])", .b = R"([{"id": "t", "kind": "talk", "why": "b"}])"}, + .expectation = {.same = false}}, + {.name = "written fallback differs", .input = {.a = R"([{"id": "f", "kind": "battle"}])", .b = R"([{"id": "f", "kind": "battle", "args": {"may_lose": false}}])"}, + .expectation = {.same = false}}, + {.name = "another target differs", .input = { .a = ROUTE_B3F, + .b = R"([{"id": "go", "kind": "route", "args": {"target": {"map": "RocketHideout_B3F", "tiles": [[12, 14]]}}}])"}, + .expectation = {.same = false}}, + {.name = "another step name differs", .input = {.a = R"([{"id": "t", "kind": "talk"}])", .b = R"([{"id": "u", "kind": "talk"}])"}, + .expectation = {.same = false}}, }; -CheckpointSpec walk_row(std::string id, Lab& lab){ - CheckpointSpec spec; - spec.id = id; - spec.saves = true; - spec.build = [&lab, id]{ - Checkpoint checkpoint; - checkpoint.id = id; - Event walk; - walk.name = "walk"; - walk.act = [&lab, id](RunState&){ - lab.trace.push_back(id); - int& left = lab.failures[id]; - if (left > 0){ - left--; - return outcome_failed("blocked", "a wanderer on the path"); - } - return outcome_ok(); - }; - Recovery stop; - stop.kind = RecoveryKind::STOP; - walk.recoveries.push_back(std::move(stop)); - checkpoint.events.push_back(std::move(walk)); - return checkpoint; - }; - return spec; +std::string hash_of(std::string_view events, std::string_view source){ + const std::string json = std::format(R"({{"id": "x", "title": "x", "events": {}}})", events); + return story_steps_hash(read_story_checkpoint(parse_json(json), source)); } -Segment three_rows(Lab& lab){ - Segment segment; - segment.id = "00"; - for (std::string_view id : {"00.01", "00.02", "00.03"}){ - segment.checkpoints.push_back(walk_row(std::string(id), lab)); +std::string check_hash(const HashCase& test){ + const std::string a = hash_of(test.input.a, test.name); + const std::string b = hash_of(test.input.b, test.name); + if ((a == b) != test.expectation.same){ + return std::format("{}{}: {}, {}", "expected the hashes ", (test.expectation.same ? "equal" : "different"), a, b); + } + if (test.expectation.hash_a && a != *test.expectation.hash_a){ + return std::format("expected the pinned hash {}, got {}", *test.expectation.hash_a, a); } - return segment; + return ""; } -SegmentReport run_in(StorySaveMode mode, Lab& lab){ - Segment segment = three_rows(lab); - SegmentHooks hooks = lab.hooks(); - RunState state; - return run_segment(SegmentRange{}, SegmentPolicy{.save_mode = mode}, segment, hooks, state); -} +class Test_StepsHash : public UnitTest{ +public: + explicit Test_StepsHash(HashCase test) + : UnitTest(std::format("Pokemon::StorySavePolicy - steps hash: {}", test.name)) + , m_case(std::move(test)) + {} -UnitTestResult test_none_never_saves(){ - Lab lab; - const SegmentReport report = run_in(StorySaveMode::NONE, lab); - const std::vector expected = {"00.01", "00.02", "00.03"}; - if (!report.completed || lab.trace != expected){ - return std::format("Expected three checkpoints and no save:{}", lab.all()); - } - if (!lab.has("[Segment] 00: save mode none") || !lab.has("[Checkpoint] 00.02: not saving (save mode none)")){ - return std::format("Expected the skipped saves logged:{}", lab.all()); + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::string error = check_hash(m_case); + return error.empty() ? UnitTestResult(true) : UnitTestResult(error); } - return true; -} -UnitTestResult test_per_checkpoint_saves_each(){ - Lab lab; - const SegmentReport report = run_in(StorySaveMode::PER_CHECKPOINT, lab); - const std::vector expected = {"00.01", "save", "00.02", "save", "00.03", "save"}; - if (!report.completed || lab.trace != expected){ - return std::format("Expected a save after each checkpoint:{}", lab.all()); - } - return true; -} +private: + HashCase m_case; +}; -UnitTestResult test_after_verification_saves_at_the_end(){ - Lab lab; - const SegmentReport report = run_in(StorySaveMode::AFTER_VERIFICATION, lab); - const std::vector expected = {"00.01", "00.02", "00.03", "save"}; - if (!report.completed || lab.trace != expected){ - return std::format("Expected one save, at the end:{}", lab.all()); - } - if (!lab.has("[Postcondition] 00.03: not checked: the game has no postcondition check")){ - return std::format("Expected the missing check logged as not checked:{}", lab.all()); - } - return true; -} -UnitTestResult test_after_verification_checks_before_saving(){ - Lab lab; - lab.postconditions = outcome_ok(); - const SegmentReport report = run_in(StorySaveMode::AFTER_VERIFICATION, lab); - const std::vector expected = {"00.01", "00.02", "00.03", "check 00.03", "save"}; - if (!report.completed || lab.trace != expected){ - return std::format("Expected the check, then the save:{}", lab.all()); - } - return true; -} +// The save mode of a run of checkpoint "00.31" in `phase`, verified with +// `recorded` (none: never verified), whose steps now hash to "sha256:now". +struct SaveDecisionCase{ + std::string_view name; + struct Input{ + StoryPhase phase; + std::optional recorded; + }; -// FRLG 00.31, 2026-11-10: the run said it passed on the wrong floor. -UnitTestResult test_failed_postconditions_never_save(){ - Lab lab; - lab.postconditions = outcome_failed("postconditions", "map: expected RocketHideout_B3F, read RocketHideout_B4F"); - const SegmentReport report = run_in(StorySaveMode::AFTER_VERIFICATION, lab); - if (report.completed || report.failed_checkpoint != "00.03"){ - return std::format("Expected the run failed at its last checkpoint:{}", lab.all()); - } - if (std::find(lab.trace.begin(), lab.trace.end(), "save") != lab.trace.end()){ - return std::format("A run whose postconditions failed must not save:{}", lab.all()); - } - if (!lab.has("[Checkpoint] 00.03: postconditions failed, not saving")){ - return std::format("Expected the refused save logged:{}", lab.all()); - } - return true; -} + struct Expectation{ + StorySaveMode mode; + std::string_view reason; + }; -// Without per-checkpoint saves, a reload would go back to the run's start. -UnitTestResult test_no_reload_after_the_first(){ - Lab lab; - lab.failures["00.02"] = 1; - const SegmentReport report = run_in(StorySaveMode::NONE, lab); - const std::vector expected = {"00.01", "00.02"}; - if (report.completed || report.reloads != 0 || lab.trace != expected){ - return std::format("Expected 00.02 to fail without a reload:{}", lab.all()); - } - if (!lab.has("[Segment] 00: 00.02 failed; save mode none left no save at its start, so no reload")){ - return std::format("Expected why it didn't reload:{}", lab.all()); - } - return true; -} + Input input; + Expectation expectation; +}; -UnitTestResult test_first_row_still_reloads(){ - Lab lab; - lab.failures["00.01"] = 1; - const SegmentReport report = run_in(StorySaveMode::NONE, lab); - const std::vector expected = {"00.01", "reload", "00.01", "00.02", "00.03"}; - if (!report.completed || report.reloads != 1 || lab.trace != expected){ - return std::format("Expected the first checkpoint retried from the start save:{}", lab.all()); - } - return true; -} +const SaveDecisionCase SAVE_DECISION_CASES[] = { + {.name = "testing never saves", .input = {.phase = StoryPhase::TESTING, .recorded = "sha256:now"}, + .expectation = {.mode = StorySaveMode::NONE, .reason = "[Save] 00.31: phase testing: save mode none"}}, + {.name = "verification saves after its runs", .input = {.phase = StoryPhase::VERIFICATION, .recorded = "sha256:now"}, + .expectation = {.mode = StorySaveMode::AFTER_VERIFICATION, .reason = "[Save] 00.31: phase verification: save mode after_verification"}}, + {.name = "reproduction saves each checkpoint", .input = {.phase = StoryPhase::REPRODUCTION, .recorded = "sha256:now"}, + .expectation = {.mode = StorySaveMode::PER_CHECKPOINT, .reason = "[Save] 00.31: phase reproduction: save mode per_checkpoint"}}, + {.name = "changed steps are testing", .input = {.phase = StoryPhase::REPRODUCTION, .recorded = "sha256:old"}, + .expectation = {.mode = StorySaveMode::NONE, + .reason = "[Save] 00.31: phase reproduction but steps_hash mismatch (recorded sha256:old, now sha256:now): treated as testing, save mode none"}}, + {.name = "never verified is testing", .input = {.phase = StoryPhase::VERIFICATION, .recorded = std::nullopt}, + .expectation = {.mode = StorySaveMode::NONE, + .reason = "[Save] 00.31: phase verification but never verified: treated as testing, save mode none"}}, +}; -VerifyReport verify_in(StorySaveMode mode, Lab& lab){ - Segment segment = three_rows(lab); - SegmentHooks hooks = lab.hooks(); - RunState state; - const VerifyPolicy policy{.runs = 3, .passes_needed = 2, .save_mode = mode}; - return verify_checkpoint("00.02", policy, segment, hooks, state); -} +class Test_SaveDecision : public UnitTest{ +public: + explicit Test_SaveDecision(SaveDecisionCase test) + : UnitTest(std::format("Pokemon::StorySavePolicy - save decision: {}", test.name)) + , m_case(std::move(test)) + {} -UnitTestResult test_verify_saves_once_every_run_passed(){ - Lab lab; - const VerifyReport report = verify_in(StorySaveMode::AFTER_VERIFICATION, lab); - const std::vector expected = {"00.02", "reload", "00.02", "reload", "00.02", "save"}; - if (!report.verified || !report.saved || lab.trace != expected){ - return std::format("Expected one save, after the last run:{}", lab.all()); - } - if (!lab.has("[Verify] 00.02: every run passed: saved at the end state")){ - return std::format("Expected the save logged:{}", lab.all()); + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + StoryCheckpoint checkpoint; + checkpoint.id = "00.31"; + checkpoint.metadata.phase = m_case.input.phase; + if (m_case.input.recorded){ + checkpoint.metadata.verified = StoryVerification{.steps_hash = std::string(*m_case.input.recorded)}; + } + const StorySaveDecision decision = decide_story_save(checkpoint, "sha256:now"); + if (decision.mode != m_case.expectation.mode || decision.reason != m_case.expectation.reason){ + return std::format("got {}: {}", story_save_mode_name(decision.mode), decision.reason); + } + return true; } - return true; -} -UnitTestResult test_verify_no_save_after_a_failed_run(){ - Lab lab; - lab.failures["00.02"] = 1; - const VerifyReport report = verify_in(StorySaveMode::AFTER_VERIFICATION, lab); - if (!report.verified || report.saved || std::find(lab.trace.begin(), lab.trace.end(), "save") != lab.trace.end()){ - return std::format("Verified 2/3, but a failed run means no save:{}", lab.all()); - } - if (!lab.has("[Verify] 00.02: not every run passed: not saving")){ - return std::format("Expected why it didn't save:{}", lab.all()); - } - return true; -} +private: + SaveDecisionCase m_case; +}; -// FRLG 00.31's end, in the shared format: beside B3F's Rare Candy, with -// the LIFT KEY and both Rare Candies. +// A checkpoint that ends beside B3F's Rare Candy, with the LIFT KEY and +// both Rare Candies. inline constexpr std::string_view CHECKPOINT_0031 = R"({ "id": "00.31", "title": "Rocket Hideout to the LIFT KEY", @@ -329,7 +211,8 @@ UnitTestResult test_postconditions_pass(){ return true; } -// The 2026-11-10 false pass: still on B4F, the Rare Candy not picked up. +// A run that reports success while still on B4F, the Rare Candy not +// picked up. UnitTestResult test_postconditions_fail(){ const PostconditionReport report = check_postconditions(checkpoint_0031(), read_0031("RocketHideout_B4F", 1)); const std::vector lines = postcondition_log_lines(report); @@ -399,106 +282,46 @@ UnitTestResult test_end_pose_from_the_plan(){ } -// A save-policy scenario and its expected result. -struct SavePolicyCase{ - // The scenario to run. +// A named test; it returns its own failure. +struct SavePolicyTest{ std::string name; - struct Input{ - UnitTestResult (*test)() = nullptr; - }; - - // What the scenario checks; it returns its own failure. - struct Expectation{ - std::string_view description; - }; - - Input input; - Expectation expectation; + UnitTestResult (*test)(); }; class Test_StorySavePolicy : public UnitTest{ public: - explicit Test_StorySavePolicy(SavePolicyCase test_case) - : UnitTest(std::format("Pokemon::StorySavePolicy - {}", test_case.name)) - , m_case(std::move(test_case)) + explicit Test_StorySavePolicy(SavePolicyTest test) + : UnitTest(std::format("Pokemon::StorySavePolicy - {}", test.name)) + , m_test(std::move(test)) {} virtual UnitTestResult run(Logger&, CancellableScope&) const override{ - return m_case.input.test(); + return m_test.test(); } private: - SavePolicyCase m_case; + SavePolicyTest m_test; }; -const SavePolicyCase SAVE_POLICY_CASES[] = { - { - .name = "decision table", .input = {.test = test_decision_table}, - .expectation = {.description = "mode x verdict x save point, all twelve rows"}, - }, - { - .name = "none never saves", .input = {.test = test_none_never_saves}, - .expectation = {.description = "three passing checkpoints, no save, each skip logged"}, - }, - { - .name = "per checkpoint saves each", .input = {.test = test_per_checkpoint_saves_each}, - .expectation = {.description = "a save after every passing checkpoint"}, - }, - { - .name = "after verification saves at the end", .input = {.test = test_after_verification_saves_at_the_end}, - .expectation = {.description = "one save after the last checkpoint; no check logged as not checked"}, - }, - { - .name = "after verification checks before saving", .input = {.test = test_after_verification_checks_before_saving}, - .expectation = {.description = "the game's postcondition check runs before the save"}, - }, - { - .name = "failed postconditions never save", .input = {.test = test_failed_postconditions_never_save}, - .expectation = {.description = "a failed check fails the run and refuses the save"}, - }, - { - .name = "no reload after the first checkpoint", .input = {.test = test_no_reload_after_the_first}, - .expectation = {.description = "a later checkpoint has no save to reload, so it fails at once"}, - }, - { - .name = "the first checkpoint still reloads", .input = {.test = test_first_row_still_reloads}, - .expectation = {.description = "the first checkpoint reloads the run's start save"}, - }, - { - .name = "verify saves once every run passed", .input = {.test = test_verify_saves_once_every_run_passed}, - .expectation = {.description = "3/3 passed: one save at the end of run 3"}, - }, - { - .name = "verify: no save after a failed run", .input = {.test = test_verify_no_save_after_a_failed_run}, - .expectation = {.description = "2/3 verified, but no save"}, - }, - { - .name = "postconditions: json", .input = {.test = test_postconditions_json}, - .expectation = {.description = "00.31's postconditions read and written back"}, - }, - { - .name = "postconditions: pass", .input = {.test = test_postconditions_pass}, - .expectation = {.description = "B3F (12, 13) north, LIFT_KEY 1, RARE_CANDY 2: passed"}, - }, - { - .name = "postconditions: fail", .input = {.test = test_postconditions_fail}, - .expectation = {.description = "still on B4F with 1 Rare Candy: failed"}, - }, - { - .name = "postconditions: not checked", .input = {.test = test_postconditions_not_checked}, - .expectation = {.description = "unread parts are not checked, never passed"}, - }, - { - .name = "postconditions: end pose from the plan", .input = {.test = test_end_pose_from_the_plan}, - .expectation = {.description = "the last route's target, or unknown after legs"}, - }, +const SavePolicyTest SAVE_POLICY_TESTS[] = { + {.name = "postconditions: json", .test = test_postconditions_json}, + {.name = "postconditions: pass", .test = test_postconditions_pass}, + {.name = "postconditions: fail", .test = test_postconditions_fail}, + {.name = "postconditions: not checked", .test = test_postconditions_not_checked}, + {.name = "postconditions: end pose from the plan", .test = test_end_pose_from_the_plan}, }; } // namespace void add_tests_StorySavePolicy(UnitTestDatabase& database){ - for (const SavePolicyCase& test_case : SAVE_POLICY_CASES){ - database.add(test_case); + for (const HashCase& test : HASH_CASES){ + database.add(test); + } + for (const SaveDecisionCase& test : SAVE_DECISION_CASES){ + database.add(test); + } + for (const SavePolicyTest& test : SAVE_POLICY_TESTS){ + database.add(test); } } From 532acb0dd6ff6309432fee284d8bef3cdfce66a5 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 15 Nov 2026 12:26:39 -0800 Subject: [PATCH 3/3] AutoStory engine: drop the phase and steps-hash save gate StoryEnginePolicy.h keeps the engine's report and CheckpointVerdict; the postcondition tests stay. --- .../Engine/Pokemon_StoryEnginePolicy.cpp | 71 --------- .../Engine/Pokemon_StoryEnginePolicy.h | 34 +---- .../Pokemon_StorySavePolicy_Tests.cpp | 143 +----------------- 3 files changed, 10 insertions(+), 238 deletions(-) delete mode 100644 SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.cpp diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.cpp deleted file mode 100644 index 5276973de5..0000000000 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.cpp +++ /dev/null @@ -1,71 +0,0 @@ -/* Story Engine Policy - * - * From: https://github.com/PokemonAutomation/ - * - */ - -#include -#include -#include "Common/Cpp/Cryptography/SHA256.h" -#include "Pokemon_StoryEnginePolicy.h" - -namespace PokemonAutomation{ -namespace Pokemon{ - - -namespace{ - - -StorySaveMode mode_of(StoryPhase phase){ - switch (phase){ - case StoryPhase::TESTING: - return StorySaveMode::NONE; - case StoryPhase::VERIFICATION: - return StorySaveMode::AFTER_VERIFICATION; - case StoryPhase::REPRODUCTION: - return StorySaveMode::PER_CHECKPOINT; - } - return StorySaveMode::NONE; -} - - -} // namespace - - -std::string story_steps_hash(const StoryCheckpoint& checkpoint){ - const std::string& text = checkpoint.steps_json; - SHA256 hash; - hash.push(text.data(), text.size()); - hash.finish(); - return std::format("sha256:{}", hash.get_hash_hex()); -} - -StorySaveDecision decide_story_save(const StoryCheckpoint& checkpoint, std::string_view steps_hash){ - const StoryCheckpointMetadata& metadata = checkpoint.metadata; - const std::string phase = story_phase_name(metadata.phase); - if (metadata.phase == StoryPhase::TESTING){ - return {.mode = StorySaveMode::NONE, .reason = std::format("[Save] {}: phase testing: save mode none", checkpoint.id)}; - } - if (!metadata.verified){ - return { - .mode = StorySaveMode::NONE, - .reason = std::format("[Save] {}: phase {} but never verified: treated as testing, save mode none", checkpoint.id, phase), - }; - } - if (metadata.verified->steps_hash != steps_hash){ - return { - .mode = StorySaveMode::NONE, - .reason = std::format( - "[Save] {}: phase {} but steps_hash mismatch (recorded {}, now {}): treated as testing, save mode none", - checkpoint.id, phase, metadata.verified->steps_hash, steps_hash - ), - }; - } - const StorySaveMode mode = mode_of(metadata.phase); - return {.mode = mode, .reason = std::format("[Save] {}: phase {}: save mode {}", checkpoint.id, phase, story_save_mode_name(mode))}; -} - - - -} // namespace Pokemon -} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.h b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.h index ac71834570..0b036eb5bf 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.h +++ b/SerialPrograms/Source/Pokemon/AutoStory/Engine/Pokemon_StoryEnginePolicy.h @@ -1,13 +1,8 @@ -/* Story Engine Policy +/* Story Engine Report * * From: https://github.com/PokemonAutomation/ * - * When an engine run of a checkpoint may save, from the checkpoint's - * metadata: a testing - * checkpoint never saves, a verification one saves once after its runs - * passed, a reproduction one at its end. A checkpoint whose steps no - * longer hash to the ones it was verified with is treated as testing: its - * verification was of other steps. + * What an engine run of one checkpoint reports. * */ @@ -15,42 +10,29 @@ #define PokemonAutomation_Pokemon_StoryEnginePolicy_H #include -#include -#include -#include #include "Pokemon/AutoStory/Pokemon_StoryCheckpointTypes.h" #include "Pokemon/AutoStory/Pokemon_StoryPostconditionCheck.h" -#include "Pokemon/AutoStory/Types/Pokemon_StoryActionSpec.h" -#include "Pokemon/AutoStory/Types/Pokemon_StoryCheckpoint.h" -#include "Pokemon/AutoStory/Types/Pokemon_StorySaveMode.h" namespace PokemonAutomation{ namespace Pokemon{ -// "sha256:" of `checkpoint`'s steps JSON as read from its file -// (StoryCheckpoint::steps_json): any edit to the steps, including a why -// or a default written out, changes it. -std::string story_steps_hash(const StoryCheckpoint& checkpoint); - -// A run's save mode and the log line that says why. -struct StorySaveDecision{ - StorySaveMode mode = StorySaveMode::NONE; - std::string reason = {}; +// How a checkpoint ended: passed (and its postconditions, where checked, +// did not fail) or not. +enum class CheckpointVerdict{ + PASSED, + FAILED, }; -// The save mode of a run of `checkpoint`, whose steps hash to `steps_hash`. -StorySaveDecision decide_story_save(const StoryCheckpoint& checkpoint, std::string_view steps_hash); - // An engine run of one checkpoint. struct StoryEngineReport{ // The steps' run. A checkpoint that failed its lint has none. CheckpointReport run = {}; - StorySaveDecision save = {}; // PASSED when the steps completed and no postcondition failed. CheckpointVerdict verdict = CheckpointVerdict::FAILED; // Checked after steps that completed. std::optional postconditions = std::nullopt; + // The game saved at the checkpoint's end. bool saved = false; // A SAVE step among its steps saved: a reload now returns there, not // to the save the checkpoint started from. diff --git a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.cpp b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.cpp index f9ca4c269d..7ac81dcb16 100644 --- a/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.cpp +++ b/SerialPrograms/Source/Pokemon/AutoStory/Pokemon_StorySavePolicy_Tests.cpp @@ -2,10 +2,8 @@ * * From: https://github.com/PokemonAutomation/ * - * The engine's save policy (Engine/Pokemon_StoryEnginePolicy.h): the - * steps hash and the save mode a checkpoint's phase and recorded hash - * give. Then the postcondition check (Pokemon_StoryPostconditionCheck.h) - * on a checkpoint's end state. + * The postcondition check (Pokemon_StoryPostconditionCheck.h) on a + * checkpoint's end state. * */ @@ -18,7 +16,6 @@ #include "Common/Cpp/Json/JsonValue.h" #include "Common/Cpp/TestRunners/UnitTest.h" #include "Data/Pokemon_StoryStepJson.h" -#include "Engine/Pokemon_StoryEnginePolicy.h" #include "Pokemon_StoryPostconditionCheck.h" #include "Pokemon_StorySavePolicy_Tests.h" @@ -29,136 +26,6 @@ namespace Pokemon{ namespace{ -struct HashCase{ - // Two checkpoints' events, hashed with the built-in actions. - struct Input{ - std::string_view a; - std::string_view b; - }; - - // Whether they hash the same, and `a`'s hash when pinned: a change to it - // means every recorded verification goes stale. - struct Expectation{ - bool same = true; - std::optional hash_a = std::nullopt; - }; - - std::string_view name; - Input input; - Expectation expectation; -}; - -inline constexpr std::string_view ROUTE_B3F = R"([{"id": "go", "kind": "route", "args": {"target": {"map": "RocketHideout_B3F", "tiles": [[12, 13]]}}}])"; - -const HashCase HASH_CASES[] = { - {.name = "pinned", .input = {.a = ROUTE_B3F, .b = ROUTE_B3F}, - .expectation = {.same = true, .hash_a = "sha256:13bc877c960a15cd0ed2f37248fa8336308b048a865623dde7654d41ac7147b8"}}, - {.name = "whitespace is not hashed", .input = {.a = R"([{"id": "t", "kind": "talk"}])", .b = R"([ { "id":"t", "kind":"talk" } ])"}, - .expectation = {.same = true}}, - {.name = "why differs", .input = {.a = R"([{"id": "t", "kind": "talk", "why": "a"}])", .b = R"([{"id": "t", "kind": "talk", "why": "b"}])"}, - .expectation = {.same = false}}, - {.name = "written fallback differs", .input = {.a = R"([{"id": "f", "kind": "battle"}])", .b = R"([{"id": "f", "kind": "battle", "args": {"may_lose": false}}])"}, - .expectation = {.same = false}}, - {.name = "another target differs", .input = { .a = ROUTE_B3F, - .b = R"([{"id": "go", "kind": "route", "args": {"target": {"map": "RocketHideout_B3F", "tiles": [[12, 14]]}}}])"}, - .expectation = {.same = false}}, - {.name = "another step name differs", .input = {.a = R"([{"id": "t", "kind": "talk"}])", .b = R"([{"id": "u", "kind": "talk"}])"}, - .expectation = {.same = false}}, -}; - -std::string hash_of(std::string_view events, std::string_view source){ - const std::string json = std::format(R"({{"id": "x", "title": "x", "events": {}}})", events); - return story_steps_hash(read_story_checkpoint(parse_json(json), source)); -} - -std::string check_hash(const HashCase& test){ - const std::string a = hash_of(test.input.a, test.name); - const std::string b = hash_of(test.input.b, test.name); - if ((a == b) != test.expectation.same){ - return std::format("{}{}: {}, {}", "expected the hashes ", (test.expectation.same ? "equal" : "different"), a, b); - } - if (test.expectation.hash_a && a != *test.expectation.hash_a){ - return std::format("expected the pinned hash {}, got {}", *test.expectation.hash_a, a); - } - return ""; -} - -class Test_StepsHash : public UnitTest{ -public: - explicit Test_StepsHash(HashCase test) - : UnitTest(std::format("Pokemon::StorySavePolicy - steps hash: {}", test.name)) - , m_case(std::move(test)) - {} - - virtual UnitTestResult run(Logger&, CancellableScope&) const override{ - const std::string error = check_hash(m_case); - return error.empty() ? UnitTestResult(true) : UnitTestResult(error); - } - -private: - HashCase m_case; -}; - - -// The save mode of a run of checkpoint "00.31" in `phase`, verified with -// `recorded` (none: never verified), whose steps now hash to "sha256:now". -struct SaveDecisionCase{ - std::string_view name; - struct Input{ - StoryPhase phase; - std::optional recorded; - }; - - struct Expectation{ - StorySaveMode mode; - std::string_view reason; - }; - - Input input; - Expectation expectation; -}; - -const SaveDecisionCase SAVE_DECISION_CASES[] = { - {.name = "testing never saves", .input = {.phase = StoryPhase::TESTING, .recorded = "sha256:now"}, - .expectation = {.mode = StorySaveMode::NONE, .reason = "[Save] 00.31: phase testing: save mode none"}}, - {.name = "verification saves after its runs", .input = {.phase = StoryPhase::VERIFICATION, .recorded = "sha256:now"}, - .expectation = {.mode = StorySaveMode::AFTER_VERIFICATION, .reason = "[Save] 00.31: phase verification: save mode after_verification"}}, - {.name = "reproduction saves each checkpoint", .input = {.phase = StoryPhase::REPRODUCTION, .recorded = "sha256:now"}, - .expectation = {.mode = StorySaveMode::PER_CHECKPOINT, .reason = "[Save] 00.31: phase reproduction: save mode per_checkpoint"}}, - {.name = "changed steps are testing", .input = {.phase = StoryPhase::REPRODUCTION, .recorded = "sha256:old"}, - .expectation = {.mode = StorySaveMode::NONE, - .reason = "[Save] 00.31: phase reproduction but steps_hash mismatch (recorded sha256:old, now sha256:now): treated as testing, save mode none"}}, - {.name = "never verified is testing", .input = {.phase = StoryPhase::VERIFICATION, .recorded = std::nullopt}, - .expectation = {.mode = StorySaveMode::NONE, - .reason = "[Save] 00.31: phase verification but never verified: treated as testing, save mode none"}}, -}; - -class Test_SaveDecision : public UnitTest{ -public: - explicit Test_SaveDecision(SaveDecisionCase test) - : UnitTest(std::format("Pokemon::StorySavePolicy - save decision: {}", test.name)) - , m_case(std::move(test)) - {} - - virtual UnitTestResult run(Logger&, CancellableScope&) const override{ - StoryCheckpoint checkpoint; - checkpoint.id = "00.31"; - checkpoint.metadata.phase = m_case.input.phase; - if (m_case.input.recorded){ - checkpoint.metadata.verified = StoryVerification{.steps_hash = std::string(*m_case.input.recorded)}; - } - const StorySaveDecision decision = decide_story_save(checkpoint, "sha256:now"); - if (decision.mode != m_case.expectation.mode || decision.reason != m_case.expectation.reason){ - return std::format("got {}: {}", story_save_mode_name(decision.mode), decision.reason); - } - return true; - } - -private: - SaveDecisionCase m_case; -}; - - // A checkpoint that ends beside B3F's Rare Candy, with the LIFT KEY and // both Rare Candies. inline constexpr std::string_view CHECKPOINT_0031 = R"({ @@ -314,12 +181,6 @@ const SavePolicyTest SAVE_POLICY_TESTS[] = { void add_tests_StorySavePolicy(UnitTestDatabase& database){ - for (const HashCase& test : HASH_CASES){ - database.add(test); - } - for (const SaveDecisionCase& test : SAVE_DECISION_CASES){ - database.add(test); - } for (const SavePolicyTest& test : SAVE_POLICY_TESTS){ database.add(test); }