diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index a11149973a..22f0db08ec 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -48,8 +48,11 @@ #include "Pokemon/Navigation/Pokemon_RouteLog_Tests.h" #include "Pokemon/Navigation/Pokemon_RuntimeMap_Tests.h" #include "Pokemon/Navigation/Pokemon_WallRecovery_Tests.h" +#include "Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.h" #include "Pokemon/Navigation/Simulation/Pokemon_SimFaults_Tests.h" #include "Pokemon/Navigation/Simulation/Pokemon_SimGolden_Tests.h" +#include "Pokemon/Navigation/Simulation/Pokemon_SimSeeds_Tests.h" +#include "Pokemon/Navigation/Simulation/Pokemon_SimSweep_Tests.h" #include "VideoGames/PokemonHome/PokemonHome_Tests.h" #include "VideoGames/PokemonSwSh/PokemonSwSh_Tests.h" #include "VideoGames/PokemonLA/PokemonLA_Tests.h" @@ -68,6 +71,14 @@ namespace ComputerPrograms{ +static void add_tests_NavigationSimulation(UnitTestDatabase& ret){ + Pokemon::add_tests_SimFaults(ret); + Pokemon::add_tests_SimGolden(ret); + Pokemon::add_tests_SimSweep(ret); + Pokemon::add_tests_SimSeeds(ret); + Pokemon::add_tests_KnownFailing(ret); +} + UnitTestDatabase make_UNIT_TESTS_ALL(){ UnitTestDatabase ret; add_tests_QtEventThreadPool(ret, GlobalThreadPools::computation_normal()); @@ -98,8 +109,7 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){ Pokemon::add_tests_MoveMechanics(ret); Pokemon::add_tests_RuntimeMap(ret); Pokemon::add_tests_RouteLog(ret); - Pokemon::add_tests_SimFaults(ret); - Pokemon::add_tests_SimGolden(ret); + add_tests_NavigationSimulation(ret); Pokemon::add_tests_AutoStory(ret); NintendoSwitch::add_tests_CheckOnlineDetector(ret); NintendoSwitch::add_tests_FailedToConnectDetector(ret); diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.cpp b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.cpp new file mode 100644 index 0000000000..e01bbe4e07 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.cpp @@ -0,0 +1,54 @@ +/* Known Failing + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Pokemon_KnownFailing.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +KnownFailing::KnownFailing(std::map fixes) + : m_fixes(std::make_move_iterator(fixes.begin()), std::make_move_iterator(fixes.end())) +{} + +UnitTestResult KnownFailing::run(Logger& logger, std::string_view name, const std::function& body) const{ + UnitTestResult result; + try{ + result = body(); + }catch (Exception& e){ + result = std::format("{}: {}", e.name(), e.message()); + } + const auto known = m_fixes.find(name); + if (known == m_fixes.end()){ + return result; + } + if (result.result == UnitTestResult::PASSED){ + return std::format("on KNOWN_FAILING ({}) but passes now: take it off the list.", known->second); + } + logger.log(std::format("Known failing until {}: {}", known->second, result.message)); + return true; +} + +std::vector KnownFailing::stale(const std::set& names) const{ + std::vector out; + for (const auto& [name, fix] : m_fixes){ + if (!names.contains(name)){ + out.push_back(name); + } + } + return out; +} + + + +} // namespace Pokemon + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.h b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.h new file mode 100644 index 0000000000..b6684a4ccb --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing.h @@ -0,0 +1,58 @@ +/* Known Failing + * + * From: https://github.com/PokemonAutomation/ + * + * The list of sim tests the program fails today, each with the change + * likely to fix it, shared by the games' fault sweeps and console-failure + * cases. A test on the list passes while it fails, so the suite stays + * green with the failure logged, and fails once it passes, so the entry + * comes off. stale() finds the entries that name no test: a renamed test + * would otherwise drop off the list silently and fail the suite. + * + */ + +#ifndef PokemonAutomation_Pokemon_KnownFailing_H +#define PokemonAutomation_Pokemon_KnownFailing_H + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "Common/Cpp/TestRunners/UnitTest.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// The tests known to fail, by name, with their likely fixes. +class KnownFailing{ +public: + // `fixes`: by test name (or case name, as the game keys it), the + // change likely to fix it. + explicit KnownFailing(std::map fixes); + + // Runs test `name`'s `body`, an Exception it throws being its failure, + // and gives its result as the list sees it: as is if `name` is not + // listed; a pass that logs the failure if it is and fails; a failure + // "on KNOWN_FAILING () but passes now: take it off the list." if + // it is and passes. + UnitTestResult run(Logger& logger, std::string_view name, const std::function& body) const; + + // The listed names not in `names` (the tests that exist), in order. + std::vector stale(const std::set& names) const; + +private: + std::map> m_fixes; +}; + + + +} // namespace Pokemon + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.cpp b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.cpp new file mode 100644 index 0000000000..ff7dfbd7b6 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.cpp @@ -0,0 +1,209 @@ +/* Known Failing Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The known-failing list on fake tests: one on the list passes while it + * fails and fails once it passes, one off the list keeps its result, an + * exception is a failure, and entries naming no test are stale. + * + */ + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "Pokemon_KnownFailing.h" +#include "Pokemon_KnownFailing_Tests.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + +class LinesLogger : public Logger{ +public: + using Logger::log; + virtual void log(const std::string& msg, Color) override{ + lines.push_back(msg); + } + + std::vector lines; +}; + +const KnownFailing& known_failing(){ + static const KnownFailing list({ + {"listed", "the planner's goal kinds"}, + {"listed and gone", "a fix for a test since renamed"}, + }); + return list; +} + +// How the fake test's body ends. +enum class BodyOutcome{ + PASSES, + FAILS, + THROWS, +}; + +struct KnownFailingCase{ + struct Input{ + std::string test; + BodyOutcome body; + }; + struct Expectation{ + bool passes; + std::string message; + std::vector log; + }; + + std::string name; + Input input; + Expectation expectation; +}; + +const KnownFailingCase KNOWN_FAILING_CASES[] = { + { + .name = "a listed test passes while it fails", + .input = {.test = "listed", .body = BodyOutcome::FAILS}, + .expectation = {.passes = true, .message = "", .log = {"Known failing until the planner's goal kinds: stopped at the gate."}}, + }, + { + .name = "a listed test fails once it passes", + .input = {.test = "listed", .body = BodyOutcome::PASSES}, + .expectation = { + .passes = false, + .message = "on KNOWN_FAILING (the planner's goal kinds) but passes now: take it off the list.", + .log = {}, + }, + }, + { + .name = "a listed test's exception is its known failure", + .input = {.test = "listed", .body = BodyOutcome::THROWS}, + .expectation = {.passes = true, .message = "", .log = {"Known failing until the planner's goal kinds: ParseException: no row 99.99"}}, + }, + { + .name = "an unlisted test keeps its failure", + .input = {.test = "unlisted", .body = BodyOutcome::FAILS}, + .expectation = {.passes = false, .message = "stopped at the gate.", .log = {}}, + }, + { + .name = "an unlisted test keeps its pass", + .input = {.test = "unlisted", .body = BodyOutcome::PASSES}, + .expectation = {.passes = true, .message = "", .log = {}}, + }, + { + .name = "an unlisted test's exception is its failure", + .input = {.test = "unlisted", .body = BodyOutcome::THROWS}, + .expectation = {.passes = false, .message = "ParseException: no row 99.99", .log = {}}, + }, +}; + +UnitTestResult fake_body(BodyOutcome outcome){ + switch (outcome){ + case BodyOutcome::PASSES: + return true; + case BodyOutcome::FAILS: + return "stopped at the gate."; + case BodyOutcome::THROWS: + throw ParseException("no row 99.99"); + } + return true; +} + +class Test_KnownFailing : public UnitTest{ +public: + explicit Test_KnownFailing(KnownFailingCase test_case) + : UnitTest(std::format("Pokemon::Navigation - Known failing: {}", test_case.name)) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + LinesLogger logger; + const UnitTestResult result = known_failing().run(logger, m_case.input.test, [this]{ + return fake_body(m_case.input.body); + }); + const KnownFailingCase::Expectation& expected = m_case.expectation; + const bool passed = result.result == UnitTestResult::PASSED; + if (passed != expected.passes){ + return std::format("expected {}, got \"{}\".", expected.passes ? "a pass" : "a failure", result.message); + } + if (result.message != expected.message){ + return std::format("message: expected \"{}\", got \"{}\".", expected.message, result.message); + } + if (logger.lines != expected.log){ + return std::format("log: expected {} lines, got {}: \"{}\".", expected.log.size(), logger.lines.size(), logger.lines.empty() ? "" : logger.lines[0]); + } + return true; + } + +private: + KnownFailingCase m_case; +}; + + +struct StaleCase{ + struct Input{ + std::set tests; + }; + struct Expectation{ + std::vector stale; + }; + + std::string name; + Input input; + Expectation expectation; +}; + +const StaleCase STALE_CASES[] = { + { + .name = "an entry naming no test is stale", + .input = {.tests = {"listed", "unlisted"}}, + .expectation = {.stale = {"listed and gone"}}, + }, + { + .name = "entries naming tests are not", + .input = {.tests = {"listed", "listed and gone"}}, + .expectation = {.stale = {}}, + }, +}; + +class Test_KnownFailingStale : public UnitTest{ +public: + explicit Test_KnownFailingStale(StaleCase test_case) + : UnitTest(std::format("Pokemon::Navigation - Known failing: {}", test_case.name)) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::vector stale = known_failing().stale(m_case.input.tests); + if (stale != m_case.expectation.stale){ + return std::format("expected {} stale entries, got {}.", m_case.expectation.stale.size(), stale.size()); + } + return true; + } + +private: + StaleCase m_case; +}; + +} // namespace + + +void add_tests_KnownFailing(UnitTestDatabase& database){ + for (const KnownFailingCase& item : KNOWN_FAILING_CASES){ + database.add(item); + } + for (const StaleCase& item : STALE_CASES){ + database.add(item); + } +} + + + +} // namespace Pokemon +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.h b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.h new file mode 100644 index 0000000000..49b7baa538 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.h @@ -0,0 +1,23 @@ +/* Known Failing Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_Pokemon_KnownFailing_Tests_H +#define PokemonAutomation_Pokemon_KnownFailing_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// Registers the known-failing list tests. +void add_tests_KnownFailing(UnitTestDatabase& database); + + + +} // namespace Pokemon +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.cpp b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.cpp new file mode 100644 index 0000000000..a56b6e9e25 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.cpp @@ -0,0 +1,63 @@ +/* Sim Seeds + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "Pokemon_SimSeeds.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + +// The mean and worst (largest) of `values`, which has one at least. +SeedStats stats_of(const std::vector& values){ + return { + .mean = std::accumulate(values.begin(), values.end(), 0.) / static_cast(values.size()), + .worst = *std::max_element(values.begin(), values.end()), + }; +} + +} // namespace + + +SeedsResult run_seeds( + Logger& logger, std::string_view label, const std::function& run_one, + uint32_t seeds +){ + SeedsResult out; + out.seeds = seeds; + std::map> passed; + for (uint32_t seed = 1; seed <= seeds; seed++){ + SeedRun run = run_one(seed); + if (run.failure){ + out.failures.push_back({"", std::format("seed {}", seed), std::move(*run.failure)}); + continue; + } + for (const auto& [measure, value] : run.measures){ + passed[measure].push_back(value); + } + } + std::string line = std::format("Seeds {}: passed {}/{}", label, seeds - out.failures.size(), seeds); + for (const auto& [measure, values] : passed){ + const SeedStats stats = stats_of(values); + line += std::format("; {} mean {:.1f}, worst {:.1f}", measure, stats.mean, stats.worst); + out.stats.emplace(measure, stats); + } + logger.log(line); + return out; +} + + + +} // namespace Pokemon + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.h b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.h new file mode 100644 index 0000000000..2fc7088181 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Simulation/Pokemon_SimSeeds.h @@ -0,0 +1,71 @@ +/* Sim Seeds + * + * From: https://github.com/PokemonAutomation/ + * + * The games' Monte Carlo rows, shared by BDSP's and FRLG's: a seeded + * row (wild battles at the game's odds, wandering people) runs once per + * seed, each seed feeding the sim's std::mt19937. Its failures are a + * fault sweep's (sweep_result()), a variant per seed, and it logs the + * mean and worst of what each seed measured. + * + */ + +#ifndef PokemonAutomation_Pokemon_SimSeeds_H +#define PokemonAutomation_Pokemon_SimSeeds_H + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "Pokemon_SimSweep.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// How many seeds a seeded row runs unless it asks for others. +inline constexpr uint32_t DEFAULT_SWEEP_SEEDS = 30; + +// One seed's run of a row: why it failed, or nothing; and what it measured +// by name ("s", "stops", ...), only what applies to the seed. +struct SeedRun{ + std::optional failure = std::nullopt; + std::map measures = {}; +}; + +// A measure over the seeds that passed and measured it. +struct SeedStats{ + double mean = 0; + double worst = 0; +}; + +// The seeds a row ran, those that failed, and what the rest measured. +struct SeedsResult{ + uint32_t seeds = 0; + // A failure per failed seed, its variant "seed N". + std::vector failures = {}; + // By measure, over the passed seeds; a measure no passed seed has is + // left out. + std::map stats = {}; +}; + +// Runs `run_one` from seeds 1 to `seeds`; logs "Seeds