Skip to content
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/* Unit Test Runner

Check warning on line 1 in SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

<file>: file_lines=509 (advisory above 500, limit 1000)
*
* From: https://github.com/PokemonAutomation/
*
Expand Down Expand Up @@ -48,8 +48,11 @@
#include "Pokemon/Navigation/Pokemon_RouteLog_Tests.h"
#include "Pokemon/Navigation/Pokemon_RuntimeMap_Tests.h"
#include "Pokemon/Navigation/Pokemon_WallRecovery_Tests.h"
#include "Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.h"
#include "Pokemon/Navigation/Simulation/Pokemon_SimFaults_Tests.h"
#include "Pokemon/Navigation/Simulation/Pokemon_SimGolden_Tests.h"
#include "Pokemon/Navigation/Simulation/Pokemon_SimSeeds_Tests.h"
#include "Pokemon/Navigation/Simulation/Pokemon_SimSweep_Tests.h"
#include "VideoGames/PokemonHome/PokemonHome_Tests.h"
#include "VideoGames/PokemonSwSh/PokemonSwSh_Tests.h"
#include "VideoGames/PokemonLA/PokemonLA_Tests.h"
Expand All @@ -68,7 +71,15 @@



static void add_tests_NavigationSimulation(UnitTestDatabase& ret){
Pokemon::add_tests_SimFaults(ret);
Pokemon::add_tests_SimGolden(ret);
Pokemon::add_tests_SimSweep(ret);
Pokemon::add_tests_SimSeeds(ret);
Pokemon::add_tests_KnownFailing(ret);
}

UnitTestDatabase make_UNIT_TESTS_ALL(){

Check warning on line 82 in SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

PokemonAutomation::ComputerPrograms::make_UNIT_TESTS_ALL(): function_lines=50 (advisory above 25, limit 50)
UnitTestDatabase ret;
add_tests_QtEventThreadPool(ret, GlobalThreadPools::computation_normal());
add_tests_Sockets(ret, GlobalThreadPools::unlimited_normal());
Expand Down Expand Up @@ -98,8 +109,7 @@
Pokemon::add_tests_MoveMechanics(ret);
Pokemon::add_tests_RuntimeMap(ret);
Pokemon::add_tests_RouteLog(ret);
Pokemon::add_tests_SimFaults(ret);
Pokemon::add_tests_SimGolden(ret);
add_tests_NavigationSimulation(ret);
Pokemon::add_tests_AutoStory(ret);
NintendoSwitch::add_tests_CheckOnlineDetector(ret);
NintendoSwitch::add_tests_FailedToConnectDetector(ret);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/* Known Failing
*
* From: https://github.com/PokemonAutomation/
*
*/

#include <format>
#include <utility>
#include "Common/Cpp/Exceptions.h"
#include "Pokemon_KnownFailing.h"

namespace PokemonAutomation{
namespace Pokemon{


KnownFailing::KnownFailing(std::map<std::string, std::string> fixes)
: m_fixes(std::make_move_iterator(fixes.begin()), std::make_move_iterator(fixes.end()))
{}

UnitTestResult KnownFailing::run(Logger& logger, std::string_view name, const std::function<UnitTestResult()>& body) const{
UnitTestResult result;
try{
result = body();
}catch (Exception& e){
result = std::format("{}: {}", e.name(), e.message());
}
const auto known = m_fixes.find(name);
if (known == m_fixes.end()){
return result;
}
if (result.result == UnitTestResult::PASSED){
return std::format("on KNOWN_FAILING ({}) but passes now: take it off the list.", known->second);
}
logger.log(std::format("Known failing until {}: {}", known->second, result.message));
return true;
}

std::vector<std::string> KnownFailing::stale(const std::set<std::string>& names) const{
std::vector<std::string> out;
for (const auto& [name, fix] : m_fixes){
if (!names.contains(name)){
out.push_back(name);
}
}
return out;
}



} // namespace Pokemon



} // namespace PokemonAutomation
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/* Known Failing
*
* From: https://github.com/PokemonAutomation/
*
* The list of sim tests the program fails today, each with the change
* likely to fix it, shared by the games' fault sweeps and console-failure
* cases. A test on the list passes while it fails, so the suite stays
* green with the failure logged, and fails once it passes, so the entry
* comes off. stale() finds the entries that name no test: a renamed test
* would otherwise drop off the list silently and fail the suite.
*
*/

#ifndef PokemonAutomation_Pokemon_KnownFailing_H
#define PokemonAutomation_Pokemon_KnownFailing_H

#include <functional>
#include <map>
#include <set>
#include <string>
#include <string_view>
#include <vector>
#include "Common/Cpp/Logging/AbstractLogger.h"
#include "Common/Cpp/TestRunners/UnitTest.h"

namespace PokemonAutomation{
namespace Pokemon{


// The tests known to fail, by name, with their likely fixes.
class KnownFailing{
public:
// `fixes`: by test name (or case name, as the game keys it), the
// change likely to fix it.
explicit KnownFailing(std::map<std::string, std::string> fixes);

// Runs test `name`'s `body`, an Exception it throws being its failure,
// and gives its result as the list sees it: as is if `name` is not
// listed; a pass that logs the failure if it is and fails; a failure
// "on KNOWN_FAILING (<fix>) but passes now: take it off the list." if
// it is and passes.
UnitTestResult run(Logger& logger, std::string_view name, const std::function<UnitTestResult()>& body) const;

// The listed names not in `names` (the tests that exist), in order.
std::vector<std::string> stale(const std::set<std::string>& names) const;

private:
std::map<std::string, std::string, std::less<>> m_fixes;
};



} // namespace Pokemon



} // namespace PokemonAutomation
#endif
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
/* Known Failing Tests
*
* From: https://github.com/PokemonAutomation/
*
* The known-failing list on fake tests: one on the list passes while it
* fails and fails once it passes, one off the list keeps its result, an
* exception is a failure, and entries naming no test are stale.
*
*/

#include <format>
#include <optional>
#include <set>
#include <string>
#include <utility>
#include <vector>
#include "Common/Cpp/Exceptions.h"
#include "Common/Cpp/Logging/AbstractLogger.h"
#include "Common/Cpp/TestRunners/UnitTest.h"
#include "Common/Cpp/TestRunners/UnitTestDatabase.h"
#include "Pokemon_KnownFailing.h"
#include "Pokemon_KnownFailing_Tests.h"

namespace PokemonAutomation{
namespace Pokemon{


namespace{

class LinesLogger : public Logger{
public:
using Logger::log;
virtual void log(const std::string& msg, Color) override{
lines.push_back(msg);
}

std::vector<std::string> lines;
};

const KnownFailing& known_failing(){
static const KnownFailing list({
{"listed", "the planner's goal kinds"},
{"listed and gone", "a fix for a test since renamed"},
});
return list;
}

// How the fake test's body ends.
enum class BodyOutcome{
PASSES,
FAILS,
THROWS,
};

struct KnownFailingCase{
struct Input{
std::string test;
BodyOutcome body;
};
struct Expectation{
bool passes;
std::string message;
std::vector<std::string> log;
};

std::string name;
Input input;
Expectation expectation;
};

const KnownFailingCase KNOWN_FAILING_CASES[] = {
{
.name = "a listed test passes while it fails",
.input = {.test = "listed", .body = BodyOutcome::FAILS},
.expectation = {.passes = true, .message = "", .log = {"Known failing until the planner's goal kinds: stopped at the gate."}},
},
{
.name = "a listed test fails once it passes",
.input = {.test = "listed", .body = BodyOutcome::PASSES},
.expectation = {
.passes = false,
.message = "on KNOWN_FAILING (the planner's goal kinds) but passes now: take it off the list.",
.log = {},
},
},
{
.name = "a listed test's exception is its known failure",
.input = {.test = "listed", .body = BodyOutcome::THROWS},
.expectation = {.passes = true, .message = "", .log = {"Known failing until the planner's goal kinds: ParseException: no row 99.99"}},
},
{
.name = "an unlisted test keeps its failure",
.input = {.test = "unlisted", .body = BodyOutcome::FAILS},
.expectation = {.passes = false, .message = "stopped at the gate.", .log = {}},
},
{
.name = "an unlisted test keeps its pass",
.input = {.test = "unlisted", .body = BodyOutcome::PASSES},
.expectation = {.passes = true, .message = "", .log = {}},
},
{
.name = "an unlisted test's exception is its failure",
.input = {.test = "unlisted", .body = BodyOutcome::THROWS},
.expectation = {.passes = false, .message = "ParseException: no row 99.99", .log = {}},
},
};

UnitTestResult fake_body(BodyOutcome outcome){
switch (outcome){
case BodyOutcome::PASSES:
return true;
case BodyOutcome::FAILS:
return "stopped at the gate.";
case BodyOutcome::THROWS:
throw ParseException("no row 99.99");
}
return true;
}

class Test_KnownFailing : public UnitTest{
public:
explicit Test_KnownFailing(KnownFailingCase test_case)
: UnitTest(std::format("Pokemon::Navigation - Known failing: {}", test_case.name))
, m_case(std::move(test_case))
{}
virtual UnitTestResult run(Logger&, CancellableScope&) const override{
LinesLogger logger;
const UnitTestResult result = known_failing().run(logger, m_case.input.test, [this]{
return fake_body(m_case.input.body);
});
const KnownFailingCase::Expectation& expected = m_case.expectation;
const bool passed = result.result == UnitTestResult::PASSED;
if (passed != expected.passes){
return std::format("expected {}, got \"{}\".", expected.passes ? "a pass" : "a failure", result.message);
}
if (result.message != expected.message){
return std::format("message: expected \"{}\", got \"{}\".", expected.message, result.message);
}
if (logger.lines != expected.log){
return std::format("log: expected {} lines, got {}: \"{}\".", expected.log.size(), logger.lines.size(), logger.lines.empty() ? "" : logger.lines[0]);
}
return true;
}

private:
KnownFailingCase m_case;
};


struct StaleCase{
struct Input{
std::set<std::string> tests;
};
struct Expectation{
std::vector<std::string> stale;
};

std::string name;
Input input;
Expectation expectation;
};

const StaleCase STALE_CASES[] = {
{
.name = "an entry naming no test is stale",
.input = {.tests = {"listed", "unlisted"}},
.expectation = {.stale = {"listed and gone"}},
},
{
.name = "entries naming tests are not",
.input = {.tests = {"listed", "listed and gone"}},
.expectation = {.stale = {}},
},
};

class Test_KnownFailingStale : public UnitTest{
public:
explicit Test_KnownFailingStale(StaleCase test_case)
: UnitTest(std::format("Pokemon::Navigation - Known failing: {}", test_case.name))
, m_case(std::move(test_case))
{}
virtual UnitTestResult run(Logger&, CancellableScope&) const override{
const std::vector<std::string> stale = known_failing().stale(m_case.input.tests);
if (stale != m_case.expectation.stale){
return std::format("expected {} stale entries, got {}.", m_case.expectation.stale.size(), stale.size());
}
return true;
}

private:
StaleCase m_case;
};

} // namespace


void add_tests_KnownFailing(UnitTestDatabase& database){
for (const KnownFailingCase& item : KNOWN_FAILING_CASES){
database.add<Test_KnownFailing>(item);
}
for (const StaleCase& item : STALE_CASES){
database.add<Test_KnownFailingStale>(item);
}
}



} // namespace Pokemon
} // namespace PokemonAutomation
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
/* Known Failing Tests
*
* From: https://github.com/PokemonAutomation/
*
*/

#ifndef PokemonAutomation_Pokemon_KnownFailing_Tests_H
#define PokemonAutomation_Pokemon_KnownFailing_Tests_H

#include "Common/Cpp/TestRunners/UnitTestDatabase.h"

namespace PokemonAutomation{
namespace Pokemon{


// Registers the known-failing list tests.
void add_tests_KnownFailing(UnitTestDatabase& database);



} // namespace Pokemon
} // namespace PokemonAutomation
#endif
Loading
Loading