Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,10 @@
#include "Pokemon/Navigation/Pokemon_RouteLog_Tests.h"
#include "Pokemon/Navigation/Pokemon_RuntimeMap_Tests.h"
#include "Pokemon/Navigation/Pokemon_WallRecovery_Tests.h"
#include "Pokemon/Navigation/Simulation/Pokemon_KnownFailing_Tests.h"
#include "Pokemon/Navigation/Simulation/Pokemon_SimGolden_Tests.h"
#include "Pokemon/Navigation/Simulation/Pokemon_SimSeeds_Tests.h"
#include "Pokemon/Navigation/Simulation/Pokemon_SimSweep_Tests.h"
#include "VideoGames/PokemonHome/PokemonHome_Tests.h"
#include "VideoGames/PokemonSwSh/PokemonSwSh_Tests.h"
#include "VideoGames/PokemonLA/PokemonLA_Tests.h"
Expand All @@ -62,7 +65,7 @@



UnitTestDatabase make_UNIT_TESTS_ALL(){

Check warning on line 68 in SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp

View workflow job for this annotation

GitHub Actions / code-health

Code health (advisory)

PokemonAutomation::ComputerPrograms::make_UNIT_TESTS_ALL(): function_lines=49 (advisory above 25, limit 50)
UnitTestDatabase ret;

add_tests_QtEventThreadPool(ret, GlobalThreadPools::computation_normal());
Expand Down Expand Up @@ -91,6 +94,9 @@
Pokemon::add_tests_RuntimeMap(ret);
Pokemon::add_tests_RouteLog(ret);
Pokemon::add_tests_SimGolden(ret);
Pokemon::add_tests_SimSweep(ret);
Pokemon::add_tests_SimSeeds(ret);
Pokemon::add_tests_KnownFailing(ret);
Pokemon::add_tests_AutoStory(ret);
NintendoSwitch::add_tests_CheckOnlineDetector(ret);
NintendoSwitch::add_tests_FailedToConnectDetector(ret);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/* Known Failing
*
* From: https://github.com/PokemonAutomation/
*
*/

#include <format>
#include <utility>
#include "Common/Cpp/Exceptions.h"
#include "Pokemon_KnownFailing.h"

namespace PokemonAutomation{
namespace Pokemon{


KnownFailing::KnownFailing(std::map<std::string, std::string> fixes)
: m_fixes(std::make_move_iterator(fixes.begin()), std::make_move_iterator(fixes.end()))
{}

UnitTestResult KnownFailing::run(Logger& logger, std::string_view name, const std::function<UnitTestResult()>& body) const{
UnitTestResult result;
try{
result = body();
}catch (Exception& e){
result = std::format("{}: {}", e.name(), e.message());
}
const auto known = m_fixes.find(name);
if (known == m_fixes.end()){
return result;
}
if (result.result == UnitTestResult::PASSED){
return std::format("on KNOWN_FAILING ({}) but passes now: take it off the list.", known->second);
}
logger.log(std::format("Known failing until {}: {}", known->second, result.message));
return true;
}

std::vector<std::string> KnownFailing::stale(const std::set<std::string>& names) const{
std::vector<std::string> out;
for (const auto& [name, fix] : m_fixes){
if (!names.contains(name)){
out.push_back(name);
}
}
return out;
}



} // namespace Pokemon



} // namespace PokemonAutomation
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/* Known Failing
*
* From: https://github.com/PokemonAutomation/
*
* The list of sim tests the program fails today, each with the change
* likely to fix it, shared by the games' fault sweeps and console-failure
* cases. A test on the list passes while it fails, so the suite stays
* green with the failure logged, and fails once it passes, so the entry
* comes off. stale() finds the entries that name no test: a renamed test
* would otherwise drop off the list silently and fail the suite.
*
*/

#ifndef PokemonAutomation_Pokemon_KnownFailing_H
#define PokemonAutomation_Pokemon_KnownFailing_H

#include <functional>
#include <map>
#include <set>
#include <string>
#include <string_view>
#include <vector>
#include "Common/Cpp/Logging/AbstractLogger.h"
#include "Common/Cpp/TestRunners/UnitTest.h"

namespace PokemonAutomation{
namespace Pokemon{


// The tests known to fail, by name, with their likely fixes.
class KnownFailing{
public:
// `fixes`: by test name (or case name, as the game keys it), the
// change likely to fix it.
explicit KnownFailing(std::map<std::string, std::string> fixes);

// Runs test `name`'s `body`, an Exception it throws being its failure,
// and gives its result as the list sees it: as is if `name` is not
// listed; a pass that logs the failure if it is and fails; a failure
// "on KNOWN_FAILING (<fix>) but passes now: take it off the list." if
// it is and passes.
UnitTestResult run(Logger& logger, std::string_view name, const std::function<UnitTestResult()>& body) const;

// The listed names not in `names` (the tests that exist), in order.
std::vector<std::string> stale(const std::set<std::string>& names) const;

private:
std::map<std::string, std::string, std::less<>> m_fixes;
};



} // namespace Pokemon



} // namespace PokemonAutomation
#endif
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
/* Known Failing Tests
*
* From: https://github.com/PokemonAutomation/
*
* The known-failing list on fake tests: one on the list passes while it
* fails and fails once it passes, one off the list keeps its result, an
* exception is a failure, and entries naming no test are stale.
*
*/

#include <format>
#include <optional>
#include <set>
#include <string>
#include <utility>
#include <vector>
#include "Common/Cpp/Exceptions.h"
#include "Common/Cpp/Logging/AbstractLogger.h"
#include "Common/Cpp/TestRunners/UnitTest.h"
#include "Common/Cpp/TestRunners/UnitTestDatabase.h"
#include "Pokemon_KnownFailing.h"
#include "Pokemon_KnownFailing_Tests.h"

namespace PokemonAutomation{
namespace Pokemon{


namespace{

class LinesLogger : public Logger{
public:
using Logger::log;
virtual void log(const std::string& msg, Color) override{
lines.push_back(msg);
}

std::vector<std::string> lines;
};

const KnownFailing& known_failing(){
static const KnownFailing list({
{"listed", "the planner's goal kinds"},
{"listed and gone", "a fix for a test since renamed"},
});
return list;
}

// How the fake test's body ends.
enum class BodyOutcome{
PASSES,
FAILS,
THROWS,
};

struct KnownFailingCase{
struct Input{
std::string test;
BodyOutcome body;
};
struct Expectation{
bool passes;
std::string message;
std::vector<std::string> log;
};

std::string name;
Input input;
Expectation expectation;
};

const KnownFailingCase KNOWN_FAILING_CASES[] = {
{
.name = "a listed test passes while it fails",
.input = {.test = "listed", .body = BodyOutcome::FAILS},
.expectation = {.passes = true, .message = "", .log = {"Known failing until the planner's goal kinds: stopped at the gate."}},
},
{
.name = "a listed test fails once it passes",
.input = {.test = "listed", .body = BodyOutcome::PASSES},
.expectation = {
.passes = false,
.message = "on KNOWN_FAILING (the planner's goal kinds) but passes now: take it off the list.",
.log = {},
},
},
{
.name = "a listed test's exception is its known failure",
.input = {.test = "listed", .body = BodyOutcome::THROWS},
.expectation = {.passes = true, .message = "", .log = {"Known failing until the planner's goal kinds: ParseException: no row 99.99"}},
},
{
.name = "an unlisted test keeps its failure",
.input = {.test = "unlisted", .body = BodyOutcome::FAILS},
.expectation = {.passes = false, .message = "stopped at the gate.", .log = {}},
},
{
.name = "an unlisted test keeps its pass",
.input = {.test = "unlisted", .body = BodyOutcome::PASSES},
.expectation = {.passes = true, .message = "", .log = {}},
},
{
.name = "an unlisted test's exception is its failure",
.input = {.test = "unlisted", .body = BodyOutcome::THROWS},
.expectation = {.passes = false, .message = "ParseException: no row 99.99", .log = {}},
},
};

UnitTestResult fake_body(BodyOutcome outcome){
switch (outcome){
case BodyOutcome::PASSES:
return true;
case BodyOutcome::FAILS:
return "stopped at the gate.";
case BodyOutcome::THROWS:
throw ParseException("no row 99.99");
}
return true;
}

class Test_KnownFailing : public UnitTest{
public:
explicit Test_KnownFailing(KnownFailingCase test_case)
: UnitTest(std::format("Pokemon::Navigation - Known failing: {}", test_case.name))
, m_case(std::move(test_case))
{}
virtual UnitTestResult run(Logger&, CancellableScope&) const override{
LinesLogger logger;
const UnitTestResult result = known_failing().run(logger, m_case.input.test, [this]{
return fake_body(m_case.input.body);
});
const KnownFailingCase::Expectation& expected = m_case.expectation;
const bool passed = result.result == UnitTestResult::PASSED;
if (passed != expected.passes){
return std::format("expected {}, got \"{}\".", expected.passes ? "a pass" : "a failure", result.message);
}
if (result.message != expected.message){
return std::format("message: expected \"{}\", got \"{}\".", expected.message, result.message);
}
if (logger.lines != expected.log){
return std::format("log: expected {} lines, got {}: \"{}\".", expected.log.size(), logger.lines.size(), logger.lines.empty() ? "" : logger.lines[0]);
}
return true;
}

private:
KnownFailingCase m_case;
};


struct StaleCase{
struct Input{
std::set<std::string> tests;
};
struct Expectation{
std::vector<std::string> stale;
};

std::string name;
Input input;
Expectation expectation;
};

const StaleCase STALE_CASES[] = {
{
.name = "an entry naming no test is stale",
.input = {.tests = {"listed", "unlisted"}},
.expectation = {.stale = {"listed and gone"}},
},
{
.name = "entries naming tests are not",
.input = {.tests = {"listed", "listed and gone"}},
.expectation = {.stale = {}},
},
};

class Test_KnownFailingStale : public UnitTest{
public:
explicit Test_KnownFailingStale(StaleCase test_case)
: UnitTest(std::format("Pokemon::Navigation - Known failing: {}", test_case.name))
, m_case(std::move(test_case))
{}
virtual UnitTestResult run(Logger&, CancellableScope&) const override{
const std::vector<std::string> stale = known_failing().stale(m_case.input.tests);
if (stale != m_case.expectation.stale){
return std::format("expected {} stale entries, got {}.", m_case.expectation.stale.size(), stale.size());
}
return true;
}

private:
StaleCase m_case;
};

} // namespace


void add_tests_KnownFailing(UnitTestDatabase& database){
for (const KnownFailingCase& item : KNOWN_FAILING_CASES){
database.add<Test_KnownFailing>(item);
}
for (const StaleCase& item : STALE_CASES){
database.add<Test_KnownFailingStale>(item);
}
}



} // namespace Pokemon
} // namespace PokemonAutomation
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
/* Known Failing Tests
*
* From: https://github.com/PokemonAutomation/
*
*/

#ifndef PokemonAutomation_Pokemon_KnownFailing_Tests_H
#define PokemonAutomation_Pokemon_KnownFailing_Tests_H

#include "Common/Cpp/TestRunners/UnitTestDatabase.h"

namespace PokemonAutomation{
namespace Pokemon{


// Registers the known-failing list tests.
void add_tests_KnownFailing(UnitTestDatabase& database);



} // namespace Pokemon
} // namespace PokemonAutomation
#endif
Loading
Loading