Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion SerialPrograms/Source/CommonFramework/Main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -220,20 +220,34 @@ int run_program(int argc, char *argv[]){
GlobalThreadPools::qt_event_threadpool();

bool augment_test_color_distortions = false;
std::string test_substring;
bool test_substring_missing_value = false;
for (size_t i = 0; i < argc; i++){
constexpr const char* force_run_tests = "--command-line-test-mode";
constexpr const char* distort_test_images = "--augment-color-distortions";
constexpr const char* filter_tests_substring = "--test-substring";

if (strcmp(argv[i], force_run_tests) == 0){
GlobalSettings::instance().COMMAND_LINE_TEST_MODE = true;
}
if (strcmp(argv[i], distort_test_images) == 0){
augment_test_color_distortions = true;
}
if (strcmp(argv[i], filter_tests_substring) == 0){
if (i + 1 < argc){
test_substring = argv[++i];
}else{
test_substring_missing_value = true;
}
}
}
if (test_substring_missing_value){
global_logger_command_line().log("--test-substring requires a value", COLOR_RED);

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AI review: FYI: this missing-value error fires even when --command-line-test-mode isn't passed, i.e. a stray --test-substring on a GUI launch also exits 1. That strict behavior seems right for a malformed flag, just noting the flag is parsed outside of test mode.

return 1;
}

if (GlobalSettings::instance().COMMAND_LINE_TEST_MODE){
return run_command_line_tests(augment_test_color_distortions);
return run_command_line_tests(augment_test_color_distortions, test_substring);
}

// Check whether the hardware is powerful enough to run this program.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,25 @@ const StringSelectDatabase& TEST_DATABASE(){
}


namespace{

// For each test in `UNIT_TESTS_ALL()` whose name contains `filter`, in name order, call
// `add_test(test)`. An empty `filter` matches all the tests. Return the number of tests
// visited.
template <class AddTest>
size_t add_tests_matching(const std::string& filter, AddTest&& add_test){
size_t count = 0;
for (const auto& test : UNIT_TESTS_ALL()){
if (filter.empty() || test.first.contains(filter)){
count++;
add_test(test.second);
}
}
return count;
}

}



UnitTestRunner_Descriptor::UnitTestRunner_Descriptor()
Expand Down Expand Up @@ -237,15 +256,12 @@ void UnitTestRunner::on_config_value_changed(void* object){
if (object == &SUBSTRING_TEST){
std::string name = SUBSTRING_TEST;
TESTS_TO_RUN.set("");
size_t count = 0;
for (const auto& test : UNIT_TESTS_ALL()){
if (test.first.contains(name)){
count++;
if (count <= 1000){
TESTS_TO_RUN.append(test.first + "\n");
}
size_t num_shown = 0;
size_t count = add_tests_matching(name, [&](std::shared_ptr<const UnitTest> test){
if (++num_shown <= 1000){
TESTS_TO_RUN.append(test->name() + "\n");
}
}
});
if (count > 1000){
TESTS_TO_RUN.append("<" + tostr_u_commas(count - 1000) + " more...>\n");
}
Expand Down Expand Up @@ -276,6 +292,12 @@ void UnitTestRunner::program(ProgramEnvironment& env, CancellableScope& scope){

const UnitTestDatabase& all_tests = UNIT_TESTS_ALL();

auto add_matching = [&](const std::string& filter){
add_tests_matching(filter, [&](std::shared_ptr<const UnitTest> test){
runner.add_test(test);
});
};

switch ((RunMode)RUN_MODE){
case RunMode::RUN_ONE:{
auto iter = all_tests.find(SINGLE_TEST.slug());
Expand All @@ -289,22 +311,13 @@ void UnitTestRunner::program(ProgramEnvironment& env, CancellableScope& scope){
runner.add_test(iter->second);
break;
}
case RunMode::RUN_SUBSTRING_MATCH:{
std::string name = SUBSTRING_TEST;
for (const auto& test : all_tests){
if (test.first.contains(name)){
runner.add_test(test.second);
}
}
case RunMode::RUN_SUBSTRING_MATCH:
add_matching(SUBSTRING_TEST);
break;
}
case RunMode::RUN_ALL:{
for (const auto& test : all_tests){
runner.add_test(test.second);
}
case RunMode::RUN_ALL:
add_matching("");
break;
}
}

runner.run();
}
Expand Down Expand Up @@ -365,7 +378,7 @@ void UnitTestRunner::on_test_finished(



bool CommandLineUnitTestRunner::run(bool augment_color_distortions){
bool CommandLineUnitTestRunner::run(bool augment_color_distortions, const std::string& test_substring){
if (augment_color_distortions){
// Load the distortion data before any test runs. Each `ImageUnitTest` checks the global
// manager when it runs, and only runs with distortions if it is loaded.
Expand All @@ -391,8 +404,15 @@ bool CommandLineUnitTestRunner::run(bool augment_color_distortions){
GlobalThreadPools::unlimited_normal()
);
runner.add_listener(*this);
for (const auto& test : UNIT_TESTS_ALL()){
runner.add_test(test.second);
size_t num_matching = add_tests_matching(test_substring, [&](std::shared_ptr<const UnitTest> test){
runner.add_test(test);
});
if (!test_substring.empty()){
if (num_matching == 0){
m_logger.log("No test name contains \"" + test_substring + "\". No tests will be run.", COLOR_ORANGE);

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AI review: FYI (documented, so not a blocker): a substring that matches nothing exits 0, so a typo'd filter in CI would produce a green run with zero tests - the orange warning is the only signal. Worth knowing if anyone wires this into a pipeline; if that becomes a problem, a distinct exit code for "zero tests ran due to filter" would be the natural fix.

}else{
m_logger.log("Running the " + tostr_u_commas(num_matching) + " tests whose name contains \"" + test_substring + "\"");

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AI review: Nit: when exactly one test matches this reads "Running the 1 tests whose name contains ...". No need to block on it, but if you're touching this line anyway a "test(s)" or singular/plural branch would read better.

}
}
runner.run();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,12 +89,15 @@ class CommandLineUnitTestRunner : public PokemonAutomation::UnitTestRunner::List
, m_failed_tests(0)
{}

// Run all the unit tests. Return true if any test failed.
// Run the unit tests. Return true if any test failed.
// If `augment_color_distortions` is true, also run each image test under different color
// distortion augmentation to widen the test coverage, using the distortion data in the
// CaptureCardCalibration/ folder of the test repo. If that data can't be loaded, log the
// error and return true without running any test.
bool run(bool augment_color_distortions);
// If `test_substring` is not empty, only run the tests whose name contains that substring
// (case sensitive, the same matching as the GUI's "Run All Tests Containing This Substring"
// mode). If nothing matches, no test runs and a warning is logged.
bool run(bool augment_color_distortions, const std::string& test_substring);


private:
Expand Down
10 changes: 7 additions & 3 deletions SerialPrograms/Source/Tests/CommandLineTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,21 @@ using std::endl;
namespace PokemonAutomation{


int run_command_line_tests(bool augment_color_distortions){
int run_command_line_tests(bool augment_color_distortions, const std::string& test_substring){
install_crash_backtrace_handlers();
// Keep stdout line-buffered: when piped, stdio defaults to full
// buffering, and the last few KB of test output - including the lines
// right before a crash - would otherwise be lost in the buffer. An
// explicit buffer size is required: with 0, MSVC's ucrt can end up with
// a zero-size buffer, which corrupts the next stdout write.
setvbuf(stdout, nullptr, _IOLBF, 8192);
cout << "Running parallel unit tests..." << endl;
if (test_substring.empty()){
cout << "Running parallel unit tests..." << endl;
}else{
cout << "Running parallel unit tests matching substring \"" << test_substring << "\"..." << endl;
}
ComputerPrograms::CommandLineUnitTestRunner runner(global_logger_command_line());
if (runner.run(augment_color_distortions)){
if (runner.run(augment_color_distortions, test_substring)){
return 1;
}
cout << "Running parallel unit tests... Done!" << endl;
Expand Down
12 changes: 11 additions & 1 deletion SerialPrograms/Source/Tests/CommandLineTests.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@
* loaded, no test runs and the program returns 1. See SerialPrograms/Source/Tests/ImageUnitTest.h
* and Common/Cpp/TestRunners/TestImageDistortionManager.h.
*
* Substring filter:
* Pass "--test-substring <value>" to only run the tests whose name contains <value> (case
* sensitive, the same matching as the GUI's "Run All Tests Containing This Substring" mode).
* Without it, all the tests are run. An empty value runs all the tests too.
*
* The same tests can also be run from the GUI with the "Unit Test Runner" program, which can
* run a single test, the tests matching a substring, or all tests.
*/
Expand All @@ -41,15 +46,20 @@
#define PokemonAutomation_Tests_CommandLineTests_H


#include <string>


namespace PokemonAutomation{


// Called by main() to run all the unit tests on command line, without launching any GUI.
// This function is only called when GlobalSettings::COMMAND_LINE_TEST_MODE is true.
// `augment_color_distortions` is set by the "--augment-color-distortions" argument: also run image tests
// with color distortion augmentation, using the distortion data from the test repo.
// `test_substring` is set by the "--test-substring <value>" argument: only run the tests whose name
// contains that substring. An empty string runs all the tests.
// Return 0 if all tests are passed, and 1 if any test failed or the distortion data can't be loaded.
int run_command_line_tests(bool augment_color_distortions);
int run_command_line_tests(bool augment_color_distortions, const std::string& test_substring);



Expand Down
Loading