From 87d06a9211448c7059cf9a59b91ae82ef4dddc48 Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:51:02 +0000 Subject: [PATCH 1/3] Command-line test runner: filter tests by name substring Add a "--test-substring " argument to the command-line unit test runner (--command-line-test-mode): it only runs the tests whose name contains the given substring, using the same case-sensitive matching as the GUI's "Run All Tests Containing This Substring" mode. Without the argument (or with an empty value) all tests run, as before. - Main.cpp: parse the argument (error + exit 1 if it has no value). - CommandLineTests.h/.cpp: pass the substring through to the runner. - CommandLineUnitTestRunner::run(): add only the matching tests; log how many will run, and warn in orange if nothing matches. --- SerialPrograms/Source/CommonFramework/Main.cpp | 16 +++++++++++++++- .../ComputerPrograms_UnitTestRunner.cpp | 15 +++++++++++++-- .../ComputerPrograms_UnitTestRunner.h | 5 ++++- SerialPrograms/Source/Tests/CommandLineTests.cpp | 10 +++++++--- SerialPrograms/Source/Tests/CommandLineTests.h | 12 +++++++++++- 5 files changed, 50 insertions(+), 8 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Main.cpp b/SerialPrograms/Source/CommonFramework/Main.cpp index c493f48b54..ef6742e2eb 100644 --- a/SerialPrograms/Source/CommonFramework/Main.cpp +++ b/SerialPrograms/Source/CommonFramework/Main.cpp @@ -220,9 +220,12 @@ int run_program(int argc, char *argv[]){ GlobalThreadPools::qt_event_threadpool(); bool augment_test_color_distortions = false; + std::string test_substring; + bool test_substring_missing_value = false; for (size_t i = 0; i < argc; i++){ constexpr const char* force_run_tests = "--command-line-test-mode"; constexpr const char* distort_test_images = "--augment-color-distortions"; + constexpr const char* filter_tests_substring = "--test-substring"; if (strcmp(argv[i], force_run_tests) == 0){ GlobalSettings::instance().COMMAND_LINE_TEST_MODE = true; @@ -230,10 +233,21 @@ int run_program(int argc, char *argv[]){ if (strcmp(argv[i], distort_test_images) == 0){ augment_test_color_distortions = true; } + if (strcmp(argv[i], filter_tests_substring) == 0){ + if (i + 1 < argc){ + test_substring = argv[++i]; + }else{ + test_substring_missing_value = true; + } + } + } + if (test_substring_missing_value){ + global_logger_command_line().log("--test-substring requires a value", COLOR_RED); + return 1; } if (GlobalSettings::instance().COMMAND_LINE_TEST_MODE){ - return run_command_line_tests(augment_test_color_distortions); + return run_command_line_tests(augment_test_color_distortions, test_substring); } // Check whether the hardware is powerful enough to run this program. diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index 9b48682bc1..d50992c97f 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -365,7 +365,7 @@ void UnitTestRunner::on_test_finished( -bool CommandLineUnitTestRunner::run(bool augment_color_distortions){ +bool CommandLineUnitTestRunner::run(bool augment_color_distortions, const std::string& test_substring){ if (augment_color_distortions){ // Load the distortion data before any test runs. Each `ImageUnitTest` checks the global // manager when it runs, and only runs with distortions if it is loaded. @@ -391,8 +391,19 @@ bool CommandLineUnitTestRunner::run(bool augment_color_distortions){ GlobalThreadPools::unlimited_normal() ); runner.add_listener(*this); + size_t num_matching = 0; for (const auto& test : UNIT_TESTS_ALL()){ - runner.add_test(test.second); + if (test_substring.empty() || test.first.contains(test_substring)){ + num_matching++; + runner.add_test(test.second); + } + } + if (!test_substring.empty()){ + if (num_matching == 0){ + m_logger.log("No test name contains \"" + test_substring + "\". No tests will be run.", COLOR_ORANGE); + }else{ + m_logger.log("Running the " + tostr_u_commas(num_matching) + " tests whose name contains \"" + test_substring + "\""); + } } runner.run(); diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h index c3aeb94f07..4a520aff7d 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h @@ -94,7 +94,10 @@ class CommandLineUnitTestRunner : public PokemonAutomation::UnitTestRunner::List // distortion augmentation to widen the test coverage, using the distortion data in the // CaptureCardCalibration/ folder of the test repo. If that data can't be loaded, log the // error and return true without running any test. - bool run(bool augment_color_distortions); + // If `test_substring` is not empty, only run the tests whose name contains that substring + // (case sensitive, the same matching as the GUI's "Run All Tests Containing This Substring" + // mode). If nothing matches, no test runs and a warning is logged. + bool run(bool augment_color_distortions, const std::string& test_substring); private: diff --git a/SerialPrograms/Source/Tests/CommandLineTests.cpp b/SerialPrograms/Source/Tests/CommandLineTests.cpp index bed7f8997e..90a9830510 100644 --- a/SerialPrograms/Source/Tests/CommandLineTests.cpp +++ b/SerialPrograms/Source/Tests/CommandLineTests.cpp @@ -18,7 +18,7 @@ using std::endl; namespace PokemonAutomation{ -int run_command_line_tests(bool augment_color_distortions){ +int run_command_line_tests(bool augment_color_distortions, const std::string& test_substring){ install_crash_backtrace_handlers(); // Keep stdout line-buffered: when piped, stdio defaults to full // buffering, and the last few KB of test output - including the lines @@ -26,9 +26,13 @@ int run_command_line_tests(bool augment_color_distortions){ // explicit buffer size is required: with 0, MSVC's ucrt can end up with // a zero-size buffer, which corrupts the next stdout write. setvbuf(stdout, nullptr, _IOLBF, 8192); - cout << "Running parallel unit tests..." << endl; + if (test_substring.empty()){ + cout << "Running parallel unit tests..." << endl; + }else{ + cout << "Running parallel unit tests matching substring \"" << test_substring << "\"..." << endl; + } ComputerPrograms::CommandLineUnitTestRunner runner(global_logger_command_line()); - if (runner.run(augment_color_distortions)){ + if (runner.run(augment_color_distortions, test_substring)){ return 1; } cout << "Running parallel unit tests... Done!" << endl; diff --git a/SerialPrograms/Source/Tests/CommandLineTests.h b/SerialPrograms/Source/Tests/CommandLineTests.h index 010de0c81f..bea6858b51 100644 --- a/SerialPrograms/Source/Tests/CommandLineTests.h +++ b/SerialPrograms/Source/Tests/CommandLineTests.h @@ -32,6 +32,11 @@ * loaded, no test runs and the program returns 1. See SerialPrograms/Source/Tests/ImageUnitTest.h * and Common/Cpp/TestRunners/TestImageDistortionManager.h. * + * Substring filter: + * Pass "--test-substring " to only run the tests whose name contains (case + * sensitive, the same matching as the GUI's "Run All Tests Containing This Substring" mode). + * Without it, all the tests are run. An empty value runs all the tests too. + * * The same tests can also be run from the GUI with the "Unit Test Runner" program, which can * run a single test, the tests matching a substring, or all tests. */ @@ -41,6 +46,9 @@ #define PokemonAutomation_Tests_CommandLineTests_H +#include + + namespace PokemonAutomation{ @@ -48,8 +56,10 @@ namespace PokemonAutomation{ // This function is only called when GlobalSettings::COMMAND_LINE_TEST_MODE is true. // `augment_color_distortions` is set by the "--augment-color-distortions" argument: also run image tests // with color distortion augmentation, using the distortion data from the test repo. +// `test_substring` is set by the "--test-substring " argument: only run the tests whose name +// contains that substring. An empty string runs all the tests. // Return 0 if all tests are passed, and 1 if any test failed or the distortion data can't be loaded. -int run_command_line_tests(bool augment_color_distortions); +int run_command_line_tests(bool augment_color_distortions, const std::string& test_substring); From a11c7d05cce927b72dd1074c99908b471b6d3604 Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:57:12 +0000 Subject: [PATCH 2/3] Dedup the GUI and command-line unit test runners The two runners (UnitTestRunner::program and CommandLineUnitTestRunner::run) each had their own copy of the "add the tests whose name contains this substring" loop (and the GUI's "Tests to Run" preview had a third copy), and each of their on_test_finished listeners re-implemented the same result classification into passed/failed/skipped/OOM buckets. - Add a file-local add_tests_matching(filter, add_test) helper and use it in the GUI's run-substring-match / run-all modes, the GUI's "Tests to Run" preview, and the command-line runner. - Add UnitTestResultCounters, shared by both runners: one place defines which bucket each result kind goes into (OOM is a failure and has its own counter) and returns the result's 1-based index within its bucket, which the GUI uses for its 1000-entry panel caps, exactly like the old fetch_add(1) + 1 sequences. The write-only m_total_failed goes away. --- .../ComputerPrograms_UnitTestRunner.cpp | 112 +++++++++--------- .../ComputerPrograms_UnitTestRunner.h | 65 ++++++++-- 2 files changed, 113 insertions(+), 64 deletions(-) diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index d50992c97f..e63baeb5de 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -107,6 +107,25 @@ const StringSelectDatabase& TEST_DATABASE(){ } +namespace{ + +// For each test in `UNIT_TESTS_ALL()` whose name contains `filter`, in name order, call +// `add_test(test)`. An empty `filter` matches all the tests. Return the number of tests +// visited. +template +size_t add_tests_matching(const std::string& filter, AddTest&& add_test){ + size_t count = 0; + for (const auto& test : UNIT_TESTS_ALL()){ + if (filter.empty() || test.first.contains(filter)){ + count++; + add_test(test.second); + } + } + return count; +} + +} + UnitTestRunner_Descriptor::UnitTestRunner_Descriptor() @@ -237,15 +256,12 @@ void UnitTestRunner::on_config_value_changed(void* object){ if (object == &SUBSTRING_TEST){ std::string name = SUBSTRING_TEST; TESTS_TO_RUN.set(""); - size_t count = 0; - for (const auto& test : UNIT_TESTS_ALL()){ - if (test.first.contains(name)){ - count++; - if (count <= 1000){ - TESTS_TO_RUN.append(test.first + "\n"); - } + size_t num_shown = 0; + size_t count = add_tests_matching(name, [&](std::shared_ptr test){ + if (++num_shown <= 1000){ + TESTS_TO_RUN.append(test->name() + "\n"); } - } + }); if (count > 1000){ TESTS_TO_RUN.append("<" + tostr_u_commas(count - 1000) + " more...>\n"); } @@ -255,7 +271,7 @@ void UnitTestRunner::on_config_value_changed(void* object){ void UnitTestRunner::program(ProgramEnvironment& env, CancellableScope& scope){ - m_total_failed.store(0, std::memory_order_release); + m_results.reset(); PASSED_TESTS.set(""); FAILED_TESTS.set(""); @@ -276,6 +292,12 @@ void UnitTestRunner::program(ProgramEnvironment& env, CancellableScope& scope){ const UnitTestDatabase& all_tests = UNIT_TESTS_ALL(); + auto add_matching = [&](const std::string& filter){ + add_tests_matching(filter, [&](std::shared_ptr test){ + runner.add_test(test); + }); + }; + switch ((RunMode)RUN_MODE){ case RunMode::RUN_ONE:{ auto iter = all_tests.find(SINGLE_TEST.slug()); @@ -289,22 +311,13 @@ void UnitTestRunner::program(ProgramEnvironment& env, CancellableScope& scope){ runner.add_test(iter->second); break; } - case RunMode::RUN_SUBSTRING_MATCH:{ - std::string name = SUBSTRING_TEST; - for (const auto& test : all_tests){ - if (test.first.contains(name)){ - runner.add_test(test.second); - } - } + case RunMode::RUN_SUBSTRING_MATCH: + add_matching(SUBSTRING_TEST); break; - } - case RunMode::RUN_ALL:{ - for (const auto& test : all_tests){ - runner.add_test(test.second); - } + case RunMode::RUN_ALL: + add_matching(""); break; } - } runner.run(); } @@ -320,43 +333,38 @@ void UnitTestRunner::on_test_finished( } UnitTestRunner_Descriptor::Stats& stats = m_env->current_stats(); stats.tests++; + // 1-based index of this result within its bucket: PASSED -> passed #, FAILED/OOM -> failed #. + uint64_t index = m_results.add(result); switch (result.result){ case UnitTestResult::NOT_RUN: + case UnitTestResult::SKIPPED: stats.skipped++; break; - case UnitTestResult::PASSED:{ - uint64_t passed = stats.passed.fetch_add(1) + 1; - if (passed <= 1000){ + case UnitTestResult::PASSED: + stats.passed++; + if (index <= 1000){ PASSED_TESTS.append(test->name() + "\n"); - }else if (passed == 1001){ + }else if (index == 1001){ PASSED_TESTS.append("\n"); } break; - } - case UnitTestResult::FAILED:{ - uint64_t failed = m_total_failed.fetch_add(1) + 1; + case UnitTestResult::FAILED: stats.failed++; - if (failed <= 1000){ + if (index <= 1000){ FAILED_TESTS.append("FAILED: " + test->name() + "\n"); - }else if (failed == 1001){ + }else if (index == 1001){ FAILED_TESTS.append("\n"); } break; - } - case UnitTestResult::SKIPPED: - stats.skipped++; - break; - case UnitTestResult::OOM:{ - uint64_t failed = m_total_failed.fetch_add(1) + 1; + case UnitTestResult::OOM: stats.oom++; - if (failed <= 1000){ + if (index <= 1000){ FAILED_TESTS.append("OOM: " + test->name() + "\n"); - }else if (failed == 1001){ + }else if (index == 1001){ FAILED_TESTS.append("\n"); } break; } - } m_env->update_stats(); } @@ -391,13 +399,9 @@ bool CommandLineUnitTestRunner::run(bool augment_color_distortions, const std::s GlobalThreadPools::unlimited_normal() ); runner.add_listener(*this); - size_t num_matching = 0; - for (const auto& test : UNIT_TESTS_ALL()){ - if (test_substring.empty() || test.first.contains(test_substring)){ - num_matching++; - runner.add_test(test.second); - } - } + size_t num_matching = add_tests_matching(test_substring, [&](std::shared_ptr test){ + runner.add_test(test); + }); if (!test_substring.empty()){ if (num_matching == 0){ m_logger.log("No test name contains \"" + test_substring + "\". No tests will be run.", COLOR_ORANGE); @@ -409,36 +413,32 @@ bool CommandLineUnitTestRunner::run(bool augment_color_distortions, const std::s m_logger.log( "Tests Finished:" - "\n Passed: " + tostr_u_commas(m_passed_tests.load(std::memory_order_acquire)) + - "\n Failed: " + tostr_u_commas(m_failed_tests.load(std::memory_order_acquire)) + - "\n Skipped: " + tostr_u_commas(m_skipped_tests.load(std::memory_order_acquire)) + "\n Passed: " + tostr_u_commas(m_results.num_passed()) + + "\n Failed: " + tostr_u_commas(m_results.num_failed()) + + "\n Skipped: " + tostr_u_commas(m_results.num_skipped()) ); - return m_failed_tests.load(std::memory_order_acquire) > 0; + return m_results.any_failed(); } void CommandLineUnitTestRunner::on_test_finished( std::shared_ptr test, UnitTestResult result ){ + m_results.add(result); switch (result.result){ case UnitTestResult::NOT_RUN: m_logger.log("NOT RUN: " + test->name(), COLOR_ORANGE); - m_skipped_tests++; break; case UnitTestResult::PASSED: m_logger.log("PASSED: " + test->name(), COLOR_BLUE); - m_passed_tests++; break; case UnitTestResult::FAILED: m_logger.log("FAILED: " + test->name(), COLOR_RED); - m_failed_tests++; break; case UnitTestResult::SKIPPED: m_logger.log("SKIPPED: " + test->name(), COLOR_ORANGE); - m_skipped_tests++; break; case UnitTestResult::OOM: m_logger.log("OOM: " + test->name(), COLOR_RED); - m_failed_tests++; break; } } diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h index 4a520aff7d..e280658a7c 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h @@ -8,6 +8,7 @@ #define PokemonAutomation_ComputerPrograms_UnitTestRunner_H #include +#include "Common/Cpp/Exceptions.h" #include "Common/Cpp/TestRunners/ParallelUnitTestRunner.h" #include "Common/Cpp/Options/StaticTextOption.h" #include "Common/Cpp/Options/EnumDropdownOption.h" @@ -20,6 +21,59 @@ namespace PokemonAutomation{ namespace ComputerPrograms{ +// Counters for finished test results, shared by the GUI and the command-line test runners. +// Both runners count OOM as a failure, and the GUI additionally tracks OOM separately. +struct UnitTestResultCounters{ + std::atomic passed{0}; + std::atomic failed{0}; + std::atomic skipped{0}; + std::atomic oom{0}; + + // Record a finished test. OOM is also a failure, and it is additionally counted in its own + // bucket. Return the 1-based index of the result within its bucket: the order in which + // results of that bucket were recorded, unique even when called from multiple threads. + uint64_t add(UnitTestResult result){ + switch (result.result){ + case UnitTestResult::NOT_RUN: + case UnitTestResult::SKIPPED: + return skipped.fetch_add(1) + 1; + case UnitTestResult::PASSED: + return passed.fetch_add(1) + 1; + case UnitTestResult::FAILED: + return failed.fetch_add(1) + 1; + case UnitTestResult::OOM: + oom.fetch_add(1); + return failed.fetch_add(1) + 1; + default: + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Unknown unit test result"); + } + } + + // Reset the counters for a new run. + void reset(){ + passed.store(0, std::memory_order_release); + failed.store(0, std::memory_order_release); + skipped.store(0, std::memory_order_release); + oom.store(0, std::memory_order_release); + } + + uint64_t num_passed() const{ + return passed.load(std::memory_order_acquire); + } + uint64_t num_failed() const{ + return failed.load(std::memory_order_acquire); + } + uint64_t num_skipped() const{ + return skipped.load(std::memory_order_acquire); + } + + // True if any test failed, including OOM. + bool any_failed() const{ + return num_failed() > 0; + } +}; + + class UnitTestRunner_Descriptor : public ComputerProgramDescriptor{ public: UnitTestRunner_Descriptor(); @@ -75,7 +129,7 @@ class UnitTestRunner TextEditOption FAILED_TESTS; ProgramEnvironment* m_env = nullptr; - std::atomic m_total_failed; + UnitTestResultCounters m_results; }; @@ -84,12 +138,9 @@ class CommandLineUnitTestRunner : public PokemonAutomation::UnitTestRunner::List public: CommandLineUnitTestRunner(Logger& logger) : m_logger(logger) - , m_skipped_tests(0) - , m_passed_tests(0) - , m_failed_tests(0) {} - // Run all the unit tests. Return true if any test failed. + // Run the unit tests. Return true if any test failed. // If `augment_color_distortions` is true, also run each image test under different color // distortion augmentation to widen the test coverage, using the distortion data in the // CaptureCardCalibration/ folder of the test repo. If that data can't be loaded, log the @@ -109,9 +160,7 @@ class CommandLineUnitTestRunner : public PokemonAutomation::UnitTestRunner::List private: Logger& m_logger; - std::atomic m_skipped_tests; - std::atomic m_passed_tests; - std::atomic m_failed_tests; + UnitTestResultCounters m_results; }; From 7b7cacf8efc220beb8f62cca4a6ec13d0332d359 Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Thu, 12 Nov 2026 02:06:12 +0000 Subject: [PATCH 3/3] Revert the shared result counters; keep the shared test selection Per review: UnitTestResultCounters/m_results was redundant for the GUI, whose Stats already track the same passed/failed/skipped/OOM buckets, and its one unique use (the combined FAILED+OOM cap sequence) was just the old m_total_failed. Revert both runners to their original atomics and per-case counting, keeping only the shared add_tests_matching() helper from the previous commit. --- .../ComputerPrograms_UnitTestRunner.cpp | 47 ++++++++------ .../ComputerPrograms_UnitTestRunner.h | 63 +++---------------- 2 files changed, 35 insertions(+), 75 deletions(-) diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index e63baeb5de..d2a39b8cba 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -271,7 +271,7 @@ void UnitTestRunner::on_config_value_changed(void* object){ void UnitTestRunner::program(ProgramEnvironment& env, CancellableScope& scope){ - m_results.reset(); + m_total_failed.store(0, std::memory_order_release); PASSED_TESTS.set(""); FAILED_TESTS.set(""); @@ -333,38 +333,43 @@ void UnitTestRunner::on_test_finished( } UnitTestRunner_Descriptor::Stats& stats = m_env->current_stats(); stats.tests++; - // 1-based index of this result within its bucket: PASSED -> passed #, FAILED/OOM -> failed #. - uint64_t index = m_results.add(result); switch (result.result){ case UnitTestResult::NOT_RUN: - case UnitTestResult::SKIPPED: stats.skipped++; break; - case UnitTestResult::PASSED: - stats.passed++; - if (index <= 1000){ + case UnitTestResult::PASSED:{ + uint64_t passed = stats.passed.fetch_add(1) + 1; + if (passed <= 1000){ PASSED_TESTS.append(test->name() + "\n"); - }else if (index == 1001){ + }else if (passed == 1001){ PASSED_TESTS.append("\n"); } break; - case UnitTestResult::FAILED: + } + case UnitTestResult::FAILED:{ + uint64_t failed = m_total_failed.fetch_add(1) + 1; stats.failed++; - if (index <= 1000){ + if (failed <= 1000){ FAILED_TESTS.append("FAILED: " + test->name() + "\n"); - }else if (index == 1001){ + }else if (failed == 1001){ FAILED_TESTS.append("\n"); } break; - case UnitTestResult::OOM: + } + case UnitTestResult::SKIPPED: + stats.skipped++; + break; + case UnitTestResult::OOM:{ + uint64_t failed = m_total_failed.fetch_add(1) + 1; stats.oom++; - if (index <= 1000){ + if (failed <= 1000){ FAILED_TESTS.append("OOM: " + test->name() + "\n"); - }else if (index == 1001){ + }else if (failed == 1001){ FAILED_TESTS.append("\n"); } break; } + } m_env->update_stats(); } @@ -413,32 +418,36 @@ bool CommandLineUnitTestRunner::run(bool augment_color_distortions, const std::s m_logger.log( "Tests Finished:" - "\n Passed: " + tostr_u_commas(m_results.num_passed()) + - "\n Failed: " + tostr_u_commas(m_results.num_failed()) + - "\n Skipped: " + tostr_u_commas(m_results.num_skipped()) + "\n Passed: " + tostr_u_commas(m_passed_tests.load(std::memory_order_acquire)) + + "\n Failed: " + tostr_u_commas(m_failed_tests.load(std::memory_order_acquire)) + + "\n Skipped: " + tostr_u_commas(m_skipped_tests.load(std::memory_order_acquire)) ); - return m_results.any_failed(); + return m_failed_tests.load(std::memory_order_acquire) > 0; } void CommandLineUnitTestRunner::on_test_finished( std::shared_ptr test, UnitTestResult result ){ - m_results.add(result); switch (result.result){ case UnitTestResult::NOT_RUN: m_logger.log("NOT RUN: " + test->name(), COLOR_ORANGE); + m_skipped_tests++; break; case UnitTestResult::PASSED: m_logger.log("PASSED: " + test->name(), COLOR_BLUE); + m_passed_tests++; break; case UnitTestResult::FAILED: m_logger.log("FAILED: " + test->name(), COLOR_RED); + m_failed_tests++; break; case UnitTestResult::SKIPPED: m_logger.log("SKIPPED: " + test->name(), COLOR_ORANGE); + m_skipped_tests++; break; case UnitTestResult::OOM: m_logger.log("OOM: " + test->name(), COLOR_RED); + m_failed_tests++; break; } } diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h index e280658a7c..3baffcc745 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h @@ -8,7 +8,6 @@ #define PokemonAutomation_ComputerPrograms_UnitTestRunner_H #include -#include "Common/Cpp/Exceptions.h" #include "Common/Cpp/TestRunners/ParallelUnitTestRunner.h" #include "Common/Cpp/Options/StaticTextOption.h" #include "Common/Cpp/Options/EnumDropdownOption.h" @@ -21,59 +20,6 @@ namespace PokemonAutomation{ namespace ComputerPrograms{ -// Counters for finished test results, shared by the GUI and the command-line test runners. -// Both runners count OOM as a failure, and the GUI additionally tracks OOM separately. -struct UnitTestResultCounters{ - std::atomic passed{0}; - std::atomic failed{0}; - std::atomic skipped{0}; - std::atomic oom{0}; - - // Record a finished test. OOM is also a failure, and it is additionally counted in its own - // bucket. Return the 1-based index of the result within its bucket: the order in which - // results of that bucket were recorded, unique even when called from multiple threads. - uint64_t add(UnitTestResult result){ - switch (result.result){ - case UnitTestResult::NOT_RUN: - case UnitTestResult::SKIPPED: - return skipped.fetch_add(1) + 1; - case UnitTestResult::PASSED: - return passed.fetch_add(1) + 1; - case UnitTestResult::FAILED: - return failed.fetch_add(1) + 1; - case UnitTestResult::OOM: - oom.fetch_add(1); - return failed.fetch_add(1) + 1; - default: - throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Unknown unit test result"); - } - } - - // Reset the counters for a new run. - void reset(){ - passed.store(0, std::memory_order_release); - failed.store(0, std::memory_order_release); - skipped.store(0, std::memory_order_release); - oom.store(0, std::memory_order_release); - } - - uint64_t num_passed() const{ - return passed.load(std::memory_order_acquire); - } - uint64_t num_failed() const{ - return failed.load(std::memory_order_acquire); - } - uint64_t num_skipped() const{ - return skipped.load(std::memory_order_acquire); - } - - // True if any test failed, including OOM. - bool any_failed() const{ - return num_failed() > 0; - } -}; - - class UnitTestRunner_Descriptor : public ComputerProgramDescriptor{ public: UnitTestRunner_Descriptor(); @@ -129,7 +75,7 @@ class UnitTestRunner TextEditOption FAILED_TESTS; ProgramEnvironment* m_env = nullptr; - UnitTestResultCounters m_results; + std::atomic m_total_failed; }; @@ -138,6 +84,9 @@ class CommandLineUnitTestRunner : public PokemonAutomation::UnitTestRunner::List public: CommandLineUnitTestRunner(Logger& logger) : m_logger(logger) + , m_skipped_tests(0) + , m_passed_tests(0) + , m_failed_tests(0) {} // Run the unit tests. Return true if any test failed. @@ -160,7 +109,9 @@ class CommandLineUnitTestRunner : public PokemonAutomation::UnitTestRunner::List private: Logger& m_logger; - UnitTestResultCounters m_results; + std::atomic m_skipped_tests; + std::atomic m_passed_tests; + std::atomic m_failed_tests; };