Repository navigation
Command-line test runner: filter tests by name substring #1895
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -107,6 +107,25 @@ const StringSelectDatabase& TEST_DATABASE(){ | |
| } | ||
|
|
||
|
|
||
| namespace{ | ||
|
|
||
| // For each test in `UNIT_TESTS_ALL()` whose name contains `filter`, in name order, call | ||
| // `add_test(test)`. An empty `filter` matches all the tests. Return the number of tests | ||
| // visited. | ||
| template <class AddTest> | ||
| size_t add_tests_matching(const std::string& filter, AddTest&& add_test){ | ||
| size_t count = 0; | ||
| for (const auto& test : UNIT_TESTS_ALL()){ | ||
| if (filter.empty() || test.first.contains(filter)){ | ||
| count++; | ||
| add_test(test.second); | ||
| } | ||
| } | ||
| return count; | ||
| } | ||
|
|
||
| } | ||
|
|
||
|
|
||
|
|
||
| UnitTestRunner_Descriptor::UnitTestRunner_Descriptor() | ||
|
|
@@ -237,15 +256,12 @@ void UnitTestRunner::on_config_value_changed(void* object){ | |
| if (object == &SUBSTRING_TEST){ | ||
| std::string name = SUBSTRING_TEST; | ||
| TESTS_TO_RUN.set(""); | ||
| size_t count = 0; | ||
| for (const auto& test : UNIT_TESTS_ALL()){ | ||
| if (test.first.contains(name)){ | ||
| count++; | ||
| if (count <= 1000){ | ||
| TESTS_TO_RUN.append(test.first + "\n"); | ||
| } | ||
| size_t num_shown = 0; | ||
| size_t count = add_tests_matching(name, [&](std::shared_ptr<const UnitTest> test){ | ||
| if (++num_shown <= 1000){ | ||
| TESTS_TO_RUN.append(test->name() + "\n"); | ||
| } | ||
| } | ||
| }); | ||
| if (count > 1000){ | ||
| TESTS_TO_RUN.append("<" + tostr_u_commas(count - 1000) + " more...>\n"); | ||
| } | ||
|
|
@@ -276,6 +292,12 @@ void UnitTestRunner::program(ProgramEnvironment& env, CancellableScope& scope){ | |
|
|
||
| const UnitTestDatabase& all_tests = UNIT_TESTS_ALL(); | ||
|
|
||
| auto add_matching = [&](const std::string& filter){ | ||
| add_tests_matching(filter, [&](std::shared_ptr<const UnitTest> test){ | ||
| runner.add_test(test); | ||
| }); | ||
| }; | ||
|
|
||
| switch ((RunMode)RUN_MODE){ | ||
| case RunMode::RUN_ONE:{ | ||
| auto iter = all_tests.find(SINGLE_TEST.slug()); | ||
|
|
@@ -289,22 +311,13 @@ void UnitTestRunner::program(ProgramEnvironment& env, CancellableScope& scope){ | |
| runner.add_test(iter->second); | ||
| break; | ||
| } | ||
| case RunMode::RUN_SUBSTRING_MATCH:{ | ||
| std::string name = SUBSTRING_TEST; | ||
| for (const auto& test : all_tests){ | ||
| if (test.first.contains(name)){ | ||
| runner.add_test(test.second); | ||
| } | ||
| } | ||
| case RunMode::RUN_SUBSTRING_MATCH: | ||
| add_matching(SUBSTRING_TEST); | ||
| break; | ||
| } | ||
| case RunMode::RUN_ALL:{ | ||
| for (const auto& test : all_tests){ | ||
| runner.add_test(test.second); | ||
| } | ||
| case RunMode::RUN_ALL: | ||
| add_matching(""); | ||
| break; | ||
| } | ||
| } | ||
|
|
||
| runner.run(); | ||
| } | ||
|
|
@@ -365,7 +378,7 @@ void UnitTestRunner::on_test_finished( | |
|
|
||
|
|
||
|
|
||
| bool CommandLineUnitTestRunner::run(bool augment_color_distortions){ | ||
| bool CommandLineUnitTestRunner::run(bool augment_color_distortions, const std::string& test_substring){ | ||
| if (augment_color_distortions){ | ||
| // Load the distortion data before any test runs. Each `ImageUnitTest` checks the global | ||
| // manager when it runs, and only runs with distortions if it is loaded. | ||
|
|
@@ -391,8 +404,15 @@ bool CommandLineUnitTestRunner::run(bool augment_color_distortions){ | |
| GlobalThreadPools::unlimited_normal() | ||
| ); | ||
| runner.add_listener(*this); | ||
| for (const auto& test : UNIT_TESTS_ALL()){ | ||
| runner.add_test(test.second); | ||
| size_t num_matching = add_tests_matching(test_substring, [&](std::shared_ptr<const UnitTest> test){ | ||
| runner.add_test(test); | ||
| }); | ||
| if (!test_substring.empty()){ | ||
| if (num_matching == 0){ | ||
| m_logger.log("No test name contains \"" + test_substring + "\". No tests will be run.", COLOR_ORANGE); | ||
|
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. AI review: FYI (documented, so not a blocker): a substring that matches nothing exits 0, so a typo'd filter in CI would produce a green run with zero tests - the orange warning is the only signal. Worth knowing if anyone wires this into a pipeline; if that becomes a problem, a distinct exit code for "zero tests ran due to filter" would be the natural fix. |
||
| }else{ | ||
| m_logger.log("Running the " + tostr_u_commas(num_matching) + " tests whose name contains \"" + test_substring + "\""); | ||
|
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. AI review: Nit: when exactly one test matches this reads "Running the 1 tests whose name contains ...". No need to block on it, but if you're touching this line anyway a "test(s)" or singular/plural branch would read better. |
||
| } | ||
| } | ||
| runner.run(); | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
AI review: FYI: this missing-value error fires even when
--command-line-test-modeisn't passed, i.e. a stray--test-substringon a GUI launch also exits 1. That strict behavior seems right for a malformed flag, just noting the flag is parsed outside of test mode.