From bd9752818776625fe17af668c8ef124e77a68369 Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Mon, 7 Sep 2026 12:19:34 +0000 Subject: [PATCH 1/3] wpilib.testing: Extract shared pytest plugin test helpers Move robot module generation and plugin configuration into a reusable helper module for the existing suite and upcoming ordering tests. --- .../tests/pytest_plugin_test_helpers.py | 92 ++++++++++++++++++ .../tests/test_pytest_plugins.py | 97 +------------------ 2 files changed, 97 insertions(+), 92 deletions(-) create mode 100644 subprojects/robotpy-wpilib/tests/pytest_plugin_test_helpers.py diff --git a/subprojects/robotpy-wpilib/tests/pytest_plugin_test_helpers.py b/subprojects/robotpy-wpilib/tests/pytest_plugin_test_helpers.py new file mode 100644 index 000000000..0f6558f4d --- /dev/null +++ b/subprojects/robotpy-wpilib/tests/pytest_plugin_test_helpers.py @@ -0,0 +1,92 @@ +import pathlib + + +def _make_robot_module(pytester): + pytester.makepyfile(robot_module=""" +import wpilib + + +class DummyRobot(wpilib.TimedRobot): + def __init__(self): + super().__init__() + self.did_init = True + + +class AutonomousPeriodicFailed(wpilib.TimedRobot): + def autonomous_periodic(self): + assert False + + +class TeleopPeriodicFailed(wpilib.TimedRobot): + def teleop_periodic(self): + assert False + + +class TeleopInitFailed(wpilib.TimedRobot): + def teleop_init(self): + assert False + + +class IterativeStateRobot(wpilib.TimedRobot): + def disabled_init(self): + self.did_disabled_init = True + + def disabled_periodic(self): + self.did_disabled_periodic = True + + def autonomous_init(self): + self.did_auto_init = True + + def autonomous_periodic(self): + self.did_auto_periodic = True + + def teleop_init(self): + self.did_teleop_init = True + + def teleop_periodic(self): + self.did_teleop_periodic = True +""") + + +def _configure_robot_testing_plugin(pytester, robot_class="DummyRobot"): + pytester.makeconftest(f""" +import pathlib + +from wpilib.testing.pytest_plugin import RobotTestingPlugin + +from robot_module import {robot_class} + + +def pytest_configure(config): + robot_file = pathlib.Path(__file__).resolve() + config.pluginmanager.register(RobotTestingPlugin({robot_class}, robot_file, False)) +""") + + +def _configure_isolated_plugin( + pytester, + parallelism=1, + robot_class="DummyRobot", + robot_module="robot_module", + robot_file_name=None, +): + if robot_file_name is None: + robot_file = "pathlib.Path(__file__).resolve()" + else: + robot_file = f"pathlib.Path(__file__).parent / {robot_file_name!r}" + + pytester.makeconftest(f""" +import pathlib + +from wpilib.testing.pytest_isolated_tests_plugin import IsolatedTestsPlugin + +from {robot_module} import {robot_class} + +def pytest_configure(config): + if "--no-header" in config.invocation_params.args: + return + robot_file = {robot_file} + config.pluginmanager.register( + IsolatedTestsPlugin({robot_class}, robot_file, False, False, {parallelism}) + ) +""") diff --git a/subprojects/robotpy-wpilib/tests/test_pytest_plugins.py b/subprojects/robotpy-wpilib/tests/test_pytest_plugins.py index 40326e880..a12bb4e0a 100644 --- a/subprojects/robotpy-wpilib/tests/test_pytest_plugins.py +++ b/subprojects/robotpy-wpilib/tests/test_pytest_plugins.py @@ -3,98 +3,11 @@ import pytest - -def _make_robot_module(pytester): - pytester.makepyfile(robot_module=""" -import wpilib - - -class DummyRobot(wpilib.TimedRobot): - def __init__(self): - super().__init__() - self.did_init = True - - -class AutonomousPeriodicFailed(wpilib.TimedRobot): - def autonomous_periodic(self): - assert False - - -class TeleopPeriodicFailed(wpilib.TimedRobot): - def teleop_periodic(self): - assert False - - -class TeleopInitFailed(wpilib.TimedRobot): - def teleop_init(self): - assert False - - -class IterativeStateRobot(wpilib.TimedRobot): - - def disabled_init(self): - self.did_disabled_init = True - - def disabled_periodic(self): - self.did_disabled_periodic = True - - def autonomous_init(self): - self.did_auto_init = True - - def autonomous_periodic(self): - self.did_auto_periodic = True - - def teleop_init(self): - self.did_teleop_init = True - - def teleop_periodic(self): - self.did_teleop_periodic = True - -""") - - -def _configure_robot_testing_plugin(pytester, robot_class="DummyRobot"): - pytester.makeconftest(f""" -import pathlib - -from wpilib.testing.pytest_plugin import RobotTestingPlugin - -from robot_module import {robot_class} - - -def pytest_configure(config): - robot_file = pathlib.Path(__file__).resolve() - config.pluginmanager.register(RobotTestingPlugin({robot_class}, robot_file, False)) -""") - - -def _configure_isolated_plugin( - pytester, - parallelism=1, - robot_class="DummyRobot", - robot_module="robot_module", - robot_file_name=None, -): - if robot_file_name is None: - robot_file = "pathlib.Path(__file__).resolve()" - else: - robot_file = f"pathlib.Path(__file__).parent / {robot_file_name!r}" - - pytester.makeconftest(f""" -import pathlib - -from wpilib.testing.pytest_isolated_tests_plugin import IsolatedTestsPlugin - -from {robot_module} import {robot_class} - -def pytest_configure(config): - if "--no-header" in config.invocation_params.args: - return - robot_file = {robot_file} - config.pluginmanager.register( - IsolatedTestsPlugin({robot_class}, robot_file, False, False, {parallelism}) - ) -""") +from pytest_plugin_test_helpers import ( + _configure_isolated_plugin, + _configure_robot_testing_plugin, + _make_robot_module, +) def test_robot_testing_plugin_success(pytester): From bb7d8a9708b6a101701216e2391c15fd6cc86525 Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Mon, 7 Sep 2026 12:19:34 +0000 Subject: [PATCH 2/3] wpilib.testing: Propagate isolated worker exits and honor fail-fast Report abnormal worker exits even when no test report was produced, and propagate worker interruptions to the parent session. Prefer the failed session status when the failure limit has been reached. Poll completed workers before starting deferred in-process tests so -x does not allow more tests to run after an isolated failure. Add regression coverage for empty worker collection and fail-fast with deferred tests. --- .../tests/test_pytest_plugins.py | 49 +++++++++++++++++++ .../testing/pytest_isolated_tests_plugin.py | 21 ++++++-- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/subprojects/robotpy-wpilib/tests/test_pytest_plugins.py b/subprojects/robotpy-wpilib/tests/test_pytest_plugins.py index a12bb4e0a..48b6676f8 100644 --- a/subprojects/robotpy-wpilib/tests/test_pytest_plugins.py +++ b/subprojects/robotpy-wpilib/tests/test_pytest_plugins.py @@ -224,6 +224,28 @@ def test_robot_two(robot): ) +def test_isolated_plugin_reports_worker_collection_exit(pytester): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + with pytester.path.joinpath("conftest.py").open("a") as fp: + fp.write(""" + + +def pytest_collection_modifyitems(config, items): + if "--no-header" in config.invocation_params.args: + items.clear() +""") + pytester.makepyfile(test_isolated=""" +def test_robot(robot): + assert robot is not None +""") + + result = pytester.runpytest_subprocess("-v") + + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines(["*subprocess exited with exit code 5*"]) + + def test_isolated_plugin_maxfail_stops_early(pytester): _make_robot_module(pytester) _configure_isolated_plugin(pytester) @@ -242,6 +264,33 @@ def test_robot_second(robot): assert not any("test_robot_second" in line for line in result.outlines) +def test_isolated_plugin_maxfail_stops_deferred_tests(pytester): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester, parallelism=2) + pytester.makepyfile(test_isolated=""" +import pathlib +import time + + +def test_robot_failure(robot): + assert False + + +def test_plain_waits_for_failure(): + time.sleep(1) + + +def test_plain_must_not_run(): + pathlib.Path("plain-ran").touch() +""") + + result = pytester.runpytest_subprocess("-v", "-x") + + assert result.parseoutcomes()["failed"] == 1 + assert result.ret == pytest.ExitCode.TESTS_FAILED + assert not (pytester.path / "plain-ran").exists() + + @pytest.mark.parametrize("isolated", [False, True]) def test_builtin_tests_module(pytester, isolated): _make_robot_module(pytester) diff --git a/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_tests_plugin.py b/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_tests_plugin.py index 27c2b6440..df5d905be 100644 --- a/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_tests_plugin.py +++ b/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_tests_plugin.py @@ -258,6 +258,11 @@ def pytest_runtestloop(self, session: pytest.Session) -> bool: # Run the in-process tests now while the robot tests are finishing for idx, item in enumerate(deferred): + # Observe completed isolated failures before starting more + # in-process work, so --maxfail can stop the group promptly. + if any(job.conn.poll() for job in running): + self._wait_for_jobs(running, session) + nextitem = deferred[idx + 1] if idx + 1 < len(deferred) else None session.config.hook.pytest_runtest_protocol( item=item, nextitem=nextitem @@ -402,10 +407,10 @@ def _cleanup_job(self, job: IsolatedTestJob): job.process.close() def _maybe_raise(self, session: pytest.Session): - if self._shouldstop: - raise session.Interrupted(self._shouldstop) if session.shouldfail: raise session.Failed(session.shouldfail) + if self._shouldstop: + raise session.Interrupted(self._shouldstop) if session.shouldstop: raise session.Interrupted(session.shouldstop) @@ -458,7 +463,17 @@ def worker_finished(self, job: IsolatedTestJob, exit_code: object | None = None) if exit_code is not None: job.exit_code = int(exit_code) - job.worker_completed = True + if job.exit_code == pytest.ExitCode.INTERRUPTED and not self._shouldstop: + self._shouldstop = "interrupted in worker" + + # Normal test failures have already produced reports, and interruptions + # stop the parent session. Other exit codes need a synthetic failure + # report from _finalize_job. + job.worker_completed = job.exit_code in ( + pytest.ExitCode.OK, + pytest.ExitCode.TESTS_FAILED, + pytest.ExitCode.INTERRUPTED, + ) job.finished = True def _handlefailures(self, rep: pytest.TestReport): From a65eb270d71adfbd06743fb3a38f51eae9e82213 Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Mon, 7 Sep 2026 12:19:34 +0000 Subject: [PATCH 3/3] wpilib.testing: Respect pytest-order in isolated test scheduling Consume the final collection order and drain workers at ordering-marker, dependency, and structural group boundaries. Retain parallel execution within groups and preserve fixture scopes across in-process tests. Keep pytest-order available in workers for markers, options, and test generation, but disable its sorter and carry failed-ordering fixtures from the parent. Require the optional plugin only when ordering is used. Add pytest-order to test requirements and cover marker forms, inheritance, scope and dependency options, worker compatibility, interruptions, and unordered parallel execution. --- .../robotpy-wpilib/tests/requirements.txt | 1 + .../test_pytest_isolated_order_adapter.py | 1129 +++++++++++++++++ .../tests/test_pytest_plugins.py | 54 + .../testing/pytest_isolated_order_adapter.py | 145 +++ .../testing/pytest_isolated_tests_plugin.py | 113 +- 5 files changed, 1407 insertions(+), 35 deletions(-) create mode 100644 subprojects/robotpy-wpilib/tests/test_pytest_isolated_order_adapter.py create mode 100644 subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_order_adapter.py diff --git a/subprojects/robotpy-wpilib/tests/requirements.txt b/subprojects/robotpy-wpilib/tests/requirements.txt index e079f8a60..114768e8b 100644 --- a/subprojects/robotpy-wpilib/tests/requirements.txt +++ b/subprojects/robotpy-wpilib/tests/requirements.txt @@ -1 +1,2 @@ pytest +pytest-order diff --git a/subprojects/robotpy-wpilib/tests/test_pytest_isolated_order_adapter.py b/subprojects/robotpy-wpilib/tests/test_pytest_isolated_order_adapter.py new file mode 100644 index 000000000..c8a488ed9 --- /dev/null +++ b/subprojects/robotpy-wpilib/tests/test_pytest_isolated_order_adapter.py @@ -0,0 +1,1129 @@ +import pathlib +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from pytest_plugin_test_helpers import ( + _configure_isolated_plugin, + _make_robot_module, +) +from wpilib.testing.pytest_isolated_order_adapter import PytestOrderAdapter + + +def _configure_recording_isolated_plugin(pytester): + pytester.makeconftest(""" +import pathlib + +from wpilib.testing.pytest_isolated_tests_plugin import IsolatedTestsPlugin + +from robot_module import DummyRobot + + +class RecordingIsolatedTestsPlugin(IsolatedTestsPlugin): + def _start_isolated_test(self, item): + pathlib.Path("isolated-started").touch() + return super()._start_isolated_test(item) + + +def pytest_configure(config): + if "--no-header" in config.invocation_params.args: + return + config.pluginmanager.register( + RecordingIsolatedTestsPlugin( + DummyRobot, pathlib.Path(__file__).resolve(), False, False, 2 + ) + ) +""") + + +class _AdapterConfig: + def __init__(self, **options): + self._options = options + self.pluginmanager = SimpleNamespace(get_plugin=lambda name: object()) + + def getoption(self, name, default=None): + return self._options.get(name, default) + + +@pytest.mark.parametrize( + "order_scope,group_scope,want", + [ + (None, None, None), + ("session", "module", "module"), + ("session", "class", "class"), + ("module", "class", "class"), + ("module", "session", None), + ("class", "module", None), + ("invalid", "module", "module"), + ], +) +def test_effective_group_scope(order_scope, group_scope, want): + config = _AdapterConfig( + order_scope=order_scope, + order_group_scope=group_scope, + ) + assert PytestOrderAdapter(config)._group_scope == want + + +@pytest.mark.parametrize( + "first_mark,second_mark", + [ + ("@pytest.mark.order(1)", "@pytest.mark.order(2)"), + ("@pytest.mark.order(-2)", "@pytest.mark.order(-1)"), + ("@pytest.mark.order('first')", "@pytest.mark.order('second')"), + ("@pytest.mark.order(index=1)", "@pytest.mark.order(index=2)"), + ("@pytest.mark.order(before='test_second')", ""), + ("", "@pytest.mark.order(after='test_first')"), + ], + ids=["numeric", "negative", "named", "index", "before", "after"], +) +def test_documented_marker_forms_preserve_boundaries(pytester, first_mark, second_mark): + _make_robot_module(pytester) + _configure_recording_isolated_plugin(pytester) + pytester.makepyfile(test_marker_forms=f""" +import pathlib +import pytest + +{second_mark} +def test_second(robot): + assert pathlib.Path("first-finished").exists() + +{first_mark} +def test_first(): + assert not pathlib.Path("isolated-started").exists() + pathlib.Path("first-finished").touch() +""") + + result = pytester.runpytest_subprocess("-vv") + + result.assert_outcomes(passed=2) + + +@pytest.mark.parametrize("container", ["list", "tuple"]) +@pytest.mark.parametrize("direction", ["before", "after"]) +def test_relative_marker_sequences(pytester, container, direction): + _make_robot_module(pytester) + _configure_recording_isolated_plugin(pytester) + if direction == "after": + refs = ( + '["test_first", "test_middle"]' + if container == "list" + else '("test_first", "test_middle")' + ) + first_mark = "" + middle_mark = '@pytest.mark.order(after="test_first")' + last_mark = f"@pytest.mark.order(after={refs})" + else: + refs = ( + '["test_middle", "test_last"]' + if container == "list" + else '("test_middle", "test_last")' + ) + first_mark = f"@pytest.mark.order(before={refs})" + middle_mark = '@pytest.mark.order(before="test_last")' + last_mark = "" + + pytester.makepyfile(test_relative_sequences=f""" +import pathlib +import pytest + +{last_mark} +def test_last(robot): + assert pathlib.Path("first-finished").exists() + assert pathlib.Path("middle-finished").exists() + +{middle_mark} +def test_middle(): + assert pathlib.Path("first-finished").exists() + pathlib.Path("middle-finished").touch() + +{first_mark} +def test_first(): + assert not pathlib.Path("isolated-started").exists() + pathlib.Path("first-finished").touch() +""") + + result = pytester.runpytest_subprocess("-vv") + + result.assert_outcomes(passed=3) + + +def test_isolated_plugin_worker_interruption_stops_session(pytester): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makepyfile(test_isolated=""" +import pathlib +import pytest + + +@pytest.mark.order(1) +def test_robot_exit(robot): + pytest.exit("worker requested exit") + + +@pytest.mark.order(2) +def test_later_group_must_not_run(): + pathlib.Path("later-group-ran").touch() +""") + + result = pytester.runpytest_subprocess("-v") + + assert result.ret == pytest.ExitCode.INTERRUPTED + assert not (pytester.path / "later-group-ran").exists() + + +def test_plain_order_groups_preserve_shared_fixture_scopes(pytester): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makepyfile( + test_plain=""" +import pathlib + +import pytest + + +EVENTS = pathlib.Path("fixture-events.txt") + + +def record(event): + with EVENTS.open("a") as fp: + fp.write(event + "\\n") + + +@pytest.fixture(scope="session") +def session_fixture(): + record("session setup") + yield + record("session teardown") + + +@pytest.fixture(scope="module") +def module_fixture(): + record("module setup") + yield + record("module teardown") + + +@pytest.fixture(scope="class") +def class_fixture(): + record("class setup") + yield + record("class teardown") + + +class TestPlain: + @pytest.mark.order(1) + def test_first(self, session_fixture, module_fixture, class_fixture): + record("first") + + @pytest.mark.order(3) + def test_second(self, session_fixture, module_fixture, class_fixture): + record("second") +""", + test_robot=""" +import pytest + + +@pytest.mark.order(2) +def test_between_plain_groups(robot): + pass +""", + ) + + result = pytester.runpytest_subprocess("-q") + + result.assert_outcomes(passed=3) + assert (pytester.path / "fixture-events.txt").read_text().splitlines() == [ + "session setup", + "module setup", + "class setup", + "first", + "second", + "class teardown", + "module teardown", + "session teardown", + ] + + +@pytest.mark.parametrize("scope", ["module", "class"]) +def test_order_group_scope_preserves_structural_boundaries(pytester, scope): + _make_robot_module(pytester) + _configure_recording_isolated_plugin(pytester) + + if scope == "module": + pytester.makepyfile( + test_early=""" +import pathlib +import pytest + +@pytest.mark.order(1) +def test_order_anchor(): pass + +def test_plain_tail(): + assert not pathlib.Path("isolated-started").exists() +""", + test_late=""" +import pytest + +def test_robot_head(robot): pass + +@pytest.mark.order(-1) +def test_order_anchor(): pass +""", + ) + else: + pytester.makepyfile(test_groups=""" +import pathlib +import pytest + +class TestEarly: + @pytest.mark.order(1) + def test_order_anchor(self): pass + + def test_plain_tail(self): + assert not pathlib.Path("isolated-started").exists() + +class TestLate: + def test_robot_head(self, robot): pass + + @pytest.mark.order(-1) + def test_order_anchor(self): pass +""") + + result = pytester.runpytest_subprocess("-vv", f"--order-group-scope={scope}") + + result.assert_outcomes(passed=4) + + +@pytest.mark.parametrize( + "level,separate_scopes", + [(0, False), (-1, False), (1, False), (2, True), (5, True)], +) +def test_directory_order_scope_preserves_scheduler_boundaries( + pytester, level, separate_scopes +): + _make_robot_module(pytester) + _configure_recording_isolated_plugin(pytester) + early = pytester.path / "tests" / "early" + late = pytester.path / "tests" / "late" + early.mkdir(parents=True) + late.mkdir(parents=True) + (early / "test_plain.py").write_text(f""" +import pathlib + + +def test_plain_tail(): + assert pathlib.Path("isolated-started").exists() is {not separate_scopes} +""") + (late / "test_robot.py").write_text(""" +def test_robot_head(robot): + assert robot is not None +""") + + result = pytester.runpytest_subprocess("-q", f"--order-scope-level={level}") + + result.assert_outcomes(passed=2) + + +def test_class_order_group_scope_preserves_nested_class_boundaries(pytester): + _make_robot_module(pytester) + _configure_recording_isolated_plugin(pytester) + pytester.makepyfile(test_nested_groups=""" +import pathlib +import pytest + +class TestOuter: + class TestEarly: + @pytest.mark.order(1) + def test_order_anchor(self): pass + + def test_plain_tail(self): + assert not pathlib.Path("isolated-started").exists() + + class TestLate: + def test_robot_head(self, robot): pass + + @pytest.mark.order(-1) + def test_order_anchor(self): pass +""") + + result = pytester.runpytest_subprocess("-vv", "--order-group-scope=class") + + result.assert_outcomes(passed=4) + + +def test_module_order_marker_forms_one_parallel_group(pytester): + items = pytester.getitems(""" +import pytest + +pytestmark = pytest.mark.order(1) + + +def test_one(): pass + +def test_two(): pass +""") + adapter = PytestOrderAdapter(_AdapterConfig()) + + groups = list(adapter.groups(items)) + + assert [[item.name for item in group] for group in groups] == [ + ["test_one", "test_two"] + ] + assert items[0].get_closest_marker("order") is items[1].get_closest_marker("order") + + +def test_order_marker_groups_use_real_pytest_marker_inheritance(pytester): + items = pytester.getitems(""" +import pytest + + +@pytest.mark.order(1) +class TestInherited: + def test_inherited_one(self): + pass + + def test_inherited_two(self): + pass + + +@pytest.mark.order(1) +def test_independent_one(): + pass + + +@pytest.mark.order(1) +def test_independent_two(): + pass + + +@pytest.mark.order(1) +class TestOverride: + def test_class_marker(self): + pass + + @pytest.mark.order(2) + def test_function_override(self): + pass +""") + items_by_name = {item.name: item for item in items} + ordered_items = [ + items_by_name[name] + for name in ( + "test_inherited_one", + "test_inherited_two", + "test_independent_one", + "test_independent_two", + "test_class_marker", + "test_function_override", + ) + ] + + config = SimpleNamespace( + getoption=lambda name, default=None: default, + pluginmanager=SimpleNamespace(get_plugin=lambda name: object()), + ) + adapter = PytestOrderAdapter(config) + groups = adapter.groups(ordered_items) + + assert [[item.name for item in group] for group in groups] == [ + ["test_inherited_one", "test_inherited_two"], + ["test_independent_one"], + ["test_independent_two"], + ["test_class_marker"], + ["test_function_override"], + ] + assert items_by_name["test_class_marker"].get_closest_marker("order").args == (1,) + assert items_by_name["test_function_override"].get_closest_marker("order").args == ( + 2, + ) + + +class _SchedulerHook: + def __init__(self, events): + self.events = events + + def pytest_runtest_protocol(self, item, nextitem): + self.events.append(("plain", item.name)) + + +class _SchedulerSession: + Interrupted = pytest.Session.Interrupted + Failed = pytest.Session.Failed + + def __init__(self, items, events): + self.items = items + self.testsfailed = 0 + self.shouldfail = False + self.shouldstop = False + self.config = SimpleNamespace( + option=SimpleNamespace( + continue_on_collection_errors=False, + collectonly=False, + ), + hook=_SchedulerHook(events), + getoption=lambda name, default=None: default, + ) + + +def _scheduler_item(name, marker, uses_robot): + item = MagicMock(spec=pytest.Function) + item.name = name + item.fixturenames = ["robot"] if uses_robot else [] + item.get_closest_marker.return_value = marker + return item + + +@pytest.mark.parametrize( + "order_scope,group_scope,expected", + [ + (None, None, [["first", "second"], ["third"]]), + ("session", "module", [["first"], ["second"], ["third"]]), + ("session", "class", [["first"], ["second"], ["third"]]), + ("module", None, [["first", "second", "third"]]), + ("class", None, [["first", "second", "third"]]), + ], +) +def test_directory_scope_combines_with_group_scope(order_scope, group_scope, expected): + items = [ + _scheduler_item(name, None, False) for name in ("first", "second", "third") + ] + for item, nodeid in zip( + items, + ("early/test_a.py::first", "early/test_b.py::second", "late/test_c.py::third"), + ): + item.nodeid = nodeid + item.getparent.return_value = None + adapter = PytestOrderAdapter( + _AdapterConfig( + order_scope_level=1, + order_scope=order_scope, + order_group_scope=group_scope, + ) + ) + + assert [ + [item.name for item in group] for group in adapter.groups(items) + ] == expected + + +def _scheduler_events(monkeypatch, items): + from wpilib.testing.pytest_isolated_order_adapter import PytestOrderAdapter + from wpilib.testing.pytest_isolated_tests_plugin import IsolatedTestsPlugin + + events = [] + session = _SchedulerSession(items, events) + plugin = IsolatedTestsPlugin(object, pathlib.Path("robot.py"), False, False, 4) + plugin._ordering = PytestOrderAdapter(session.config) + + def start(item): + events.append(("start", item.name)) + conn = SimpleNamespace(poll=lambda: False) + return SimpleNamespace(item=item, conn=conn) + + def wait(running, session): + events.extend(("finish", job.item.name) for job in running) + running.clear() + + monkeypatch.setattr(plugin, "_start_isolated_test", start) + monkeypatch.setattr(plugin, "_wait_for_jobs", wait) + + assert plugin.pytest_runtestloop(session) is True + return events + + +_ORDERED_TYPE_CHAINS = [ + ("ROBOT", "ROBOT", None), + ("ROBOT", "PLAIN", None), + ("PLAIN", "ROBOT", None), + ("PLAIN", "PLAIN", None), + ("ROBOT", "ROBOT", "PLAIN"), + ("ROBOT", "PLAIN", "ROBOT"), + ("PLAIN", "ROBOT", "ROBOT"), + ("ROBOT", "PLAIN", "PLAIN"), + ("PLAIN", "ROBOT", "PLAIN"), + ("PLAIN", "PLAIN", "ROBOT"), +] + + +@pytest.mark.parametrize("marker_style", ["numeric", "after", "before"]) +@pytest.mark.parametrize("test_types", _ORDERED_TYPE_CHAINS) +def test_order_groups_do_not_cross_scheduler_boundaries( + monkeypatch, marker_style, test_types +): + # pytest-order has already sorted these items. The three patterns model + # which tests carry markers for ordinal, after=, and before= ordering. + first_marker = object() + middle_marker = object() + last_marker = object() + markers = { + "numeric": (first_marker, middle_marker, last_marker), + "after": (None, middle_marker, last_marker), + "before": (first_marker, middle_marker, None), + }[marker_style] + + items = [] + expected = [] + for name, test_type, marker in zip( + ("first", "middle", "last"), test_types, markers + ): + if test_type is None: + continue + + uses_robot = test_type == "ROBOT" + items.append(_scheduler_item(name, marker, uses_robot)) + if uses_robot: + expected.extend((("start", name), ("finish", name))) + else: + expected.append(("plain", name)) + + assert _scheduler_events(monkeypatch, items) == expected + + +def test_order_marker_implies_dependency_boundaries(pytester): + _make_robot_module(pytester) + pytester.makeini( + "[pytest]\nmarkers = dependency(*args, **kwargs): test dependency\n" + ) + _configure_recording_isolated_plugin(pytester) + pytester.makepyfile(test_dependency_order=""" +import pathlib + +import pytest + + +@pytest.mark.order(1) +class TestChain: + @pytest.mark.dependency(depends=["prerequisite"]) + def test_isolated_dependent(self, robot): + assert pathlib.Path("prerequisite-finished").exists() + + @pytest.mark.dependency(name="prerequisite") + def test_plain_prerequisite(self): + assert not pathlib.Path("isolated-started").exists() + pathlib.Path("prerequisite-finished").touch() +""") + + result = pytester.runpytest_subprocess("-vv") + + result.assert_outcomes(passed=2) + + +def test_prefixed_order_marker_implies_dependency_boundaries(pytester): + _make_robot_module(pytester) + pytester.makeini( + "[pytest]\n" + "markers =\n" + " sequence1: first test\n" + " dependency(*args, **kwargs): test dependency\n" + ) + _configure_recording_isolated_plugin(pytester) + pytester.makepyfile(test_dependency_order=""" +import pathlib + +import pytest + + +@pytest.mark.sequence1 +class TestChain: + @pytest.mark.dependency(name="prerequisite") + def test_plain_prerequisite(self): + assert not pathlib.Path("isolated-started").exists() + pathlib.Path("prerequisite-finished").touch() + + @pytest.mark.dependency(depends=["prerequisite"]) + def test_isolated_dependent(self, robot): + assert pathlib.Path("prerequisite-finished").exists() +""") + + result = pytester.runpytest_subprocess("-vv", "--order-marker-prefix=sequence") + + result.assert_outcomes(passed=2) + + +def test_dependency_ordering_does_not_cross_scheduler_boundaries(pytester): + _make_robot_module(pytester) + pytester.makeini( + "[pytest]\nmarkers = dependency(*args, **kwargs): test dependency\n" + ) + pytester.makeconftest(""" +import pathlib + +from wpilib.testing.pytest_isolated_tests_plugin import IsolatedTestsPlugin + +from robot_module import DummyRobot + + +class RecordingIsolatedTestsPlugin(IsolatedTestsPlugin): + def _start_isolated_test(self, item): + pathlib.Path("isolated-started").touch() + return super()._start_isolated_test(item) + + +def pytest_configure(config): + if "--no-header" in config.invocation_params.args: + return + config.pluginmanager.register( + RecordingIsolatedTestsPlugin( + DummyRobot, pathlib.Path(__file__).resolve(), False, False, 2 + ) + ) +""") + pytester.makepyfile(test_dependency_order=""" +import pathlib + +import pytest + + +@pytest.mark.dependency(depends=["prerequisite"]) +def test_isolated_dependent(robot): + assert pathlib.Path("prerequisite-finished").exists() + + +@pytest.mark.dependency(name="prerequisite") +def test_plain_prerequisite(): + assert not pathlib.Path("isolated-started").exists() + pathlib.Path("prerequisite-finished").touch() +""") + + result = pytester.runpytest_subprocess("-vv", "--order-dependencies") + + result.assert_outcomes(passed=2) + + +@pytest.mark.parametrize( + "first_marker,second_marker", + [ + ("sequence1", "sequence2"), + ("sequence1_setup", "sequence2_robot"), + ], + ids=["documented", "pytest_order_1_5_compatibility"], +) +def test_prefixed_order_markers_preserve_scheduler_boundaries( + pytester, first_marker, second_marker +): + _make_robot_module(pytester) + pytester.makeini( + "[pytest]\n" + "markers =\n" + f" {first_marker}: first test\n" + f" {second_marker}: second test\n" + ) + pytester.makeconftest(""" +import pathlib + +from wpilib.testing.pytest_isolated_tests_plugin import IsolatedTestsPlugin + +from robot_module import DummyRobot + + +class RecordingIsolatedTestsPlugin(IsolatedTestsPlugin): + def _start_isolated_test(self, item): + pathlib.Path("isolated-started").touch() + return super()._start_isolated_test(item) + + +def pytest_configure(config): + if "--no-header" in config.invocation_params.args: + return + config.pluginmanager.register( + RecordingIsolatedTestsPlugin( + DummyRobot, pathlib.Path(__file__).resolve(), False, False, 2 + ) + ) +""") + pytester.makepyfile(test_prefixed_order=f""" +import pathlib + +import pytest + + +@pytest.mark.{second_marker} +def test_isolated_dependent(robot): + assert pathlib.Path("prerequisite-finished").exists() + + +@pytest.mark.{first_marker} +def test_plain_prerequisite(): + assert not pathlib.Path("isolated-started").exists() + pathlib.Path("prerequisite-finished").touch() +""") + + result = pytester.runpytest_subprocess("-vv", "--order-marker-prefix=sequence") + + result.assert_outcomes(passed=2) + + +def test_inherited_order_marker_group_runs_in_parallel(monkeypatch): + inherited_marker = object() + next_marker = object() + items = [ + _scheduler_item("plain", inherited_marker, False), + _scheduler_item("robot_one", inherited_marker, True), + _scheduler_item("robot_two", inherited_marker, True), + _scheduler_item("next_robot", next_marker, True), + ] + + assert _scheduler_events(monkeypatch, items) == [ + ("start", "robot_one"), + ("start", "robot_two"), + ("plain", "plain"), + ("finish", "robot_one"), + ("finish", "robot_two"), + ("start", "next_robot"), + ("finish", "next_robot"), + ] + + +class _EqualMarker: + def __eq__(self, other): + return isinstance(other, _EqualMarker) + + +def test_equal_but_distinct_order_markers_form_boundaries(monkeypatch): + first_marker = _EqualMarker() + second_marker = _EqualMarker() + items = [ + _scheduler_item("first", first_marker, True), + _scheduler_item("second", second_marker, True), + ] + + assert _scheduler_events(monkeypatch, items) == [ + ("start", "first"), + ("finish", "first"), + ("start", "second"), + ("finish", "second"), + ] + + +@pytest.mark.parametrize( + "source,args", + [ + ( + "@pytest.mark.sequence1\ndef test_robot(robot): pass", + ("--order-marker-prefix=sequence",), + ), + ( + "@pytest.mark.dependency(name='x')\ndef test_robot(robot): pass", + ("--order-dependencies",), + ), + ], + ids=["prefix", "dependency"], +) +def test_nonstandard_ordering_requires_active_sorter(pytester, source, args): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makeini( + "[pytest]\nmarkers =\n" + " sequence1: first\n" + " dependency(*args, **kwargs): dependency\n" + ) + pytester.makepyfile(test_ordered="import pytest\n\n" + source) + + result = pytester.runpytest_subprocess("-p", "no:orderingplugin", *args) + + assert result.ret == pytest.ExitCode.USAGE_ERROR + + +@pytest.mark.parametrize( + "disabled_plugin", ["pytest_order", "orderingplugin"], ids=["absent", "disabled"] +) +def test_order_marker_requires_pytest_order(pytester, disabled_plugin): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makepyfile(test_ordered=""" +import pytest + + +@pytest.mark.order(2) +def test_second(robot): + pass + + +@pytest.mark.order(1) +def test_first(robot): + pass +""") + + result = pytester.runpytest_subprocess("-p", f"no:{disabled_plugin}") + + assert result.ret == pytest.ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines(["*pytest-order is required to use order markers*"]) + + +def test_doctest_runs_in_process_with_isolated_tests(pytester): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + with (pytester.path / "conftest.py").open("a") as fp: + fp.write("\nimport os\nos.environ['ISOLATED_PARENT_PID'] = str(os.getpid())\n") + pytester.makepyfile(test_mixed=''' +""" +>>> import os +>>> assert os.getpid() == int(os.environ["ISOLATED_PARENT_PID"]) +""" + + +def test_robot(robot): + assert robot is not None + + +def test_plain(): + pass +''') + + result = pytester.runpytest_subprocess("--doctest-modules", "-q") + + result.assert_outcomes(passed=3) + + +def test_custom_item_without_fixtures_runs_in_process(monkeypatch, pytester): + class CustomItem(pytest.Item): + def runtest(self): + pass + + session = pytester.getitem("def test_func(): pass").session + custom = CustomItem.from_parent(session, name="custom") + items = [custom, _scheduler_item("robot", None, True)] + + assert _scheduler_events(monkeypatch, items) == [ + ("start", "robot"), + ("plain", "custom"), + ("finish", "robot"), + ] + + +def test_pytest_order_is_optional_without_order_markers(pytester): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makepyfile(test_unordered=""" +def test_robot(robot): + pass +""") + + result = pytester.runpytest_subprocess("-p", "no:pytest_order") + + result.assert_outcomes(passed=1) + + +@pytest.mark.parametrize( + "ini_body, first_mark, second_mark", + [ + ( + "addopts = --strict-markers", + "@pytest.mark.order(1)", + "@pytest.mark.order(2)", + ), + ( + "filterwarnings = error", + "@pytest.mark.order(1)", + "@pytest.mark.order(2)", + ), + ( + "filterwarnings = error", + "", + '@pytest.mark.order(after="test_first")', + ), + ], + ids=["strict_markers", "warnings_as_errors", "relative_marker_stays_quiet"], +) +def test_order_marker_is_registered_in_isolated_subprocess( + pytester, ini_body, first_mark, second_mark +): + """ + Ordered robot tests must still run when the project treats unknown markers + as a hard failure. The worker keeps pytest-order loaded for option parsing + and test generation, but unregisters its collection-ordering plugin because + a one-test worker cannot resolve relative markers. The marker must remain + registered without emitting misleading relative-order warnings. + """ + _make_robot_module(pytester) + _configure_isolated_plugin(pytester, parallelism=2) + pytester.makeini(f"[pytest]\n{ini_body}\n") + + pytester.makepyfile(test_strict_order=f"""\ +import pathlib + +import pytest + + +{second_mark} +def test_second(robot): + assert pathlib.Path("strict_sentinel.txt").exists() + + +{first_mark} +def test_first(robot): + pathlib.Path("strict_sentinel.txt").write_text("done") +""") + + result = pytester.runpytest_subprocess("-vv") + result.assert_outcomes(passed=2, errors=0) + assert not any("cannot execute" in line for line in result.outlines) + + +def test_error_on_failed_ordering_reaches_isolated_worker(pytester): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makepyfile(test_failed_order=""" +import pytest + +@pytest.mark.order(after="missing_test") +def test_robot(robot): + pass +""") + + result = pytester.runpytest_subprocess("-vv", "--error-on-failed-ordering") + + result.assert_outcomes(errors=1) + + +def test_fail_all_on_failed_ordering_stops_before_worker(pytester): + _make_robot_module(pytester) + _configure_recording_isolated_plugin(pytester) + pytester.makepyfile(test_failed_order=""" +import pytest + +@pytest.mark.order(after="missing_test") +def test_robot(robot): pass +""") + + result = pytester.runpytest_subprocess("-vv", "--fail-all-on-failed-ordering") + + assert result.ret == pytest.ExitCode.USAGE_ERROR + assert not (pytester.path / "isolated-started").exists() + + +@pytest.mark.parametrize( + "args,want", + [ + ((), ["first", "second"]), + (("--indulgent-ordering",), ["second", "first"]), + ], + ids=["default", "indulgent"], +) +def test_indulgent_ordering_consumes_final_collection_order(pytester, args, want): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + with (pytester.path / "conftest.py").open("a") as fp: + fp.write(""" + +import pytest + +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(items): + items.reverse() +""") + pytester.makepyfile(test_hook_order=""" +import pathlib +import pytest + +EVENTS = pathlib.Path("events.txt") + +def record(event): + with EVENTS.open("a") as fp: + fp.write(event + "\\n") + +@pytest.mark.order(2) +def test_second(robot): + record("second") + +@pytest.mark.order(1) +def test_first(): + record("first") +""") + + result = pytester.runpytest_subprocess("-vv", *args) + + result.assert_outcomes(passed=2) + assert (pytester.path / "events.txt").read_text().splitlines() == want + + +@pytest.mark.parametrize( + "order_after_ff,want", + [ + (False, ["failed", "ordered"]), + (True, ["ordered", "failed"]), + ], +) +def test_order_after_failed_first(pytester, order_after_ff, want): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makepyfile(test_failed_first=""" +import pathlib +import pytest + +EVENTS = pathlib.Path("events.txt") +SEEN = pathlib.Path("seen-failure.txt") + +def record(event): + with EVENTS.open("a") as fp: + fp.write(event + "\\n") + +@pytest.mark.order("first") +def test_ordered(robot): + record("ordered") + +def test_cached_failure(): + record("failed") + if not SEEN.exists(): + SEEN.touch() + assert False +""") + + first = pytester.runpytest_subprocess("-q") + first.assert_outcomes(passed=1, failed=1) + (pytester.path / "events.txt").unlink() + + args = ["-q", "--ff"] + if order_after_ff: + args.append("--order-after-ff") + second = pytester.runpytest_subprocess(*args) + + second.assert_outcomes(passed=2) + assert (pytester.path / "events.txt").read_text().splitlines() == want + + +@pytest.mark.parametrize( + "option", + [ + "--order-scope=module", + "--order-scope-level=1", + "--sparse-ordering", + "--indulgent-ordering", + "--order-after-ff", + ], +) +def test_parent_sort_options_are_available_in_worker(pytester, option): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makepyfile(test_ordered=""" +import pytest + + +@pytest.mark.order(1) +def test_robot(robot): + assert robot is not None +""") + + result = pytester.runpytest_subprocess("-v", option) + + result.assert_outcomes(passed=1) + + +def test_multiple_order_markers_collect_in_isolated_subprocess(pytester): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makepyfile(test_ordered=""" +import pytest + + +@pytest.mark.order(2) +@pytest.mark.order(1) +def test_robot(robot): + assert robot is not None +""") + + result = pytester.runpytest_subprocess("-v") + + result.assert_outcomes(passed=2) diff --git a/subprojects/robotpy-wpilib/tests/test_pytest_plugins.py b/subprojects/robotpy-wpilib/tests/test_pytest_plugins.py index 48b6676f8..3a36d964a 100644 --- a/subprojects/robotpy-wpilib/tests/test_pytest_plugins.py +++ b/subprojects/robotpy-wpilib/tests/test_pytest_plugins.py @@ -394,3 +394,57 @@ def test_state_transitions(robot, control): ) result.assert_outcomes(passed=1) + + +@pytest.mark.parametrize( + "a_fixture, b_fixture", + [("robot", "robot"), ("robot", ""), ("", "robot")], + ids=["RR", "RN", "NR"], +) +def test_unordered_tests_still_run_in_parallel(pytester, a_fixture, b_fixture): + """ + Tests WITHOUT @pytest.mark.order must not be serialised by order-marker + support. With parallelism=2, two 1.5 s tests must overlap in wall-clock + time. + + NR is the case collection order alone cannot deliver: the plain test is + collected first, so following the given order would run it to completion + before the subprocess was even spawned. Within a group the run loop is free + to schedule for throughput, so it starts the isolated test before running any + in-process test and the two overlap regardless of which was collected first. + + NN is omitted: two in-process tests both run here, and this process runs one + test at a time -- serial by design, and no scheduling can change it. + """ + _make_robot_module(pytester) + _configure_isolated_plugin(pytester, parallelism=2) + + def params(f): + return f"({f})" if f else "()" + + pytester.makepyfile(test_parallel_execution=f"""\ +import pathlib +import time + + +def test_a{params(a_fixture)}: + pathlib.Path("a_start.txt").write_text(str(time.monotonic())) + time.sleep(1.5) + pathlib.Path("a_end.txt").write_text(str(time.monotonic())) + + +def test_b{params(b_fixture)}: + pathlib.Path("b_start.txt").write_text(str(time.monotonic())) + time.sleep(1.5) + pathlib.Path("b_end.txt").write_text(str(time.monotonic())) +""") + + result = pytester.runpytest_subprocess("-vv") + result.assert_outcomes(passed=2) + + root = pathlib.Path(pytester.path) + a_end = float((root / "a_end.txt").read_text()) + b_start = float((root / "b_start.txt").read_text()) + assert ( + b_start < a_end + ), f"Expected parallel: b_start={b_start:.3f} a_end={a_end:.3f}" diff --git a/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_order_adapter.py b/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_order_adapter.py new file mode 100644 index 000000000..451be7fff --- /dev/null +++ b/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_order_adapter.py @@ -0,0 +1,145 @@ +import dataclasses +import typing as T + +import pytest + +_FAILED_ORDER_FIXTURE = "fail_after_cannot_order" + + +@dataclasses.dataclass(frozen=True) +class PytestOrderWorkerState: + extra_fixtures: tuple[str, ...] = () + + +class PytestOrderWorkerPlugin: + def __init__(self, state: PytestOrderWorkerState): + self._state = state + + @pytest.hookimpl(trylast=True) + def pytest_configure(self, config: pytest.Config): + ordering_plugin = config.pluginmanager.get_plugin("orderingplugin") + if ordering_plugin is not None: + config.pluginmanager.unregister(ordering_plugin) + + @pytest.hookimpl + def pytest_collection_modifyitems(self, items: list[pytest.Item]): + for item in items: + for fixture in reversed(self._state.extra_fixtures): + if fixture not in item.fixturenames: + item.fixturenames.insert(0, fixture) + + +class PytestOrderAdapter: + def __init__(self, config: pytest.Config): + self._config = config + self._dependency_ordering = config.getoption( + "order_dependencies", default=False + ) + self._marker_prefix = config.getoption("order_marker_prefix", default=None) + self._group_scope = self._effective_group_scope(config) + self._scope_level = config.getoption("order_scope_level", default=None) or 0 + if config.getoption("order_scope", default=None) in ("module", "class"): + self._scope_level = 0 + + def validate(self, items: list[pytest.Item]) -> None: + if self._config.pluginmanager.get_plugin("orderingplugin") is None and any( + self._requests_ordering(item) for item in items + ): + raise pytest.UsageError( + "pytest-order is required to use order markers with isolated tests" + ) + + def groups(self, items: list[pytest.Item]) -> T.Iterator[list[pytest.Item]]: + group: list[pytest.Item] = [] + previous_order: object = object() + previous_dependency: object = object() + previous_structure: object = object() + + for item in items: + literal_order = item.get_closest_marker("order") + order = ( + literal_order + if literal_order is not None + else self._prefixed_marker(item) + ) + dependency = ( + item.get_closest_marker("dependency") + if self._dependency_ordering or order is not None + else None + ) + # pytest-order partitions directory scopes before applying group + # scopes. Preserve both boundaries in the isolated scheduler. + directory = ( + "/".join(item.nodeid.split("::", 1)[0].split("/")[: self._scope_level]) + if self._scope_level > 0 + else None + ) + structure = (directory, self._structural_group(item)) + + if group and ( + order is not previous_order + or dependency is not previous_dependency + or structure != previous_structure + ): + yield group + group = [] + + group.append(item) + previous_order = order + previous_dependency = dependency + previous_structure = structure + + if group: + yield group + + def worker_state(self, item: pytest.Function) -> PytestOrderWorkerState: + fixtures = ( + (_FAILED_ORDER_FIXTURE,) + if _FAILED_ORDER_FIXTURE in item.fixturenames + else () + ) + return PytestOrderWorkerState(fixtures) + + @staticmethod + def _effective_group_scope(config: pytest.Config) -> str | None: + ranks = {"class": 1, "module": 2, "session": 3} + order_scope = config.getoption("order_scope", default=None) + if order_scope not in ranks: + order_scope = "session" + group_scope = config.getoption("order_group_scope", default=None) + if group_scope not in ranks: + group_scope = order_scope + if ranks[group_scope] >= ranks[order_scope]: + return None + return group_scope + + def _structural_group(self, item: pytest.Item) -> str | None: + if self._group_scope == "module": + return item.nodeid.split("::", 1)[0] + if self._group_scope == "class": + class_collector = item.getparent(pytest.Class) + if class_collector is not None: + return class_collector.nodeid + return item.nodeid.split("::", 1)[0] + return None + + def _requests_ordering(self, item: pytest.Item) -> bool: + if item.get_closest_marker("order") is not None: + return True + if self._prefixed_marker(item) is not None: + return True + return self._dependency_ordering and ( + item.get_closest_marker("dependency") is not None + ) + + def _prefixed_marker(self, item: pytest.Item): + if not self._marker_prefix: + return None + for marker in item.iter_markers(): + if marker.name.startswith(self._marker_prefix): + try: + int(marker.name[len(self._marker_prefix)]) + except (IndexError, ValueError): + continue + return marker + return None diff --git a/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_tests_plugin.py b/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_tests_plugin.py index df5d905be..cf9351f40 100644 --- a/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_tests_plugin.py +++ b/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_tests_plugin.py @@ -16,6 +16,11 @@ import wpilib +from .pytest_isolated_order_adapter import ( + PytestOrderAdapter, + PytestOrderWorkerPlugin, + PytestOrderWorkerState, +) from .pytest_plugin import RobotTestingPlugin @@ -128,7 +133,14 @@ def pytest_runtest_logreport(self, report: pytest.TestReport): def _run_test( - item_nodeid, config_args, robot_class_data, robot_file, verbose, pipe, root_path + item_nodeid, + config_args, + robot_class_data, + robot_file, + verbose, + order_state: PytestOrderWorkerState, + pipe, + root_path, ): """This function runs in a subprocess""" logging.root.addHandler(logging.NullHandler()) @@ -147,10 +159,17 @@ def _run_test( # and we don't want it to die and deadlock plugin = RobotTestingPlugin(robot_class, robot_file, True) worker_plugin = WorkerPlugin(pipe) + order_plugin = PytestOrderWorkerPlugin(order_state) ec = pytest.main( - [item_nodeid, "--no-header", "-p", "no:terminalreporter", *config_args], - plugins=[plugin, worker_plugin], + [ + item_nodeid, + "--no-header", + "-p", + "no:terminalreporter", + *config_args, + ], + plugins=[plugin, worker_plugin, order_plugin], ) # ensure output is printed out @@ -212,6 +231,13 @@ def __init__( self._parallelism = max(1, parallelism) self._shouldstop = False + @pytest.hookimpl(trylast=True) + def pytest_collection_modifyitems( + self, config: pytest.Config, items: list[pytest.Item] + ): + self._ordering = PytestOrderAdapter(config) + self._ordering.validate(items) + @pytest.hookimpl(wrapper=True) def pytest_sessionstart(self, session: pytest.Session): self._config = session.config @@ -236,41 +262,57 @@ def pytest_runtestloop(self, session: pytest.Session) -> bool: if session.config.option.collectonly: return True + inprocess_items = [ + item + for item in session.items + if not isinstance(item, pytest.Function) or "robot" not in item.fixturenames + ] + inprocess_index = 0 running: list[IsolatedTestJob] = [] - deferred: list[pytest.Item] = [] try: - # Start any tests that use the robot fixture first. Tests that don't - # use the robot fixture will be ran later - for item in session.items: - if not ( - # pytest plugins may generate their own items. - isinstance(item, pytest.Function) - and "robot" in item.fixturenames - ): - deferred.append(item) - continue - - while len(running) >= self._parallelism: + # pytest-order has already sorted session.items. Preserve boundaries + # between ordering-marker groups while retaining the existing parallel + # scheduling within each group. + for group in self._ordering.groups(session.items): + deferred: list[pytest.Item] = [] + + # Start this group's robot tests first, then overlap its plain + # tests with the isolated jobs just as the original loop did. + for item in group: + if not ( + # pytest plugins may generate their own items. + isinstance(item, pytest.Function) + and "robot" in item.fixturenames + ): + deferred.append(item) + continue + + while len(running) >= self._parallelism: + self._wait_for_jobs(running, session) + + running.append(self._start_isolated_test(item)) + self._maybe_raise(session) + + for item in deferred: + # Observe completed isolated failures before starting more + # in-process work, so --maxfail can stop the group promptly. + if any(job.conn.poll() for job in running): + self._wait_for_jobs(running, session) + + nextitem = ( + inprocess_items[inprocess_index + 1] + if inprocess_index + 1 < len(inprocess_items) + else None + ) + inprocess_index += 1 + session.config.hook.pytest_runtest_protocol( + item=item, nextitem=nextitem + ) + self._maybe_raise(session) + + # No work from one marker group may cross into the next. + while running: self._wait_for_jobs(running, session) - - running.append(self._start_isolated_test(item)) - self._maybe_raise(session) - - # Run the in-process tests now while the robot tests are finishing - for idx, item in enumerate(deferred): - # Observe completed isolated failures before starting more - # in-process work, so --maxfail can stop the group promptly. - if any(job.conn.poll() for job in running): - self._wait_for_jobs(running, session) - - nextitem = deferred[idx + 1] if idx + 1 < len(deferred) else None - session.config.hook.pytest_runtest_protocol( - item=item, nextitem=nextitem - ) - self._maybe_raise(session) - - while running: - self._wait_for_jobs(running, session) finally: for job in running: self._cleanup_job(job) @@ -295,6 +337,7 @@ def _start_isolated_test(self, item: pytest.Function) -> IsolatedTestJob: pickle.dumps(self._robot_class), self._robot_file, self._verbose, + self._ordering.worker_state(item), cconn, self._config.rootpath, ),