diff --git a/subprojects/robotpy-wpilib/tests/pytest_plugin_test_helpers.py b/subprojects/robotpy-wpilib/tests/pytest_plugin_test_helpers.py new file mode 100644 index 000000000..0f6558f4d --- /dev/null +++ b/subprojects/robotpy-wpilib/tests/pytest_plugin_test_helpers.py @@ -0,0 +1,92 @@ +import pathlib + + +def _make_robot_module(pytester): + pytester.makepyfile(robot_module=""" +import wpilib + + +class DummyRobot(wpilib.TimedRobot): + def __init__(self): + super().__init__() + self.did_init = True + + +class AutonomousPeriodicFailed(wpilib.TimedRobot): + def autonomous_periodic(self): + assert False + + +class TeleopPeriodicFailed(wpilib.TimedRobot): + def teleop_periodic(self): + assert False + + +class TeleopInitFailed(wpilib.TimedRobot): + def teleop_init(self): + assert False + + +class IterativeStateRobot(wpilib.TimedRobot): + def disabled_init(self): + self.did_disabled_init = True + + def disabled_periodic(self): + self.did_disabled_periodic = True + + def autonomous_init(self): + self.did_auto_init = True + + def autonomous_periodic(self): + self.did_auto_periodic = True + + def teleop_init(self): + self.did_teleop_init = True + + def teleop_periodic(self): + self.did_teleop_periodic = True +""") + + +def _configure_robot_testing_plugin(pytester, robot_class="DummyRobot"): + pytester.makeconftest(f""" +import pathlib + +from wpilib.testing.pytest_plugin import RobotTestingPlugin + +from robot_module import {robot_class} + + +def pytest_configure(config): + robot_file = pathlib.Path(__file__).resolve() + config.pluginmanager.register(RobotTestingPlugin({robot_class}, robot_file, False)) +""") + + +def _configure_isolated_plugin( + pytester, + parallelism=1, + robot_class="DummyRobot", + robot_module="robot_module", + robot_file_name=None, +): + if robot_file_name is None: + robot_file = "pathlib.Path(__file__).resolve()" + else: + robot_file = f"pathlib.Path(__file__).parent / {robot_file_name!r}" + + pytester.makeconftest(f""" +import pathlib + +from wpilib.testing.pytest_isolated_tests_plugin import IsolatedTestsPlugin + +from {robot_module} import {robot_class} + +def pytest_configure(config): + if "--no-header" in config.invocation_params.args: + return + robot_file = {robot_file} + config.pluginmanager.register( + IsolatedTestsPlugin({robot_class}, robot_file, False, False, {parallelism}) + ) +""") diff --git a/subprojects/robotpy-wpilib/tests/requirements.txt b/subprojects/robotpy-wpilib/tests/requirements.txt index e079f8a60..114768e8b 100644 --- a/subprojects/robotpy-wpilib/tests/requirements.txt +++ b/subprojects/robotpy-wpilib/tests/requirements.txt @@ -1 +1,2 @@ pytest +pytest-order diff --git a/subprojects/robotpy-wpilib/tests/test_pytest_isolated_order_adapter.py b/subprojects/robotpy-wpilib/tests/test_pytest_isolated_order_adapter.py new file mode 100644 index 000000000..c8a488ed9 --- /dev/null +++ b/subprojects/robotpy-wpilib/tests/test_pytest_isolated_order_adapter.py @@ -0,0 +1,1129 @@ +import pathlib +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from pytest_plugin_test_helpers import ( + _configure_isolated_plugin, + _make_robot_module, +) +from wpilib.testing.pytest_isolated_order_adapter import PytestOrderAdapter + + +def _configure_recording_isolated_plugin(pytester): + pytester.makeconftest(""" +import pathlib + +from wpilib.testing.pytest_isolated_tests_plugin import IsolatedTestsPlugin + +from robot_module import DummyRobot + + +class RecordingIsolatedTestsPlugin(IsolatedTestsPlugin): + def _start_isolated_test(self, item): + pathlib.Path("isolated-started").touch() + return super()._start_isolated_test(item) + + +def pytest_configure(config): + if "--no-header" in config.invocation_params.args: + return + config.pluginmanager.register( + RecordingIsolatedTestsPlugin( + DummyRobot, pathlib.Path(__file__).resolve(), False, False, 2 + ) + ) +""") + + +class _AdapterConfig: + def __init__(self, **options): + self._options = options + self.pluginmanager = SimpleNamespace(get_plugin=lambda name: object()) + + def getoption(self, name, default=None): + return self._options.get(name, default) + + +@pytest.mark.parametrize( + "order_scope,group_scope,want", + [ + (None, None, None), + ("session", "module", "module"), + ("session", "class", "class"), + ("module", "class", "class"), + ("module", "session", None), + ("class", "module", None), + ("invalid", "module", "module"), + ], +) +def test_effective_group_scope(order_scope, group_scope, want): + config = _AdapterConfig( + order_scope=order_scope, + order_group_scope=group_scope, + ) + assert PytestOrderAdapter(config)._group_scope == want + + +@pytest.mark.parametrize( + "first_mark,second_mark", + [ + ("@pytest.mark.order(1)", "@pytest.mark.order(2)"), + ("@pytest.mark.order(-2)", "@pytest.mark.order(-1)"), + ("@pytest.mark.order('first')", "@pytest.mark.order('second')"), + ("@pytest.mark.order(index=1)", "@pytest.mark.order(index=2)"), + ("@pytest.mark.order(before='test_second')", ""), + ("", "@pytest.mark.order(after='test_first')"), + ], + ids=["numeric", "negative", "named", "index", "before", "after"], +) +def test_documented_marker_forms_preserve_boundaries(pytester, first_mark, second_mark): + _make_robot_module(pytester) + _configure_recording_isolated_plugin(pytester) + pytester.makepyfile(test_marker_forms=f""" +import pathlib +import pytest + +{second_mark} +def test_second(robot): + assert pathlib.Path("first-finished").exists() + +{first_mark} +def test_first(): + assert not pathlib.Path("isolated-started").exists() + pathlib.Path("first-finished").touch() +""") + + result = pytester.runpytest_subprocess("-vv") + + result.assert_outcomes(passed=2) + + +@pytest.mark.parametrize("container", ["list", "tuple"]) +@pytest.mark.parametrize("direction", ["before", "after"]) +def test_relative_marker_sequences(pytester, container, direction): + _make_robot_module(pytester) + _configure_recording_isolated_plugin(pytester) + if direction == "after": + refs = ( + '["test_first", "test_middle"]' + if container == "list" + else '("test_first", "test_middle")' + ) + first_mark = "" + middle_mark = '@pytest.mark.order(after="test_first")' + last_mark = f"@pytest.mark.order(after={refs})" + else: + refs = ( + '["test_middle", "test_last"]' + if container == "list" + else '("test_middle", "test_last")' + ) + first_mark = f"@pytest.mark.order(before={refs})" + middle_mark = '@pytest.mark.order(before="test_last")' + last_mark = "" + + pytester.makepyfile(test_relative_sequences=f""" +import pathlib +import pytest + +{last_mark} +def test_last(robot): + assert pathlib.Path("first-finished").exists() + assert pathlib.Path("middle-finished").exists() + +{middle_mark} +def test_middle(): + assert pathlib.Path("first-finished").exists() + pathlib.Path("middle-finished").touch() + +{first_mark} +def test_first(): + assert not pathlib.Path("isolated-started").exists() + pathlib.Path("first-finished").touch() +""") + + result = pytester.runpytest_subprocess("-vv") + + result.assert_outcomes(passed=3) + + +def test_isolated_plugin_worker_interruption_stops_session(pytester): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makepyfile(test_isolated=""" +import pathlib +import pytest + + +@pytest.mark.order(1) +def test_robot_exit(robot): + pytest.exit("worker requested exit") + + +@pytest.mark.order(2) +def test_later_group_must_not_run(): + pathlib.Path("later-group-ran").touch() +""") + + result = pytester.runpytest_subprocess("-v") + + assert result.ret == pytest.ExitCode.INTERRUPTED + assert not (pytester.path / "later-group-ran").exists() + + +def test_plain_order_groups_preserve_shared_fixture_scopes(pytester): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makepyfile( + test_plain=""" +import pathlib + +import pytest + + +EVENTS = pathlib.Path("fixture-events.txt") + + +def record(event): + with EVENTS.open("a") as fp: + fp.write(event + "\\n") + + +@pytest.fixture(scope="session") +def session_fixture(): + record("session setup") + yield + record("session teardown") + + +@pytest.fixture(scope="module") +def module_fixture(): + record("module setup") + yield + record("module teardown") + + +@pytest.fixture(scope="class") +def class_fixture(): + record("class setup") + yield + record("class teardown") + + +class TestPlain: + @pytest.mark.order(1) + def test_first(self, session_fixture, module_fixture, class_fixture): + record("first") + + @pytest.mark.order(3) + def test_second(self, session_fixture, module_fixture, class_fixture): + record("second") +""", + test_robot=""" +import pytest + + +@pytest.mark.order(2) +def test_between_plain_groups(robot): + pass +""", + ) + + result = pytester.runpytest_subprocess("-q") + + result.assert_outcomes(passed=3) + assert (pytester.path / "fixture-events.txt").read_text().splitlines() == [ + "session setup", + "module setup", + "class setup", + "first", + "second", + "class teardown", + "module teardown", + "session teardown", + ] + + +@pytest.mark.parametrize("scope", ["module", "class"]) +def test_order_group_scope_preserves_structural_boundaries(pytester, scope): + _make_robot_module(pytester) + _configure_recording_isolated_plugin(pytester) + + if scope == "module": + pytester.makepyfile( + test_early=""" +import pathlib +import pytest + +@pytest.mark.order(1) +def test_order_anchor(): pass + +def test_plain_tail(): + assert not pathlib.Path("isolated-started").exists() +""", + test_late=""" +import pytest + +def test_robot_head(robot): pass + +@pytest.mark.order(-1) +def test_order_anchor(): pass +""", + ) + else: + pytester.makepyfile(test_groups=""" +import pathlib +import pytest + +class TestEarly: + @pytest.mark.order(1) + def test_order_anchor(self): pass + + def test_plain_tail(self): + assert not pathlib.Path("isolated-started").exists() + +class TestLate: + def test_robot_head(self, robot): pass + + @pytest.mark.order(-1) + def test_order_anchor(self): pass +""") + + result = pytester.runpytest_subprocess("-vv", f"--order-group-scope={scope}") + + result.assert_outcomes(passed=4) + + +@pytest.mark.parametrize( + "level,separate_scopes", + [(0, False), (-1, False), (1, False), (2, True), (5, True)], +) +def test_directory_order_scope_preserves_scheduler_boundaries( + pytester, level, separate_scopes +): + _make_robot_module(pytester) + _configure_recording_isolated_plugin(pytester) + early = pytester.path / "tests" / "early" + late = pytester.path / "tests" / "late" + early.mkdir(parents=True) + late.mkdir(parents=True) + (early / "test_plain.py").write_text(f""" +import pathlib + + +def test_plain_tail(): + assert pathlib.Path("isolated-started").exists() is {not separate_scopes} +""") + (late / "test_robot.py").write_text(""" +def test_robot_head(robot): + assert robot is not None +""") + + result = pytester.runpytest_subprocess("-q", f"--order-scope-level={level}") + + result.assert_outcomes(passed=2) + + +def test_class_order_group_scope_preserves_nested_class_boundaries(pytester): + _make_robot_module(pytester) + _configure_recording_isolated_plugin(pytester) + pytester.makepyfile(test_nested_groups=""" +import pathlib +import pytest + +class TestOuter: + class TestEarly: + @pytest.mark.order(1) + def test_order_anchor(self): pass + + def test_plain_tail(self): + assert not pathlib.Path("isolated-started").exists() + + class TestLate: + def test_robot_head(self, robot): pass + + @pytest.mark.order(-1) + def test_order_anchor(self): pass +""") + + result = pytester.runpytest_subprocess("-vv", "--order-group-scope=class") + + result.assert_outcomes(passed=4) + + +def test_module_order_marker_forms_one_parallel_group(pytester): + items = pytester.getitems(""" +import pytest + +pytestmark = pytest.mark.order(1) + + +def test_one(): pass + +def test_two(): pass +""") + adapter = PytestOrderAdapter(_AdapterConfig()) + + groups = list(adapter.groups(items)) + + assert [[item.name for item in group] for group in groups] == [ + ["test_one", "test_two"] + ] + assert items[0].get_closest_marker("order") is items[1].get_closest_marker("order") + + +def test_order_marker_groups_use_real_pytest_marker_inheritance(pytester): + items = pytester.getitems(""" +import pytest + + +@pytest.mark.order(1) +class TestInherited: + def test_inherited_one(self): + pass + + def test_inherited_two(self): + pass + + +@pytest.mark.order(1) +def test_independent_one(): + pass + + +@pytest.mark.order(1) +def test_independent_two(): + pass + + +@pytest.mark.order(1) +class TestOverride: + def test_class_marker(self): + pass + + @pytest.mark.order(2) + def test_function_override(self): + pass +""") + items_by_name = {item.name: item for item in items} + ordered_items = [ + items_by_name[name] + for name in ( + "test_inherited_one", + "test_inherited_two", + "test_independent_one", + "test_independent_two", + "test_class_marker", + "test_function_override", + ) + ] + + config = SimpleNamespace( + getoption=lambda name, default=None: default, + pluginmanager=SimpleNamespace(get_plugin=lambda name: object()), + ) + adapter = PytestOrderAdapter(config) + groups = adapter.groups(ordered_items) + + assert [[item.name for item in group] for group in groups] == [ + ["test_inherited_one", "test_inherited_two"], + ["test_independent_one"], + ["test_independent_two"], + ["test_class_marker"], + ["test_function_override"], + ] + assert items_by_name["test_class_marker"].get_closest_marker("order").args == (1,) + assert items_by_name["test_function_override"].get_closest_marker("order").args == ( + 2, + ) + + +class _SchedulerHook: + def __init__(self, events): + self.events = events + + def pytest_runtest_protocol(self, item, nextitem): + self.events.append(("plain", item.name)) + + +class _SchedulerSession: + Interrupted = pytest.Session.Interrupted + Failed = pytest.Session.Failed + + def __init__(self, items, events): + self.items = items + self.testsfailed = 0 + self.shouldfail = False + self.shouldstop = False + self.config = SimpleNamespace( + option=SimpleNamespace( + continue_on_collection_errors=False, + collectonly=False, + ), + hook=_SchedulerHook(events), + getoption=lambda name, default=None: default, + ) + + +def _scheduler_item(name, marker, uses_robot): + item = MagicMock(spec=pytest.Function) + item.name = name + item.fixturenames = ["robot"] if uses_robot else [] + item.get_closest_marker.return_value = marker + return item + + +@pytest.mark.parametrize( + "order_scope,group_scope,expected", + [ + (None, None, [["first", "second"], ["third"]]), + ("session", "module", [["first"], ["second"], ["third"]]), + ("session", "class", [["first"], ["second"], ["third"]]), + ("module", None, [["first", "second", "third"]]), + ("class", None, [["first", "second", "third"]]), + ], +) +def test_directory_scope_combines_with_group_scope(order_scope, group_scope, expected): + items = [ + _scheduler_item(name, None, False) for name in ("first", "second", "third") + ] + for item, nodeid in zip( + items, + ("early/test_a.py::first", "early/test_b.py::second", "late/test_c.py::third"), + ): + item.nodeid = nodeid + item.getparent.return_value = None + adapter = PytestOrderAdapter( + _AdapterConfig( + order_scope_level=1, + order_scope=order_scope, + order_group_scope=group_scope, + ) + ) + + assert [ + [item.name for item in group] for group in adapter.groups(items) + ] == expected + + +def _scheduler_events(monkeypatch, items): + from wpilib.testing.pytest_isolated_order_adapter import PytestOrderAdapter + from wpilib.testing.pytest_isolated_tests_plugin import IsolatedTestsPlugin + + events = [] + session = _SchedulerSession(items, events) + plugin = IsolatedTestsPlugin(object, pathlib.Path("robot.py"), False, False, 4) + plugin._ordering = PytestOrderAdapter(session.config) + + def start(item): + events.append(("start", item.name)) + conn = SimpleNamespace(poll=lambda: False) + return SimpleNamespace(item=item, conn=conn) + + def wait(running, session): + events.extend(("finish", job.item.name) for job in running) + running.clear() + + monkeypatch.setattr(plugin, "_start_isolated_test", start) + monkeypatch.setattr(plugin, "_wait_for_jobs", wait) + + assert plugin.pytest_runtestloop(session) is True + return events + + +_ORDERED_TYPE_CHAINS = [ + ("ROBOT", "ROBOT", None), + ("ROBOT", "PLAIN", None), + ("PLAIN", "ROBOT", None), + ("PLAIN", "PLAIN", None), + ("ROBOT", "ROBOT", "PLAIN"), + ("ROBOT", "PLAIN", "ROBOT"), + ("PLAIN", "ROBOT", "ROBOT"), + ("ROBOT", "PLAIN", "PLAIN"), + ("PLAIN", "ROBOT", "PLAIN"), + ("PLAIN", "PLAIN", "ROBOT"), +] + + +@pytest.mark.parametrize("marker_style", ["numeric", "after", "before"]) +@pytest.mark.parametrize("test_types", _ORDERED_TYPE_CHAINS) +def test_order_groups_do_not_cross_scheduler_boundaries( + monkeypatch, marker_style, test_types +): + # pytest-order has already sorted these items. The three patterns model + # which tests carry markers for ordinal, after=, and before= ordering. + first_marker = object() + middle_marker = object() + last_marker = object() + markers = { + "numeric": (first_marker, middle_marker, last_marker), + "after": (None, middle_marker, last_marker), + "before": (first_marker, middle_marker, None), + }[marker_style] + + items = [] + expected = [] + for name, test_type, marker in zip( + ("first", "middle", "last"), test_types, markers + ): + if test_type is None: + continue + + uses_robot = test_type == "ROBOT" + items.append(_scheduler_item(name, marker, uses_robot)) + if uses_robot: + expected.extend((("start", name), ("finish", name))) + else: + expected.append(("plain", name)) + + assert _scheduler_events(monkeypatch, items) == expected + + +def test_order_marker_implies_dependency_boundaries(pytester): + _make_robot_module(pytester) + pytester.makeini( + "[pytest]\nmarkers = dependency(*args, **kwargs): test dependency\n" + ) + _configure_recording_isolated_plugin(pytester) + pytester.makepyfile(test_dependency_order=""" +import pathlib + +import pytest + + +@pytest.mark.order(1) +class TestChain: + @pytest.mark.dependency(depends=["prerequisite"]) + def test_isolated_dependent(self, robot): + assert pathlib.Path("prerequisite-finished").exists() + + @pytest.mark.dependency(name="prerequisite") + def test_plain_prerequisite(self): + assert not pathlib.Path("isolated-started").exists() + pathlib.Path("prerequisite-finished").touch() +""") + + result = pytester.runpytest_subprocess("-vv") + + result.assert_outcomes(passed=2) + + +def test_prefixed_order_marker_implies_dependency_boundaries(pytester): + _make_robot_module(pytester) + pytester.makeini( + "[pytest]\n" + "markers =\n" + " sequence1: first test\n" + " dependency(*args, **kwargs): test dependency\n" + ) + _configure_recording_isolated_plugin(pytester) + pytester.makepyfile(test_dependency_order=""" +import pathlib + +import pytest + + +@pytest.mark.sequence1 +class TestChain: + @pytest.mark.dependency(name="prerequisite") + def test_plain_prerequisite(self): + assert not pathlib.Path("isolated-started").exists() + pathlib.Path("prerequisite-finished").touch() + + @pytest.mark.dependency(depends=["prerequisite"]) + def test_isolated_dependent(self, robot): + assert pathlib.Path("prerequisite-finished").exists() +""") + + result = pytester.runpytest_subprocess("-vv", "--order-marker-prefix=sequence") + + result.assert_outcomes(passed=2) + + +def test_dependency_ordering_does_not_cross_scheduler_boundaries(pytester): + _make_robot_module(pytester) + pytester.makeini( + "[pytest]\nmarkers = dependency(*args, **kwargs): test dependency\n" + ) + pytester.makeconftest(""" +import pathlib + +from wpilib.testing.pytest_isolated_tests_plugin import IsolatedTestsPlugin + +from robot_module import DummyRobot + + +class RecordingIsolatedTestsPlugin(IsolatedTestsPlugin): + def _start_isolated_test(self, item): + pathlib.Path("isolated-started").touch() + return super()._start_isolated_test(item) + + +def pytest_configure(config): + if "--no-header" in config.invocation_params.args: + return + config.pluginmanager.register( + RecordingIsolatedTestsPlugin( + DummyRobot, pathlib.Path(__file__).resolve(), False, False, 2 + ) + ) +""") + pytester.makepyfile(test_dependency_order=""" +import pathlib + +import pytest + + +@pytest.mark.dependency(depends=["prerequisite"]) +def test_isolated_dependent(robot): + assert pathlib.Path("prerequisite-finished").exists() + + +@pytest.mark.dependency(name="prerequisite") +def test_plain_prerequisite(): + assert not pathlib.Path("isolated-started").exists() + pathlib.Path("prerequisite-finished").touch() +""") + + result = pytester.runpytest_subprocess("-vv", "--order-dependencies") + + result.assert_outcomes(passed=2) + + +@pytest.mark.parametrize( + "first_marker,second_marker", + [ + ("sequence1", "sequence2"), + ("sequence1_setup", "sequence2_robot"), + ], + ids=["documented", "pytest_order_1_5_compatibility"], +) +def test_prefixed_order_markers_preserve_scheduler_boundaries( + pytester, first_marker, second_marker +): + _make_robot_module(pytester) + pytester.makeini( + "[pytest]\n" + "markers =\n" + f" {first_marker}: first test\n" + f" {second_marker}: second test\n" + ) + pytester.makeconftest(""" +import pathlib + +from wpilib.testing.pytest_isolated_tests_plugin import IsolatedTestsPlugin + +from robot_module import DummyRobot + + +class RecordingIsolatedTestsPlugin(IsolatedTestsPlugin): + def _start_isolated_test(self, item): + pathlib.Path("isolated-started").touch() + return super()._start_isolated_test(item) + + +def pytest_configure(config): + if "--no-header" in config.invocation_params.args: + return + config.pluginmanager.register( + RecordingIsolatedTestsPlugin( + DummyRobot, pathlib.Path(__file__).resolve(), False, False, 2 + ) + ) +""") + pytester.makepyfile(test_prefixed_order=f""" +import pathlib + +import pytest + + +@pytest.mark.{second_marker} +def test_isolated_dependent(robot): + assert pathlib.Path("prerequisite-finished").exists() + + +@pytest.mark.{first_marker} +def test_plain_prerequisite(): + assert not pathlib.Path("isolated-started").exists() + pathlib.Path("prerequisite-finished").touch() +""") + + result = pytester.runpytest_subprocess("-vv", "--order-marker-prefix=sequence") + + result.assert_outcomes(passed=2) + + +def test_inherited_order_marker_group_runs_in_parallel(monkeypatch): + inherited_marker = object() + next_marker = object() + items = [ + _scheduler_item("plain", inherited_marker, False), + _scheduler_item("robot_one", inherited_marker, True), + _scheduler_item("robot_two", inherited_marker, True), + _scheduler_item("next_robot", next_marker, True), + ] + + assert _scheduler_events(monkeypatch, items) == [ + ("start", "robot_one"), + ("start", "robot_two"), + ("plain", "plain"), + ("finish", "robot_one"), + ("finish", "robot_two"), + ("start", "next_robot"), + ("finish", "next_robot"), + ] + + +class _EqualMarker: + def __eq__(self, other): + return isinstance(other, _EqualMarker) + + +def test_equal_but_distinct_order_markers_form_boundaries(monkeypatch): + first_marker = _EqualMarker() + second_marker = _EqualMarker() + items = [ + _scheduler_item("first", first_marker, True), + _scheduler_item("second", second_marker, True), + ] + + assert _scheduler_events(monkeypatch, items) == [ + ("start", "first"), + ("finish", "first"), + ("start", "second"), + ("finish", "second"), + ] + + +@pytest.mark.parametrize( + "source,args", + [ + ( + "@pytest.mark.sequence1\ndef test_robot(robot): pass", + ("--order-marker-prefix=sequence",), + ), + ( + "@pytest.mark.dependency(name='x')\ndef test_robot(robot): pass", + ("--order-dependencies",), + ), + ], + ids=["prefix", "dependency"], +) +def test_nonstandard_ordering_requires_active_sorter(pytester, source, args): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makeini( + "[pytest]\nmarkers =\n" + " sequence1: first\n" + " dependency(*args, **kwargs): dependency\n" + ) + pytester.makepyfile(test_ordered="import pytest\n\n" + source) + + result = pytester.runpytest_subprocess("-p", "no:orderingplugin", *args) + + assert result.ret == pytest.ExitCode.USAGE_ERROR + + +@pytest.mark.parametrize( + "disabled_plugin", ["pytest_order", "orderingplugin"], ids=["absent", "disabled"] +) +def test_order_marker_requires_pytest_order(pytester, disabled_plugin): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makepyfile(test_ordered=""" +import pytest + + +@pytest.mark.order(2) +def test_second(robot): + pass + + +@pytest.mark.order(1) +def test_first(robot): + pass +""") + + result = pytester.runpytest_subprocess("-p", f"no:{disabled_plugin}") + + assert result.ret == pytest.ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines(["*pytest-order is required to use order markers*"]) + + +def test_doctest_runs_in_process_with_isolated_tests(pytester): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + with (pytester.path / "conftest.py").open("a") as fp: + fp.write("\nimport os\nos.environ['ISOLATED_PARENT_PID'] = str(os.getpid())\n") + pytester.makepyfile(test_mixed=''' +""" +>>> import os +>>> assert os.getpid() == int(os.environ["ISOLATED_PARENT_PID"]) +""" + + +def test_robot(robot): + assert robot is not None + + +def test_plain(): + pass +''') + + result = pytester.runpytest_subprocess("--doctest-modules", "-q") + + result.assert_outcomes(passed=3) + + +def test_custom_item_without_fixtures_runs_in_process(monkeypatch, pytester): + class CustomItem(pytest.Item): + def runtest(self): + pass + + session = pytester.getitem("def test_func(): pass").session + custom = CustomItem.from_parent(session, name="custom") + items = [custom, _scheduler_item("robot", None, True)] + + assert _scheduler_events(monkeypatch, items) == [ + ("start", "robot"), + ("plain", "custom"), + ("finish", "robot"), + ] + + +def test_pytest_order_is_optional_without_order_markers(pytester): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makepyfile(test_unordered=""" +def test_robot(robot): + pass +""") + + result = pytester.runpytest_subprocess("-p", "no:pytest_order") + + result.assert_outcomes(passed=1) + + +@pytest.mark.parametrize( + "ini_body, first_mark, second_mark", + [ + ( + "addopts = --strict-markers", + "@pytest.mark.order(1)", + "@pytest.mark.order(2)", + ), + ( + "filterwarnings = error", + "@pytest.mark.order(1)", + "@pytest.mark.order(2)", + ), + ( + "filterwarnings = error", + "", + '@pytest.mark.order(after="test_first")', + ), + ], + ids=["strict_markers", "warnings_as_errors", "relative_marker_stays_quiet"], +) +def test_order_marker_is_registered_in_isolated_subprocess( + pytester, ini_body, first_mark, second_mark +): + """ + Ordered robot tests must still run when the project treats unknown markers + as a hard failure. The worker keeps pytest-order loaded for option parsing + and test generation, but unregisters its collection-ordering plugin because + a one-test worker cannot resolve relative markers. The marker must remain + registered without emitting misleading relative-order warnings. + """ + _make_robot_module(pytester) + _configure_isolated_plugin(pytester, parallelism=2) + pytester.makeini(f"[pytest]\n{ini_body}\n") + + pytester.makepyfile(test_strict_order=f"""\ +import pathlib + +import pytest + + +{second_mark} +def test_second(robot): + assert pathlib.Path("strict_sentinel.txt").exists() + + +{first_mark} +def test_first(robot): + pathlib.Path("strict_sentinel.txt").write_text("done") +""") + + result = pytester.runpytest_subprocess("-vv") + result.assert_outcomes(passed=2, errors=0) + assert not any("cannot execute" in line for line in result.outlines) + + +def test_error_on_failed_ordering_reaches_isolated_worker(pytester): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makepyfile(test_failed_order=""" +import pytest + +@pytest.mark.order(after="missing_test") +def test_robot(robot): + pass +""") + + result = pytester.runpytest_subprocess("-vv", "--error-on-failed-ordering") + + result.assert_outcomes(errors=1) + + +def test_fail_all_on_failed_ordering_stops_before_worker(pytester): + _make_robot_module(pytester) + _configure_recording_isolated_plugin(pytester) + pytester.makepyfile(test_failed_order=""" +import pytest + +@pytest.mark.order(after="missing_test") +def test_robot(robot): pass +""") + + result = pytester.runpytest_subprocess("-vv", "--fail-all-on-failed-ordering") + + assert result.ret == pytest.ExitCode.USAGE_ERROR + assert not (pytester.path / "isolated-started").exists() + + +@pytest.mark.parametrize( + "args,want", + [ + ((), ["first", "second"]), + (("--indulgent-ordering",), ["second", "first"]), + ], + ids=["default", "indulgent"], +) +def test_indulgent_ordering_consumes_final_collection_order(pytester, args, want): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + with (pytester.path / "conftest.py").open("a") as fp: + fp.write(""" + +import pytest + +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(items): + items.reverse() +""") + pytester.makepyfile(test_hook_order=""" +import pathlib +import pytest + +EVENTS = pathlib.Path("events.txt") + +def record(event): + with EVENTS.open("a") as fp: + fp.write(event + "\\n") + +@pytest.mark.order(2) +def test_second(robot): + record("second") + +@pytest.mark.order(1) +def test_first(): + record("first") +""") + + result = pytester.runpytest_subprocess("-vv", *args) + + result.assert_outcomes(passed=2) + assert (pytester.path / "events.txt").read_text().splitlines() == want + + +@pytest.mark.parametrize( + "order_after_ff,want", + [ + (False, ["failed", "ordered"]), + (True, ["ordered", "failed"]), + ], +) +def test_order_after_failed_first(pytester, order_after_ff, want): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makepyfile(test_failed_first=""" +import pathlib +import pytest + +EVENTS = pathlib.Path("events.txt") +SEEN = pathlib.Path("seen-failure.txt") + +def record(event): + with EVENTS.open("a") as fp: + fp.write(event + "\\n") + +@pytest.mark.order("first") +def test_ordered(robot): + record("ordered") + +def test_cached_failure(): + record("failed") + if not SEEN.exists(): + SEEN.touch() + assert False +""") + + first = pytester.runpytest_subprocess("-q") + first.assert_outcomes(passed=1, failed=1) + (pytester.path / "events.txt").unlink() + + args = ["-q", "--ff"] + if order_after_ff: + args.append("--order-after-ff") + second = pytester.runpytest_subprocess(*args) + + second.assert_outcomes(passed=2) + assert (pytester.path / "events.txt").read_text().splitlines() == want + + +@pytest.mark.parametrize( + "option", + [ + "--order-scope=module", + "--order-scope-level=1", + "--sparse-ordering", + "--indulgent-ordering", + "--order-after-ff", + ], +) +def test_parent_sort_options_are_available_in_worker(pytester, option): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makepyfile(test_ordered=""" +import pytest + + +@pytest.mark.order(1) +def test_robot(robot): + assert robot is not None +""") + + result = pytester.runpytest_subprocess("-v", option) + + result.assert_outcomes(passed=1) + + +def test_multiple_order_markers_collect_in_isolated_subprocess(pytester): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + pytester.makepyfile(test_ordered=""" +import pytest + + +@pytest.mark.order(2) +@pytest.mark.order(1) +def test_robot(robot): + assert robot is not None +""") + + result = pytester.runpytest_subprocess("-v") + + result.assert_outcomes(passed=2) diff --git a/subprojects/robotpy-wpilib/tests/test_pytest_plugins.py b/subprojects/robotpy-wpilib/tests/test_pytest_plugins.py index 40326e880..3a36d964a 100644 --- a/subprojects/robotpy-wpilib/tests/test_pytest_plugins.py +++ b/subprojects/robotpy-wpilib/tests/test_pytest_plugins.py @@ -3,98 +3,11 @@ import pytest - -def _make_robot_module(pytester): - pytester.makepyfile(robot_module=""" -import wpilib - - -class DummyRobot(wpilib.TimedRobot): - def __init__(self): - super().__init__() - self.did_init = True - - -class AutonomousPeriodicFailed(wpilib.TimedRobot): - def autonomous_periodic(self): - assert False - - -class TeleopPeriodicFailed(wpilib.TimedRobot): - def teleop_periodic(self): - assert False - - -class TeleopInitFailed(wpilib.TimedRobot): - def teleop_init(self): - assert False - - -class IterativeStateRobot(wpilib.TimedRobot): - - def disabled_init(self): - self.did_disabled_init = True - - def disabled_periodic(self): - self.did_disabled_periodic = True - - def autonomous_init(self): - self.did_auto_init = True - - def autonomous_periodic(self): - self.did_auto_periodic = True - - def teleop_init(self): - self.did_teleop_init = True - - def teleop_periodic(self): - self.did_teleop_periodic = True - -""") - - -def _configure_robot_testing_plugin(pytester, robot_class="DummyRobot"): - pytester.makeconftest(f""" -import pathlib - -from wpilib.testing.pytest_plugin import RobotTestingPlugin - -from robot_module import {robot_class} - - -def pytest_configure(config): - robot_file = pathlib.Path(__file__).resolve() - config.pluginmanager.register(RobotTestingPlugin({robot_class}, robot_file, False)) -""") - - -def _configure_isolated_plugin( - pytester, - parallelism=1, - robot_class="DummyRobot", - robot_module="robot_module", - robot_file_name=None, -): - if robot_file_name is None: - robot_file = "pathlib.Path(__file__).resolve()" - else: - robot_file = f"pathlib.Path(__file__).parent / {robot_file_name!r}" - - pytester.makeconftest(f""" -import pathlib - -from wpilib.testing.pytest_isolated_tests_plugin import IsolatedTestsPlugin - -from {robot_module} import {robot_class} - -def pytest_configure(config): - if "--no-header" in config.invocation_params.args: - return - robot_file = {robot_file} - config.pluginmanager.register( - IsolatedTestsPlugin({robot_class}, robot_file, False, False, {parallelism}) - ) -""") +from pytest_plugin_test_helpers import ( + _configure_isolated_plugin, + _configure_robot_testing_plugin, + _make_robot_module, +) def test_robot_testing_plugin_success(pytester): @@ -311,6 +224,28 @@ def test_robot_two(robot): ) +def test_isolated_plugin_reports_worker_collection_exit(pytester): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester) + with pytester.path.joinpath("conftest.py").open("a") as fp: + fp.write(""" + + +def pytest_collection_modifyitems(config, items): + if "--no-header" in config.invocation_params.args: + items.clear() +""") + pytester.makepyfile(test_isolated=""" +def test_robot(robot): + assert robot is not None +""") + + result = pytester.runpytest_subprocess("-v") + + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines(["*subprocess exited with exit code 5*"]) + + def test_isolated_plugin_maxfail_stops_early(pytester): _make_robot_module(pytester) _configure_isolated_plugin(pytester) @@ -329,6 +264,33 @@ def test_robot_second(robot): assert not any("test_robot_second" in line for line in result.outlines) +def test_isolated_plugin_maxfail_stops_deferred_tests(pytester): + _make_robot_module(pytester) + _configure_isolated_plugin(pytester, parallelism=2) + pytester.makepyfile(test_isolated=""" +import pathlib +import time + + +def test_robot_failure(robot): + assert False + + +def test_plain_waits_for_failure(): + time.sleep(1) + + +def test_plain_must_not_run(): + pathlib.Path("plain-ran").touch() +""") + + result = pytester.runpytest_subprocess("-v", "-x") + + assert result.parseoutcomes()["failed"] == 1 + assert result.ret == pytest.ExitCode.TESTS_FAILED + assert not (pytester.path / "plain-ran").exists() + + @pytest.mark.parametrize("isolated", [False, True]) def test_builtin_tests_module(pytester, isolated): _make_robot_module(pytester) @@ -432,3 +394,57 @@ def test_state_transitions(robot, control): ) result.assert_outcomes(passed=1) + + +@pytest.mark.parametrize( + "a_fixture, b_fixture", + [("robot", "robot"), ("robot", ""), ("", "robot")], + ids=["RR", "RN", "NR"], +) +def test_unordered_tests_still_run_in_parallel(pytester, a_fixture, b_fixture): + """ + Tests WITHOUT @pytest.mark.order must not be serialised by order-marker + support. With parallelism=2, two 1.5 s tests must overlap in wall-clock + time. + + NR is the case collection order alone cannot deliver: the plain test is + collected first, so following the given order would run it to completion + before the subprocess was even spawned. Within a group the run loop is free + to schedule for throughput, so it starts the isolated test before running any + in-process test and the two overlap regardless of which was collected first. + + NN is omitted: two in-process tests both run here, and this process runs one + test at a time -- serial by design, and no scheduling can change it. + """ + _make_robot_module(pytester) + _configure_isolated_plugin(pytester, parallelism=2) + + def params(f): + return f"({f})" if f else "()" + + pytester.makepyfile(test_parallel_execution=f"""\ +import pathlib +import time + + +def test_a{params(a_fixture)}: + pathlib.Path("a_start.txt").write_text(str(time.monotonic())) + time.sleep(1.5) + pathlib.Path("a_end.txt").write_text(str(time.monotonic())) + + +def test_b{params(b_fixture)}: + pathlib.Path("b_start.txt").write_text(str(time.monotonic())) + time.sleep(1.5) + pathlib.Path("b_end.txt").write_text(str(time.monotonic())) +""") + + result = pytester.runpytest_subprocess("-vv") + result.assert_outcomes(passed=2) + + root = pathlib.Path(pytester.path) + a_end = float((root / "a_end.txt").read_text()) + b_start = float((root / "b_start.txt").read_text()) + assert ( + b_start < a_end + ), f"Expected parallel: b_start={b_start:.3f} a_end={a_end:.3f}" diff --git a/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_order_adapter.py b/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_order_adapter.py new file mode 100644 index 000000000..451be7fff --- /dev/null +++ b/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_order_adapter.py @@ -0,0 +1,145 @@ +import dataclasses +import typing as T + +import pytest + +_FAILED_ORDER_FIXTURE = "fail_after_cannot_order" + + +@dataclasses.dataclass(frozen=True) +class PytestOrderWorkerState: + extra_fixtures: tuple[str, ...] = () + + +class PytestOrderWorkerPlugin: + def __init__(self, state: PytestOrderWorkerState): + self._state = state + + @pytest.hookimpl(trylast=True) + def pytest_configure(self, config: pytest.Config): + ordering_plugin = config.pluginmanager.get_plugin("orderingplugin") + if ordering_plugin is not None: + config.pluginmanager.unregister(ordering_plugin) + + @pytest.hookimpl + def pytest_collection_modifyitems(self, items: list[pytest.Item]): + for item in items: + for fixture in reversed(self._state.extra_fixtures): + if fixture not in item.fixturenames: + item.fixturenames.insert(0, fixture) + + +class PytestOrderAdapter: + def __init__(self, config: pytest.Config): + self._config = config + self._dependency_ordering = config.getoption( + "order_dependencies", default=False + ) + self._marker_prefix = config.getoption("order_marker_prefix", default=None) + self._group_scope = self._effective_group_scope(config) + self._scope_level = config.getoption("order_scope_level", default=None) or 0 + if config.getoption("order_scope", default=None) in ("module", "class"): + self._scope_level = 0 + + def validate(self, items: list[pytest.Item]) -> None: + if self._config.pluginmanager.get_plugin("orderingplugin") is None and any( + self._requests_ordering(item) for item in items + ): + raise pytest.UsageError( + "pytest-order is required to use order markers with isolated tests" + ) + + def groups(self, items: list[pytest.Item]) -> T.Iterator[list[pytest.Item]]: + group: list[pytest.Item] = [] + previous_order: object = object() + previous_dependency: object = object() + previous_structure: object = object() + + for item in items: + literal_order = item.get_closest_marker("order") + order = ( + literal_order + if literal_order is not None + else self._prefixed_marker(item) + ) + dependency = ( + item.get_closest_marker("dependency") + if self._dependency_ordering or order is not None + else None + ) + # pytest-order partitions directory scopes before applying group + # scopes. Preserve both boundaries in the isolated scheduler. + directory = ( + "/".join(item.nodeid.split("::", 1)[0].split("/")[: self._scope_level]) + if self._scope_level > 0 + else None + ) + structure = (directory, self._structural_group(item)) + + if group and ( + order is not previous_order + or dependency is not previous_dependency + or structure != previous_structure + ): + yield group + group = [] + + group.append(item) + previous_order = order + previous_dependency = dependency + previous_structure = structure + + if group: + yield group + + def worker_state(self, item: pytest.Function) -> PytestOrderWorkerState: + fixtures = ( + (_FAILED_ORDER_FIXTURE,) + if _FAILED_ORDER_FIXTURE in item.fixturenames + else () + ) + return PytestOrderWorkerState(fixtures) + + @staticmethod + def _effective_group_scope(config: pytest.Config) -> str | None: + ranks = {"class": 1, "module": 2, "session": 3} + order_scope = config.getoption("order_scope", default=None) + if order_scope not in ranks: + order_scope = "session" + group_scope = config.getoption("order_group_scope", default=None) + if group_scope not in ranks: + group_scope = order_scope + if ranks[group_scope] >= ranks[order_scope]: + return None + return group_scope + + def _structural_group(self, item: pytest.Item) -> str | None: + if self._group_scope == "module": + return item.nodeid.split("::", 1)[0] + if self._group_scope == "class": + class_collector = item.getparent(pytest.Class) + if class_collector is not None: + return class_collector.nodeid + return item.nodeid.split("::", 1)[0] + return None + + def _requests_ordering(self, item: pytest.Item) -> bool: + if item.get_closest_marker("order") is not None: + return True + if self._prefixed_marker(item) is not None: + return True + return self._dependency_ordering and ( + item.get_closest_marker("dependency") is not None + ) + + def _prefixed_marker(self, item: pytest.Item): + if not self._marker_prefix: + return None + for marker in item.iter_markers(): + if marker.name.startswith(self._marker_prefix): + try: + int(marker.name[len(self._marker_prefix)]) + except (IndexError, ValueError): + continue + return marker + return None diff --git a/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_tests_plugin.py b/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_tests_plugin.py index 27c2b6440..cf9351f40 100644 --- a/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_tests_plugin.py +++ b/subprojects/robotpy-wpilib/wpilib/testing/pytest_isolated_tests_plugin.py @@ -16,6 +16,11 @@ import wpilib +from .pytest_isolated_order_adapter import ( + PytestOrderAdapter, + PytestOrderWorkerPlugin, + PytestOrderWorkerState, +) from .pytest_plugin import RobotTestingPlugin @@ -128,7 +133,14 @@ def pytest_runtest_logreport(self, report: pytest.TestReport): def _run_test( - item_nodeid, config_args, robot_class_data, robot_file, verbose, pipe, root_path + item_nodeid, + config_args, + robot_class_data, + robot_file, + verbose, + order_state: PytestOrderWorkerState, + pipe, + root_path, ): """This function runs in a subprocess""" logging.root.addHandler(logging.NullHandler()) @@ -147,10 +159,17 @@ def _run_test( # and we don't want it to die and deadlock plugin = RobotTestingPlugin(robot_class, robot_file, True) worker_plugin = WorkerPlugin(pipe) + order_plugin = PytestOrderWorkerPlugin(order_state) ec = pytest.main( - [item_nodeid, "--no-header", "-p", "no:terminalreporter", *config_args], - plugins=[plugin, worker_plugin], + [ + item_nodeid, + "--no-header", + "-p", + "no:terminalreporter", + *config_args, + ], + plugins=[plugin, worker_plugin, order_plugin], ) # ensure output is printed out @@ -212,6 +231,13 @@ def __init__( self._parallelism = max(1, parallelism) self._shouldstop = False + @pytest.hookimpl(trylast=True) + def pytest_collection_modifyitems( + self, config: pytest.Config, items: list[pytest.Item] + ): + self._ordering = PytestOrderAdapter(config) + self._ordering.validate(items) + @pytest.hookimpl(wrapper=True) def pytest_sessionstart(self, session: pytest.Session): self._config = session.config @@ -236,36 +262,57 @@ def pytest_runtestloop(self, session: pytest.Session) -> bool: if session.config.option.collectonly: return True + inprocess_items = [ + item + for item in session.items + if not isinstance(item, pytest.Function) or "robot" not in item.fixturenames + ] + inprocess_index = 0 running: list[IsolatedTestJob] = [] - deferred: list[pytest.Item] = [] try: - # Start any tests that use the robot fixture first. Tests that don't - # use the robot fixture will be ran later - for item in session.items: - if not ( - # pytest plugins may generate their own items. - isinstance(item, pytest.Function) - and "robot" in item.fixturenames - ): - deferred.append(item) - continue - - while len(running) >= self._parallelism: + # pytest-order has already sorted session.items. Preserve boundaries + # between ordering-marker groups while retaining the existing parallel + # scheduling within each group. + for group in self._ordering.groups(session.items): + deferred: list[pytest.Item] = [] + + # Start this group's robot tests first, then overlap its plain + # tests with the isolated jobs just as the original loop did. + for item in group: + if not ( + # pytest plugins may generate their own items. + isinstance(item, pytest.Function) + and "robot" in item.fixturenames + ): + deferred.append(item) + continue + + while len(running) >= self._parallelism: + self._wait_for_jobs(running, session) + + running.append(self._start_isolated_test(item)) + self._maybe_raise(session) + + for item in deferred: + # Observe completed isolated failures before starting more + # in-process work, so --maxfail can stop the group promptly. + if any(job.conn.poll() for job in running): + self._wait_for_jobs(running, session) + + nextitem = ( + inprocess_items[inprocess_index + 1] + if inprocess_index + 1 < len(inprocess_items) + else None + ) + inprocess_index += 1 + session.config.hook.pytest_runtest_protocol( + item=item, nextitem=nextitem + ) + self._maybe_raise(session) + + # No work from one marker group may cross into the next. + while running: self._wait_for_jobs(running, session) - - running.append(self._start_isolated_test(item)) - self._maybe_raise(session) - - # Run the in-process tests now while the robot tests are finishing - for idx, item in enumerate(deferred): - nextitem = deferred[idx + 1] if idx + 1 < len(deferred) else None - session.config.hook.pytest_runtest_protocol( - item=item, nextitem=nextitem - ) - self._maybe_raise(session) - - while running: - self._wait_for_jobs(running, session) finally: for job in running: self._cleanup_job(job) @@ -290,6 +337,7 @@ def _start_isolated_test(self, item: pytest.Function) -> IsolatedTestJob: pickle.dumps(self._robot_class), self._robot_file, self._verbose, + self._ordering.worker_state(item), cconn, self._config.rootpath, ), @@ -402,10 +450,10 @@ def _cleanup_job(self, job: IsolatedTestJob): job.process.close() def _maybe_raise(self, session: pytest.Session): - if self._shouldstop: - raise session.Interrupted(self._shouldstop) if session.shouldfail: raise session.Failed(session.shouldfail) + if self._shouldstop: + raise session.Interrupted(self._shouldstop) if session.shouldstop: raise session.Interrupted(session.shouldstop) @@ -458,7 +506,17 @@ def worker_finished(self, job: IsolatedTestJob, exit_code: object | None = None) if exit_code is not None: job.exit_code = int(exit_code) - job.worker_completed = True + if job.exit_code == pytest.ExitCode.INTERRUPTED and not self._shouldstop: + self._shouldstop = "interrupted in worker" + + # Normal test failures have already produced reports, and interruptions + # stop the parent session. Other exit codes need a synthetic failure + # report from _finalize_job. + job.worker_completed = job.exit_code in ( + pytest.ExitCode.OK, + pytest.ExitCode.TESTS_FAILED, + pytest.ExitCode.INTERRUPTED, + ) job.finished = True def _handlefailures(self, rep: pytest.TestReport):