From 39e6c812ce98af0e6ca211618c62e52fe7d37ff6 Mon Sep 17 00:00:00 2001 From: glocker Date: Mon, 16 Nov 2026 16:00:18 +0000 Subject: [PATCH 1/2] Add bus fault injection on the in-memory transport A FaultInjectingTransport decorator applies explicit drop, delay, corrupt, duplicate and reorder rules to CAN frames, on send or on receive, selected by port, addresses and the Nth matching frame or a seeded probability. Rules come from [bus_faults] in the setup or inject_bus_fault scenario steps. Delays are virtual-time events and every random choice derives from the seed, so the same seed gives a byte-identical trace. Every fault is reported in --trace, the text and JSON output, the run result and the HTML report. A telemetry payload the codec cannot read is now a reported dropped packet (reason=undecodable) instead of aborting the run. --- src/cubesat_testbed/config/__init__.py | 10 + src/cubesat_testbed/config/parser.py | 113 +++- src/cubesat_testbed/main.py | 26 + src/cubesat_testbed/report/__init__.py | 2 + src/cubesat_testbed/report/html_report.py | 55 +- src/cubesat_testbed/report/recorder.py | 16 +- src/cubesat_testbed/scenario/runner.py | 117 +++- src/cubesat_testbed/transport/__init__.py | 10 + src/cubesat_testbed/transport/faults.py | 613 +++++++++++++++++ src/cubesat_testbed/transport/tracing.py | 30 +- tests/test_bus_faults.py | 760 ++++++++++++++++++++++ tests/test_html_report.py | 2 +- 12 files changed, 1742 insertions(+), 12 deletions(-) create mode 100644 src/cubesat_testbed/transport/faults.py create mode 100644 tests/test_bus_faults.py diff --git a/src/cubesat_testbed/config/__init__.py b/src/cubesat_testbed/config/__init__.py index fb224fa..162e463 100644 --- a/src/cubesat_testbed/config/__init__.py +++ b/src/cubesat_testbed/config/__init__.py @@ -1,11 +1,16 @@ """Configuration and scenario schema parsing.""" from cubesat_testbed.config.parser import ( + BUS_FAULT_DEFAULT_HOLD_US, AssertionOperator, AssertStep, + BusFaultRule, + BusFaultsConfig, + BusFaultType, CommandMapping, ConfigParseError, FaultType, + InjectBusFaultStep, InjectFaultStep, InMemoryTransportConfig, KissTransportConfig, @@ -39,12 +44,17 @@ ) __all__ = [ + "BUS_FAULT_DEFAULT_HOLD_US", "AssertStep", "AssertionOperator", + "BusFaultRule", + "BusFaultType", + "BusFaultsConfig", "CommandMapping", "ConfigParseError", "FaultType", "InMemoryTransportConfig", + "InjectBusFaultStep", "InjectFaultStep", "KissTransportConfig", "ModuleType", diff --git a/src/cubesat_testbed/config/parser.py b/src/cubesat_testbed/config/parser.py index 930a4dc..5c4e801 100644 --- a/src/cubesat_testbed/config/parser.py +++ b/src/cubesat_testbed/config/parser.py @@ -504,6 +504,87 @@ def _validate_params(self) -> None: raise ValueError(f"invalid params for module_type {self.module_type!r}: {exc}") from exc +BusFaultType: TypeAlias = Literal["drop", "delay", "corrupt", "duplicate", "reorder"] + +BUS_FAULT_DEFAULT_HOLD_US = 1_000_000 +"""How long a ``reorder`` rule holds a frame back at most, in virtual microseconds.""" + + +class BusFaultRule(BaseModel): + """One explicit fault the in-memory bus applies to the frames passing through it. + + A rule says *which* frames (``direction`` plus optional ``port``/``source_port``/ + ``source``/``destination`` selectors), *when* (``nth``/``count``/``probability``) + and *what happens* (``type``). It is passive data: the bus applies it, nothing + here decides anything. ``port`` is the CSP destination port; every fragment of a + multi-frame packet is matched by the header of the packet it belongs to. + + ``nth`` is 1-based and counts the frames that matched this rule's selectors and + reached it; the rule fires on that frame and on the next ``count - 1`` (default + 1). Without ``nth`` it fires on every matching frame (at most ``count`` times if + given), or on each with ``probability`` drawn from the run's seeded generator. + """ + + model_config = _SCHEMA_MODEL_CONFIG + + type: BusFaultType + name: str | None = None + direction: Literal["tx", "rx"] = "tx" + port: CspPort | None = None + source_port: CspPort | None = None + source: CspAddress | None = None + destination: CspAddress | None = None + nth: int | None = Field(default=None, ge=1, strict=True) + count: int | None = Field(default=None, ge=1, strict=True) + probability: float | None = Field(default=None, gt=0.0, le=1.0) + delay: VirtualDuration | None = None + hold: VirtualDuration | None = None + byte: int | None = Field(default=None, ge=0, strict=True) + mask: int | None = Field(default=None, ge=1, le=255, strict=True) + + @field_validator("name") + @classmethod + def _validate_name(cls, value: str | None) -> str | None: + if value is None: + return None + return _validate_identifier("bus fault name", value) + + @model_validator(mode="after") + def _validate_rule(self) -> Self: + if self.nth is not None and self.probability is not None: + raise ValueError("a bus fault rule takes either nth or probability, not both") + if self.type == "delay": + if self.delay is None or self.delay <= 0: + raise ValueError("a delay bus fault needs a positive delay, for example '200ms'") + elif self.delay is not None: + raise ValueError(f"delay is only valid for a delay bus fault, not {self.type!r}") + if self.hold is not None: + if self.type != "reorder": + raise ValueError(f"hold is only valid for a reorder bus fault, not {self.type!r}") + if self.hold <= 0: + raise ValueError("hold must be positive") + if self.type != "corrupt" and (self.byte is not None or self.mask is not None): + raise ValueError( + f"byte and mask are only valid for a corrupt bus fault, not {self.type!r}" + ) + return self + + +class BusFaultsConfig(BaseModel): + """``[bus_faults]``: the seed and the rules in force from the start of the run. + + Every random choice (``probability``, the byte a ``corrupt`` rule picks when it + names none) is drawn from a generator seeded from ``seed`` and the rule's + position, never from wall-clock time, so the same seed reproduces the same run. + Scenario ``inject_bus_fault`` steps add rules after these ones. + """ + + model_config = _SCHEMA_MODEL_CONFIG + + seed: int = Field(default=0, ge=0, strict=True) + rules: tuple[BusFaultRule, ...] = () + + class TestbedConfig(BaseModel): """Validated static satellite/testbed setup loaded from TOML.""" @@ -511,12 +592,14 @@ class TestbedConfig(BaseModel): transport: TransportConfig nodes: dict[str, NodeConfig] = Field(min_length=1) + bus_faults: BusFaultsConfig | None = None @model_validator(mode="after") def _validate_cross_references(self) -> Self: self._validate_node_names() self._validate_unique_addresses() self._validate_transport_mode_combination() + self._validate_bus_faults_transport() self._validate_command_mappings() self._validate_telemetry_mappings() return self @@ -575,6 +658,14 @@ def _validate_transport_mode_combination(self) -> None: f"in-memory transport was configured for {joined_nodes}" ) + def _validate_bus_faults_transport(self) -> None: + if self.bus_faults is not None and not isinstance(self.transport, InMemoryTransportConfig): + raise ValueError( + "[bus_faults] requires transport.type='in-memory': the frames of a real " + "bus, serial line or ZeroMQ hub run on wall-clock time, so a fault there " + "could not be replayed from a seed" + ) + def _validate_command_mappings(self) -> None: for node_name, node in self.nodes.items(): for command_name, command in node.commands.items(): @@ -675,6 +766,17 @@ def _validate_fault_target_combination(self) -> Self: return self +class InjectBusFaultStep(BusFaultRule): + """Scenario step that adds a :class:`BusFaultRule` to the in-memory bus. + + The rule's own fields sit next to ``action`` (``type`` is the kind of bus fault, + as it is the kind of fault for ``inject_fault``). Frame counting for ``nth`` + starts when the step runs. + """ + + action: Literal["inject_bus_fault"] + + class SendCommandStep(BaseModel): """Scenario step that references a configured named command.""" @@ -725,7 +827,7 @@ def _validate_name(cls, value: str | None) -> str | None: ScenarioStep: TypeAlias = Annotated[ - WaitStep | InjectFaultStep | SendCommandStep | AssertStep, + WaitStep | InjectFaultStep | InjectBusFaultStep | SendCommandStep | AssertStep, Field(discriminator="action"), ] @@ -845,6 +947,12 @@ def validate_scenario_references(scenario: ScenarioScript, setup: TestbedConfig) prefix = f"step {index} ({step.action})" if isinstance(step, InjectFaultStep): _validate_fault_reference(step, configured_nodes, prefix, errors) + elif isinstance(step, InjectBusFaultStep): + if not isinstance(setup.transport, InMemoryTransportConfig): + errors.append( + f"{prefix}: bus faults require transport.type='in-memory', " + f"not {setup.transport.type!r}" + ) elif isinstance(step, SendCommandStep): _validate_command_reference(step, setup, configured_nodes, prefix, errors) elif isinstance(step, AssertStep) and step.signal not in telemetry_signals: @@ -1049,10 +1157,13 @@ def _read_text(path: str | Path) -> str: __all__ = [ "AssertStep", "AssertionOperator", + "BusFaultRule", + "BusFaultsConfig", "CommandMapping", "ConfigParseError", "FaultType", "InMemoryTransportConfig", + "InjectBusFaultStep", "InjectFaultStep", "ModuleType", "NodeConfig", diff --git a/src/cubesat_testbed/main.py b/src/cubesat_testbed/main.py index d385777..a1321d6 100644 --- a/src/cubesat_testbed/main.py +++ b/src/cubesat_testbed/main.py @@ -38,6 +38,7 @@ run_scenario_files, ) from cubesat_testbed.scenario.assertions import AssertionResult +from cubesat_testbed.transport import format_bus_fault _EXIT_SUCCESS = 0 _EXIT_ASSERTION_FAILURE = 1 @@ -276,6 +277,8 @@ def _run_command(args: argparse.Namespace) -> int: elif not args.quiet: for dropped in result.dropped_packets: print(format_dropped_packet(dropped)) + for fault in result.bus_faults: + print(format_bus_fault(fault)) _print_summary(result) return exit_code @@ -430,6 +433,11 @@ def _print_summary(result: ScenarioRunResult) -> None: f"assertions={total} passed={passed} failed={failed} " f"started_at={result.started_at} finished_at={result.finished_at}" + (f" dropped_packets={len(result.dropped_packets)}" if result.dropped_packets else "") + + ( + f" bus_faults={len(result.bus_faults)} bus_fault_seed={result.bus_fault_seed}" + if result.bus_fault_seed is not None + else "" + ) ) @@ -460,6 +468,24 @@ def _result_payload(result: ScenarioRunResult, *, exit_code: int) -> dict[str, o } for dropped in result.dropped_packets ], + "bus_fault_seed": result.bus_fault_seed, + "bus_faults": [ + { + "time": fault.time, + "rule": fault.rule, + "type": fault.type, + "direction": fault.direction, + "match": fault.match, + "can_id": fault.can_id, + "data": fault.data.hex(), + "source": fault.source, + "destination": fault.destination, + "destination_port": fault.destination_port, + "source_port": fault.source_port, + "detail": fault.detail, + } + for fault in result.bus_faults + ], } diff --git a/src/cubesat_testbed/report/__init__.py b/src/cubesat_testbed/report/__init__.py index 35b794a..900d076 100644 --- a/src/cubesat_testbed/report/__init__.py +++ b/src/cubesat_testbed/report/__init__.py @@ -4,6 +4,7 @@ from cubesat_testbed.report.recorder import ( RecordedAssertion, RecordedDrop, + RecordedFault, RecordedFrame, RecordedStep, RunRecorder, @@ -14,6 +15,7 @@ __all__ = [ "RecordedAssertion", "RecordedDrop", + "RecordedFault", "RecordedFrame", "RecordedStep", "RunRecorder", diff --git a/src/cubesat_testbed/report/html_report.py b/src/cubesat_testbed/report/html_report.py index 9dd46b3..97b25ed 100644 --- a/src/cubesat_testbed/report/html_report.py +++ b/src/cubesat_testbed/report/html_report.py @@ -20,6 +20,7 @@ from cubesat_testbed.report.recorder import ( RecordedAssertion, RecordedDrop, + RecordedFault, RecordedFrame, RecordedStep, RunRecorder, @@ -366,6 +367,27 @@ def _drop_row(entry: RecordedDrop, names: dict[int, str], number: int) -> str: ) +def _fault_row(entry: RecordedFault, names: dict[int, str], number: int) -> str: + event = entry.event + nodes = " ".join(str(a) for a in (event.source, event.destination) if a is not None) + ports = " ".join(str(a) for a in (event.destination_port, event.source_port) if a is not None) + route = "" + if event.source is not None and event.destination is not None: + route = f" {_node_label(event.source, names)} → {_node_label(event.destination, names)}" + ( + f" port {event.destination_port}" if event.destination_port is not None else "" + ) + detail = f" {event.detail}" if event.detail else "" + return ( + f'' + f"fault{_esc(format_virtual_time(event.time))}" + f"{_esc(event.direction)}" + f'FAULT {_esc(event.type)} ' + f"rule {_esc(event.rule)}, match {event.match}{_esc(route)}{_esc(detail)}, " + f'data {_esc(_hex(event.data))}' + ) + + def _step_row(entry: RecordedStep) -> str: return ( f' str: def _timeline_section(recorder: RunRecorder, names: dict[int, str]) -> str: rows: list[str] = [] drops = 0 + faults = 0 addresses: set[int] = set() ports: set[int] = set() for entry in recorder.timeline: @@ -395,6 +418,15 @@ def _timeline_section(recorder: RunRecorder, names: dict[int, str]) -> str: for address in (entry.sender, entry.destination): if address is not None: addresses.add(address) + elif isinstance(entry, RecordedFault): + faults += 1 + rows.append(_fault_row(entry, names, faults)) + for address in (entry.event.source, entry.event.destination): + if address is not None: + addresses.add(address) + for port in (entry.event.destination_port, entry.event.source_port): + if port is not None: + ports.add(port) else: rows.append(_step_row(entry)) node_options = "".join( @@ -409,6 +441,7 @@ def _timeline_section(recorder: RunRecorder, names: dict[int, str]) -> str: f'' '' '' + '' '' '' '' @@ -422,7 +455,7 @@ def _timeline_section(recorder: RunRecorder, names: dict[int, str]) -> str: return ( '

Timeline

' '

Every frame that crossed the transport boundary, in order, with the ' - "scenario steps and dropped packets. On the in-memory bus a frame appears twice, " + "scenario steps, dropped packets and injected bus faults. On the in-memory bus a frame appears twice, " "TX when sent and RX when read back: that round trip is what an assertion observes." f"

{controls}{table}
" ) @@ -472,6 +505,7 @@ def _timeline_section(recorder: RunRecorder, names: dict[int, str]) -> str: port=document.getElementById('f-port'),text=document.getElementById('f-text'), count=document.getElementById('f-count'); var kinds={frame:document.getElementById('k-frame'),drop:document.getElementById('k-drop'), +fault:document.getElementById('k-fault'), step:document.getElementById('k-step')}; if(!rows.length||!dir){return;} function has(list,value){return (' '+list+' ').indexOf(' '+value+' ')>=0;} @@ -483,7 +517,7 @@ def _timeline_section(recorder: RunRecorder, names: dict[int, str]) -> str: if(dir.value&&row.getAttribute('data-dir')!==dir.value){show=false;} if(show&&node.value&&!has(row.getAttribute('data-nodes'),node.value)){show=false;} if(show&&port.value&&!has(row.getAttribute('data-ports'),port.value)){show=false;} -}else if(show&&kind==='drop'&&node.value&&!has(row.getAttribute('data-nodes'),node.value)){show=false;} +}else if(show&&(kind==='drop'||kind==='fault')&&node.value&&!has(row.getAttribute('data-nodes'),node.value)){show=false;} if(show&&needle&&row.textContent.toLowerCase().indexOf(needle)<0){show=false;} row.hidden=!show;if(show){shown++;} }); @@ -534,7 +568,11 @@ def render_report( status, label = "fail", "FAIL" last_time = max( - [entry.time for entry in recorder.timeline if not isinstance(entry, RecordedFrame)] + [ + entry.event.time if isinstance(entry, RecordedFault) else entry.time + for entry in recorder.timeline + if not isinstance(entry, RecordedFrame) + ] + [ entry.record.time for entry in recorder.timeline @@ -549,10 +587,21 @@ def render_report( passed = sum(1 for a in recorder.assertions if a.result.passed) failed = len(recorder.assertions) - passed drops = sum(1 for entry in recorder.timeline if isinstance(entry, RecordedDrop)) + bus_faults = sum(1 for entry in recorder.timeline if isinstance(entry, RecordedFault)) summary = [ f"assertions: {passed} passed, {failed} failed", f"frames traced: {len(recorder.frames)}", f"dropped packets: {drops}", + *( + [ + ( + f"bus faults: {bus_faults} " + f"(seed {result.bus_fault_seed})" + ) + ] + if result is not None and result.bus_fault_seed is not None + else [] + ), ( f"virtual time: {_esc(format_virtual_time(start))} → " f"{_esc(format_virtual_time(end))}" diff --git a/src/cubesat_testbed/report/recorder.py b/src/cubesat_testbed/report/recorder.py index 562bda0..a0a04d2 100644 --- a/src/cubesat_testbed/report/recorder.py +++ b/src/cubesat_testbed/report/recorder.py @@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Any from cubesat_testbed.transport.base import TransportEnvelope +from cubesat_testbed.transport.faults import BusFaultEvent from cubesat_testbed.transport.tracing import FrameRecord if TYPE_CHECKING: # pragma: no cover - typing only, avoids a runtime import cycle @@ -43,6 +44,13 @@ class RecordedDrop: limit: int | None +@dataclass(frozen=True, slots=True) +class RecordedFault: + """A fault the bus injected, at its virtual time.""" + + event: BusFaultEvent + + @dataclass(frozen=True, slots=True) class RecordedStep: """A scenario step starting, so the timeline shows what the scenario was doing.""" @@ -67,7 +75,7 @@ class RecordedAssertion: evidence_frame: int | None -TimelineEntry = RecordedFrame | RecordedDrop | RecordedStep +TimelineEntry = RecordedFrame | RecordedDrop | RecordedFault | RecordedStep class RunRecorder: @@ -125,6 +133,9 @@ def drop(self, dropped: DroppedPacket) -> None: ) ) + def fault(self, event: BusFaultEvent) -> None: + self.timeline.append(RecordedFault(event=event)) + def signal(self, time: int, signal: str, value: object) -> None: self.signals.setdefault(signal, []).append((time, value)) @@ -132,7 +143,7 @@ def assertion(self, result: AssertionResult, evidence_frame: int | None) -> None self.assertions.append(RecordedAssertion(result=result, evidence_frame=evidence_frame)) -_DURATION_KEYS = frozenset({"virtual_time", "duration", "timeout"}) +_DURATION_KEYS = frozenset({"virtual_time", "duration", "timeout", "delay", "hold"}) def describe_step(step: Any) -> str: @@ -160,6 +171,7 @@ def format_virtual_time(microseconds: int) -> str: __all__ = [ "RecordedAssertion", "RecordedDrop", + "RecordedFault", "RecordedFrame", "RecordedStep", "RunRecorder", diff --git a/src/cubesat_testbed/scenario/runner.py b/src/cubesat_testbed/scenario/runner.py index 12e312e..7bf8fb2 100644 --- a/src/cubesat_testbed/scenario/runner.py +++ b/src/cubesat_testbed/scenario/runner.py @@ -9,7 +9,7 @@ import sys import warnings -from collections.abc import Iterable, Mapping +from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass, field from pathlib import Path from typing import Protocol, TextIO, runtime_checkable @@ -20,6 +20,7 @@ AssertionOperator, AssertStep, FaultType, + InjectBusFaultStep, InjectFaultStep, InMemoryTransportConfig, ScenarioScript, @@ -72,12 +73,18 @@ from cubesat_testbed.transport import ( EndpointId, FrameAnnotator, + InMemoryBusAdapter, PacketSender, TracingTransportAdapter, TransportAdapter, TransportEnvelope, build_transport_adapter, ) +from cubesat_testbed.transport.faults import ( + BusFaultEvent, + FaultInjectingTransport, + format_bus_fault_trace_line, +) from cubesat_testbed.transport.tracing import TRACE_PREFIX _PHYSICAL_STEP_US: VirtualTime = 1_000_000 @@ -195,6 +202,8 @@ class ScenarioRuntime: """Stream ``DROP`` lines are written to when ``--trace`` is on.""" recorder: RunRecorder | None = None """Observer for the HTML report; never read back by the run.""" + bus_faults: FaultInjectingTransport | None = None + """The bus fault injector, present on the in-memory bus (and only there).""" def module(self, name: str) -> SimulatedModule: """Return a configured simulated module by node name.""" @@ -232,6 +241,10 @@ class ScenarioRunResult: finished_at: VirtualTime assertions: tuple[AssertionResult, ...] dropped_packets: tuple[DroppedPacket, ...] = () + bus_faults: tuple[BusFaultEvent, ...] = () + """Every fault the bus injected, in order; the bus never fails silently.""" + bus_fault_seed: int | None = None + """The seed the injected faults derive from; ``None`` when no bus fault rule existed.""" @property def passed(self) -> bool: @@ -335,6 +348,9 @@ def __init__( self._dropped_packets: list[DroppedPacket] = [] self._signal_evidence: dict[str, int | None] = {} self.runtime.engine.add_handler(EventKind.TELEMETRY, self._record_telemetry) + if runtime.bus_faults is not None: + # A frame the bus held back is delivered in the instant it is released. + runtime.bus_faults.set_release_hook(self._run_current_events) self._arm_physical_step() @property @@ -349,6 +365,8 @@ def run(self, scenario: ScenarioScript) -> ScenarioRunResult: started_at = self.runtime.engine.now self._assertion_results = [] self._dropped_packets = [] + bus_faults = self.runtime.bus_faults + first_bus_fault = 0 if bus_faults is None else len(bus_faults.events) recorder = self.runtime.recorder if recorder is not None: recorder.begin(self.runtime.setup, scenario) @@ -360,6 +378,8 @@ def run(self, scenario: ScenarioScript) -> ScenarioRunResult: self.wait(step.virtual_time) elif isinstance(step, InjectFaultStep): self.inject_fault(step) + elif isinstance(step, InjectBusFaultStep): + self.inject_bus_fault(step) elif isinstance(step, SendCommandStep): self.send_command(step) elif isinstance(step, AssertStep): @@ -376,6 +396,10 @@ def run(self, scenario: ScenarioScript) -> ScenarioRunResult: finished_at=self.runtime.engine.now, assertions=tuple(self._assertion_results), dropped_packets=tuple(self._dropped_packets), + bus_faults=() if bus_faults is None else bus_faults.events[first_bus_fault:], + bus_fault_seed=( + bus_faults.seed if bus_faults is not None and bus_faults.has_rules else None + ), ) def wait(self, virtual_time: VirtualTime) -> None: @@ -455,6 +479,15 @@ def inject_fault(self, step: InjectFaultStep) -> None: self._run_current_events() self._emit_configured_telemetry() + def inject_bus_fault(self, step: InjectBusFaultStep) -> None: + """Put one bus fault rule in force on the in-memory bus, from this instant on.""" + + if self.runtime.bus_faults is None: + raise ScenarioRuntimeError( + "inject_bus_fault needs the in-memory bus; this runtime has no bus fault injector" + ) + self.runtime.bus_faults.add_rule(step) + def send_command(self, step: SendCommandStep) -> None: """Send one configured named command through the in-memory CSP path.""" @@ -714,6 +747,12 @@ def _deliver_whole_packet(self, packet: CspPacketFrame, envelope: TransportEnvel def _collect_transport_drops(self) -> None: """Record what the transport itself rejected (a serial link's bad frames).""" + if self.runtime.bus_faults is not None: + # A frame a bus fault dropped will never be delivered back, so whatever was + # kept for it (a command's semantic payload) would otherwise live forever. + for sequence in self.runtime.bus_faults.take_lost_sequences(): + self._semantic_payloads_by_envelope.pop(sequence, None) + for drop in self.runtime.transport.take_drops(): self._append_drop( DroppedPacket( @@ -828,7 +867,23 @@ def _deliver_command( ) def _deliver_telemetry(self, route: _TelemetryRoute, packet: CspPacketView) -> None: - value = decode_telemetry_value(route.field_name, route.mapping, packet.payload) + try: + value = decode_telemetry_value(route.field_name, route.mapping, packet.payload) + except TelemetryCodecError as exc: + # A payload the signal's codec cannot read (a value with no enum label, a + # corrupted byte) is a packet the receiver discards, reported like any other + # drop; it does not end the run, whose assertions then judge what the bus + # actually delivered. + self._append_drop( + DroppedPacket( + time=self.runtime.engine.now, + destination=packet.fields.destination, + sender=packet.fields.source, + reason="undecodable", + detail=f"{route.signal}: {exc}", + ) + ) + return self.runtime.engine.schedule_telemetry(route.signal, value, source=route.node_name) def _resolve_command_route(self, step: SendCommandStep) -> _CommandRoute: @@ -970,16 +1025,33 @@ def build_runtime( ), max_packet_bytes=max(node.max_packet_bytes for node in setup.nodes.values()), ) + bus_faults: FaultInjectingTransport | None = None + if isinstance(transport, InMemoryBusAdapter): + # Always present on the in-memory bus, and a plain pass-through until a rule is + # added, so a scenario can add one mid-run without the runtime knowing in advance. + bus_faults = FaultInjectingTransport( + transport, + now=lambda: engine.now, + schedule=_scheduler_for(engine), + seed=0 if setup.bus_faults is None else setup.bus_faults.seed, + ) + if setup.bus_faults is not None: + for fault_rule in setup.bus_faults.rules: + bus_faults.add_rule(fault_rule) + transport = bus_faults if trace is not None or recorder is not None: # Wrapping here, rather than tracing inside the runner, is what makes # frames an OBC Peer module sends straight to the transport visible too. - transport = TracingTransportAdapter( + tracing = TracingTransportAdapter( transport, stream=trace, now=lambda: engine.now, annotate=_build_frame_annotator(command_routes, telemetry_routes), observer=None if recorder is None else recorder.observe_frame, ) + transport = tracing + if bus_faults is not None: + bus_faults.set_event_handler(_bus_fault_reporter(tracing, trace, recorder)) # Both the construction order and the per-tick order come from the module # registry, so a module type this package has never heard of takes part in @@ -1037,9 +1109,48 @@ def build_runtime( clock=VirtualClock() if clock is None else clock, trace=trace, recorder=recorder, + bus_faults=bus_faults, ) +def _scheduler_for(engine: DiscreteEventEngine) -> Callable[[int, Callable[[], None]], None]: + """Let the bus fault injector run a callback after a virtual delay, as an engine event.""" + + def schedule(delay: int, action: Callable[[], None]) -> None: + engine.schedule_timer(delay, handler=lambda _event, _engine: action()) + + return schedule + + +def _bus_fault_reporter( + tracing: TracingTransportAdapter, + trace: TextIO | None, + recorder: RunRecorder | None, +) -> Callable[[BusFaultEvent], None]: + """Report each injected fault to the trace and the HTML recorder. + + A fault on the send side happens while the frame is still being sent, before the + tracing adapter has traced it, so it is deferred until the frame has been: the + trace reads ``TX`` first and ``FAULT`` after. A fault on the receive side has no + such line to wait for and is reported at once. + """ + + def report(event: BusFaultEvent) -> None: + def emit() -> None: + if trace is not None: + trace.write(format_bus_fault_trace_line(event) + "\n") + trace.flush() + if recorder is not None: + recorder.fault(event) + + if event.direction == "TX": + tracing.after_frame(emit) + else: + emit() + + return report + + def build_in_memory_runtime( setup: TestbedConfig, *, diff --git a/src/cubesat_testbed/transport/__init__.py b/src/cubesat_testbed/transport/__init__.py index ea41c92..90e60ed 100644 --- a/src/cubesat_testbed/transport/__init__.py +++ b/src/cubesat_testbed/transport/__init__.py @@ -13,6 +13,12 @@ WholePacketSender, ) from cubesat_testbed.transport.factory import build_transport_adapter +from cubesat_testbed.transport.faults import ( + BusFaultEvent, + FaultInjectingTransport, + format_bus_fault, + format_bus_fault_trace_line, +) from cubesat_testbed.transport.in_memory import InMemoryBusAdapter from cubesat_testbed.transport.kiss import KissAdapter from cubesat_testbed.transport.serial_line import SerialLine, TermiosSerialLine @@ -27,9 +33,11 @@ from cubesat_testbed.transport.zmq_link import MessageLink, PyzmqMessageLink __all__ = [ + "BusFaultEvent", "CanBusLike", "CanPacketSender", "EndpointId", + "FaultInjectingTransport", "Frame", "FrameAnnotator", "InMemoryBusAdapter", @@ -50,5 +58,7 @@ "WholePacketSender", "ZmqAdapter", "build_transport_adapter", + "format_bus_fault", + "format_bus_fault_trace_line", "format_trace_line", ] diff --git a/src/cubesat_testbed/transport/faults.py b/src/cubesat_testbed/transport/faults.py new file mode 100644 index 0000000..9c5e9f9 --- /dev/null +++ b/src/cubesat_testbed/transport/faults.py @@ -0,0 +1,613 @@ +"""Bus fault injection: a transport decorator that breaks the bus on request. + +:class:`FaultInjectingTransport` wraps the in-memory bus and applies explicit +:class:`~cubesat_testbed.config.BusFaultRule` requests to the CAN frames passing +through it: drop one, delay one by a virtual duration, flip a byte, duplicate it, +or deliver it after the frame that followed it. It is the bus-level counterpart of +the passive :class:`~cubesat_testbed.fault_injection.FaultInjectionEngine`, and +obeys the same rule: it decides nothing on its own, it applies the rules it was +given. + +Determinism is the point. Time comes from the engine (``now``/``schedule`` are +passed in, so a delay is a scheduled virtual-time event, never a sleep) and every +random choice -- a ``probability`` draw, the byte a ``corrupt`` rule picks when it +names none -- comes from one generator per rule, seeded from the run's seed and the +rule's position. The same seed, rules and scenario give the same events in the same +order, and the seed is part of the run's reported result. + +Where it sits. Stack it directly on the bus and put the tracing adapter *above* it, +so a trace shows a node's view of the wire: ``TX`` is a frame a node tried to send, +``RX`` is a frame that reached the receiver, and each :class:`BusFaultEvent` (a +``FAULT`` line) is what the bus did in between. A dropped frame therefore appears +as a ``TX`` with no ``RX`` and a ``FAULT drop`` after it, and is visible as dropped +rather than simply missing. + +The decorator owns envelope sequence numbers. A delayed frame is not on the bus yet +when ``send`` returns its envelope, so the sequence handed back is the decorator's +own and every envelope read back is translated to it; the runner keys a command's +semantic payload by that number and finds it again whatever happened to the frame. + +Scope, on purpose: CAN frames on the in-memory bus. Faults on the receive side act +on the default receive stream (the bus monitor, which is where the runner decodes +every frame); endpoint-specific streams pass through. A real bus, a serial line and a +ZeroMQ hub deliver on wall-clock time, so a delay or an ordering there could not be +reproduced from a seed; setup validation refuses ``[bus_faults]`` on them. +""" + +from __future__ import annotations + +import random +from collections import deque +from collections.abc import Callable +from dataclasses import dataclass, replace + +from cubesat_testbed.config import BUS_FAULT_DEFAULT_HOLD_US, BusFaultRule +from cubesat_testbed.protocol.csp_v2 import ( + CspCanFrame, + CspV2CodecError, + decode_can_id, + decode_first_frame_header, +) +from cubesat_testbed.transport.base import ( + EndpointId, + Frame, + PacketSender, + TransportAdapter, + TransportDrop, + TransportEnvelope, + TransportError, +) +from cubesat_testbed.transport.in_memory import InMemoryBusAdapter +from cubesat_testbed.transport.tracing import TRACE_PREFIX + +NowSource = Callable[[], int] +"""Current virtual time in microseconds.""" + +Scheduler = Callable[[int, Callable[[], None]], None] +"""Run a callback after a virtual delay, as an event on the engine's timeline.""" + +FaultEventHandler = Callable[["BusFaultEvent"], None] +"""Called with every injected fault as it happens; observability only.""" + +DEFAULT_CORRUPT_MASK = 0xFF + + +@dataclass(frozen=True, slots=True) +class BusFaultEvent: + """One fault the bus injected: which rule, what it did and to which frame. + + ``match`` is the 1-based count of frames that matched the rule's selectors and + reached it, the number ``nth`` refers to. ``data`` is the frame's CAN data field + as it was before the fault. The header fields are ``None`` where the frame does + not carry them (a continuation fragment whose first frame was never seen). + """ + + time: int + rule: str + type: str + direction: str + match: int + can_id: int + data: bytes + source: int | None = None + destination: int | None = None + destination_port: int | None = None + source_port: int | None = None + detail: str = "" + + +def format_bus_fault(event: BusFaultEvent) -> str: + """Format one injected fault as a ``FAULT t=... rule=... action=...`` report line.""" + + parts = [ + "FAULT", + f"t={event.time}", + f"rule={event.rule}", + f"action={event.type}", + f"dir={event.direction}", + f"match={event.match}", + f"can_id=0x{event.can_id:08X}", + ] + if event.source is not None: + parts.append(f"src={event.source}") + if event.destination is not None: + parts.append(f"dst={event.destination}") + if event.destination_port is not None: + parts.append(f"dport={event.destination_port}") + if event.source_port is not None: + parts.append(f"sport={event.source_port}") + parts.append(f"data={event.data.hex() if event.data else '-'}") + if event.detail: + parts.append(event.detail) + return " ".join(parts) + + +def format_bus_fault_trace_line(event: BusFaultEvent) -> str: + """Format one injected fault as a ``--trace`` line (``trace`` + the FAULT report line).""" + + return f"{TRACE_PREFIX} {format_bus_fault(event)}" + + +@dataclass(frozen=True, slots=True) +class _Header: + """What the rules can select on, as far as the frame (or its packet) tells.""" + + source: int | None = None + destination: int | None = None + destination_port: int | None = None + source_port: int | None = None + + +class _ActiveRule: + """A rule plus the counters and the generator it needs while it is in force.""" + + def __init__(self, spec: BusFaultRule, *, index: int, seed: int) -> None: + self.spec = spec + self.name = spec.name or f"{spec.type}-{index + 1}" + self.rng = random.Random(f"{seed}/{index}") + self.matched = 0 + self.fired = 0 + + def selects(self, direction: str, header: _Header) -> bool: + spec = self.spec + if spec.direction != direction: + return False + return all( + wanted is None or wanted == actual + for wanted, actual in ( + (spec.port, header.destination_port), + (spec.source_port, header.source_port), + (spec.source, header.source), + (spec.destination, header.destination), + ) + ) + + def consider(self) -> int | None: + """Count one matching frame; its match number if the rule fires on it.""" + + spec = self.spec + self.matched += 1 + if spec.nth is not None: + fires = spec.nth <= self.matched < spec.nth + (spec.count or 1) + else: + fires = spec.count is None or self.fired < spec.count + if fires and spec.probability is not None: + fires = self.rng.random() < spec.probability + if not fires: + return None + self.fired += 1 + return self.matched + + +@dataclass(slots=True) +class _HeldTx: + frame: Frame + source: EndpointId | None + sequence: int + + +@dataclass(slots=True) +class _HeldRx: + envelope: TransportEnvelope + + +class FaultInjectingTransport(TransportAdapter): + """Applies bus fault rules to the CAN frames crossing an inner adapter. + + ``now`` and ``schedule`` tie it to virtual time (see :data:`NowSource`, + :data:`Scheduler`). ``seed`` seeds every rule's generator. ``on_event`` is told + of each injected fault. Rules are added with :meth:`add_rule`, from the setup + config before the run and from ``inject_bus_fault`` steps during it; a frame is + affected by at most one rule, tried in the order they were added, and a rule + counts only the frames that reached it. + """ + + def __init__( + self, + inner: InMemoryBusAdapter, + *, + now: NowSource, + schedule: Scheduler, + seed: int = 0, + on_event: FaultEventHandler | None = None, + ) -> None: + self._inner = inner + self._now = now + self._schedule = schedule + self._seed = seed + self._on_event = on_event + self._release_hook: Callable[[], None] | None = None + self._rules: list[_ActiveRule] = [] + self._events: list[BusFaultEvent] = [] + self._next_sequence = 0 + self._sequence_by_inner: dict[int, int] = {} + self._lost_sequences: list[int] = [] + self._packet_headers: dict[str, dict[tuple[int, int, int, int], _Header]] = { + "tx": {}, + "rx": {}, + } + self._held_tx: list[_HeldTx] = [] + self._held_rx: list[_HeldRx] = [] + self._rx_ready: deque[TransportEnvelope] = deque() + + # --- configuration ---------------------------------------------------------------- + + @property + def inner(self) -> InMemoryBusAdapter: + """The wrapped adapter that actually carries the frames.""" + + return self._inner + + @property + def seed(self) -> int: + """The seed every random choice of this run derives from.""" + + return self._seed + + @property + def has_rules(self) -> bool: + """Whether any rule was ever added, so a result can say a seed mattered.""" + + return bool(self._rules) + + def add_rule(self, rule: BusFaultRule) -> str: + """Put ``rule`` in force from now on; returns the name it is reported under.""" + + active = _ActiveRule(rule, index=len(self._rules), seed=self._seed) + self._rules.append(active) + return active.name + + def set_event_handler(self, handler: FaultEventHandler | None) -> None: + """Replace the observer told of each injected fault.""" + + self._on_event = handler + + def set_release_hook(self, hook: Callable[[], None] | None) -> None: + """Set what runs right after a held frame is released at its virtual time. + + The scenario runner uses it to deliver the released frame in the same + instant instead of at the next time it happens to drain the bus. + """ + + self._release_hook = hook + + @property + def events(self) -> tuple[BusFaultEvent, ...]: + """Every fault injected so far, in order.""" + + return tuple(self._events) + + def take_lost_sequences(self) -> tuple[int, ...]: + """Sequences of frames a fault dropped, and forget them. + + A frame that never arrives cannot release whatever the runner keyed by its + sequence, so the runner asks which ones to let go of. + """ + + lost = tuple(self._lost_sequences) + self._lost_sequences.clear() + return lost + + # --- the in-memory bus API, which this decorator stands in for --------------------- + + @property + def endpoints(self) -> tuple[EndpointId, ...]: + """Connected endpoint identifiers of the bus underneath.""" + + return self._inner.endpoints + + def connect(self, endpoint: EndpointId) -> None: + """Create an endpoint-specific receive queue on the bus underneath.""" + + self._inner.connect(endpoint) + + def connect_endpoint(self, endpoint: EndpointId) -> None: + self._inner.connect_endpoint(endpoint) + + def disconnect(self, endpoint: EndpointId) -> None: + self._inner.disconnect(endpoint) + + def disconnect_endpoint(self, endpoint: EndpointId) -> None: + self._inner.disconnect_endpoint(endpoint) + + def pending_count(self, *, endpoint: EndpointId | None = None) -> int: + """Frames currently queued for a stream, including ones faults already released.""" + + pending = self._inner.pending_count(endpoint=endpoint) + return pending + len(self._rx_ready) if endpoint is None else pending + + def clear(self, *, endpoint: EndpointId | None = None) -> None: + """Drop currently queued frames from one stream.""" + + self._inner.clear(endpoint=endpoint) + if endpoint is None: + self._rx_ready.clear() + + # --- adapter surface --------------------------------------------------------------- + + def new_sender(self, *, max_packet_bytes: int = 256) -> PacketSender: + return self._inner.new_sender(max_packet_bytes=max_packet_bytes) + + def payload_limit(self, max_packet_bytes: int) -> int: + return self._inner.payload_limit(max_packet_bytes) + + def take_drops(self) -> tuple[TransportDrop, ...]: + return self._inner.take_drops() + + def close(self) -> None: + self._inner.close() + + def send( + self, + frame: Frame, + *, + source: EndpointId | None = None, + ) -> TransportEnvelope: + """Send ``frame`` as a node would, subject to the rules in force.""" + + can_frame = _require_can(frame) + sequence = self._take_sequence() + envelope = TransportEnvelope(frame=frame, sequence=sequence, source=source) + if not self._rules: + self._transmit(frame, source, sequence) + return envelope + header = self._header_of(can_frame, "tx") + rule, match = self._firing_rule("tx", header) + + if rule is None or match is None: + self._transmit(frame, source, sequence) + self._flush_held_tx() + return envelope + + spec = rule.spec + if spec.type == "drop": + self._lost_sequences.append(sequence) + self._emit(rule, match, "tx", can_frame, header, "") + elif spec.type == "delay": + assert spec.delay is not None + self._emit(rule, match, "tx", can_frame, header, f"until={self._now() + spec.delay}") + self._schedule(spec.delay, lambda: self._release_tx(frame, source, sequence)) + elif spec.type == "corrupt": + corrupted, detail = self._corrupt(rule, can_frame) + self._emit(rule, match, "tx", can_frame, header, detail) + self._transmit(corrupted, source, sequence) + self._flush_held_tx() + elif spec.type == "duplicate": + self._emit(rule, match, "tx", can_frame, header, "copies=2") + self._transmit(frame, source, sequence) + self._transmit(frame, source, sequence) + self._flush_held_tx() + else: # reorder + hold = spec.hold or BUS_FAULT_DEFAULT_HOLD_US + self._emit(rule, match, "tx", can_frame, header, f"hold={hold}") + held = _HeldTx(frame=frame, source=source, sequence=sequence) + self._held_tx.append(held) + self._schedule(hold, lambda: self._expire_held_tx(held)) + return envelope + + def receive( + self, + *, + endpoint: EndpointId | None = None, + timeout: float | None = None, + ) -> TransportEnvelope | None: + """Receive the next frame that survives the rules in force, or ``None``.""" + + if endpoint is not None: + inner = self._inner.receive(endpoint=endpoint, timeout=timeout) + return None if inner is None else self._translate(inner) + + while True: + if self._rx_ready: + return self._rx_ready.popleft() + inner = self._inner.receive(endpoint=None, timeout=timeout) + if inner is None: + return None + self._receive_one(self._translate(inner)) + + # --- the transmit side ------------------------------------------------------------- + + def _transmit(self, frame: Frame, source: EndpointId | None, sequence: int) -> None: + envelope = self._inner.send(frame, source=source) + self._sequence_by_inner[envelope.sequence] = sequence + + def _release_tx(self, frame: Frame, source: EndpointId | None, sequence: int) -> None: + self._transmit(frame, source, sequence) + self._flush_held_tx() + self._after_release() + + def _flush_held_tx(self) -> None: + held, self._held_tx = self._held_tx, [] + for entry in held: + self._transmit(entry.frame, entry.source, entry.sequence) + + def _expire_held_tx(self, entry: _HeldTx) -> None: + if entry not in self._held_tx: + return # already released behind a later frame + self._held_tx.remove(entry) + self._transmit(entry.frame, entry.source, entry.sequence) + self._after_release() + + # --- the receive side -------------------------------------------------------------- + + def _receive_one(self, envelope: TransportEnvelope) -> None: + """Apply the rules to one frame read off the bus; queue what is delivered.""" + + can_frame = _require_can(envelope.frame) + if not self._rules: + self._rx_ready.append(envelope) + return + header = self._header_of(can_frame, "rx") + rule, match = self._firing_rule("rx", header) + + if rule is None or match is None: + self._rx_ready.append(envelope) + self._flush_held_rx() + return + + spec = rule.spec + if spec.type == "drop": + self._lost_sequences.append(envelope.sequence) + self._emit(rule, match, "rx", can_frame, header, "") + elif spec.type == "delay": + assert spec.delay is not None + self._emit(rule, match, "rx", can_frame, header, f"until={self._now() + spec.delay}") + self._schedule(spec.delay, lambda: self._release_rx(envelope)) + elif spec.type == "corrupt": + corrupted, detail = self._corrupt(rule, can_frame) + self._emit(rule, match, "rx", can_frame, header, detail) + self._rx_ready.append(replace(envelope, frame=corrupted)) + self._flush_held_rx() + elif spec.type == "duplicate": + self._emit(rule, match, "rx", can_frame, header, "copies=2") + self._rx_ready.append(envelope) + self._rx_ready.append(envelope) + self._flush_held_rx() + else: # reorder + hold = spec.hold or BUS_FAULT_DEFAULT_HOLD_US + self._emit(rule, match, "rx", can_frame, header, f"hold={hold}") + held = _HeldRx(envelope=envelope) + self._held_rx.append(held) + self._schedule(hold, lambda: self._expire_held_rx(held)) + + def _release_rx(self, envelope: TransportEnvelope) -> None: + self._rx_ready.append(envelope) + self._after_release() + + def _flush_held_rx(self) -> None: + held, self._held_rx = self._held_rx, [] + self._rx_ready.extend(entry.envelope for entry in held) + + def _expire_held_rx(self, entry: _HeldRx) -> None: + if entry not in self._held_rx: + return + self._held_rx.remove(entry) + self._rx_ready.append(entry.envelope) + self._after_release() + + # --- shared ------------------------------------------------------------------------ + + def _after_release(self) -> None: + if self._release_hook is not None: + self._release_hook() + + def _take_sequence(self) -> int: + sequence = self._next_sequence + self._next_sequence += 1 + return sequence + + def _translate(self, envelope: TransportEnvelope) -> TransportEnvelope: + sequence = self._sequence_by_inner.get(envelope.sequence) + if sequence is None or sequence == envelope.sequence: + return envelope + return replace(envelope, sequence=sequence) + + def _firing_rule( + self, direction: str, header: _Header + ) -> tuple[_ActiveRule | None, int | None]: + for rule in self._rules: + if not rule.selects(direction, header): + continue + match = rule.consider() + if match is not None: + return rule, match + return None, None + + def _header_of(self, frame: CspCanFrame, direction: str) -> _Header: + """Selectable header of a frame, taking a fragment's from its first frame. + + Only the first frame of a multi-frame packet carries ports and the source + address; the others carry the sender and destination. The headers seen are + remembered, per direction and keyed the way a receiver matches fragments + (priority, destination, sender, packet counter), so a rule on a port reaches + every fragment of the packet, and a rule can single one out by ``nth``. + """ + + if not frame.is_extended_id: + return _Header() + can_fields = decode_can_id(frame.can_id) + key = ( + can_fields.priority, + can_fields.destination, + can_fields.sender, + can_fields.packet_count, + ) + known = self._packet_headers[direction] + if can_fields.begin: + try: + first = decode_first_frame_header(frame) + except CspV2CodecError: + return _Header(source=can_fields.sender, destination=can_fields.destination) + header = _Header( + source=first.source, + destination=first.destination, + destination_port=first.destination_port, + source_port=first.source_port, + ) + if can_fields.end: + known.pop(key, None) + else: + known[key] = header + return header + remembered = known.get(key) + if can_fields.end: + known.pop(key, None) + if remembered is None: + return _Header(source=can_fields.sender, destination=can_fields.destination) + return remembered + + def _corrupt(self, rule: _ActiveRule, frame: CspCanFrame) -> tuple[CspCanFrame, str]: + spec = rule.spec + data = bytearray(frame.data) + if not data: + return frame, "byte=- (empty frame)" + index = spec.byte if spec.byte is not None else int(rule.rng.random() * len(data)) + if index >= len(data): + # Reported, not raised: the rule fired and said so, the frame was too short. + return frame, f"byte={index} skipped: the frame has {len(data)} data bytes" + mask = spec.mask if spec.mask is not None else DEFAULT_CORRUPT_MASK + before = data[index] + data[index] = before ^ mask + corrupted = replace(frame, data=bytes(data)) + return corrupted, f"byte={index} mask=0x{mask:02X} {before:02x}->{data[index]:02x}" + + def _emit( + self, + rule: _ActiveRule, + match: int, + direction: str, + frame: CspCanFrame, + header: _Header, + detail: str, + ) -> None: + event = BusFaultEvent( + time=self._now(), + rule=rule.name, + type=rule.spec.type, + direction=direction.upper(), + match=match, + can_id=frame.can_id, + data=frame.data, + source=header.source, + destination=header.destination, + destination_port=header.destination_port, + source_port=header.source_port, + detail=detail, + ) + self._events.append(event) + if self._on_event is not None: + self._on_event(event) + + +def _require_can(frame: Frame) -> CspCanFrame: + if not isinstance(frame, CspCanFrame): + raise TransportError("bus faults apply to CAN frames; this link carries whole packets") + return frame + + +__all__ = [ + "BusFaultEvent", + "FaultInjectingTransport", + "format_bus_fault", + "format_bus_fault_trace_line", +] diff --git a/src/cubesat_testbed/transport/tracing.py b/src/cubesat_testbed/transport/tracing.py index 533ae00..e3a6fe3 100644 --- a/src/cubesat_testbed/transport/tracing.py +++ b/src/cubesat_testbed/transport/tracing.py @@ -157,6 +157,8 @@ def __init__( "TX": CspReassembler(max_packet_bytes=None), "RX": CspReassembler(max_packet_bytes=None), } + self._sending = False + self._after_send: list[Callable[[], None]] = [] @property def inner(self) -> TransportAdapter: @@ -177,8 +179,13 @@ def send( bus. """ - envelope = self._inner.send(frame, source=source) - self._trace("TX", envelope) + self._sending = True + try: + envelope = self._inner.send(frame, source=source) + self._trace("TX", envelope) + finally: + self._sending = False + self._run_after_send() return envelope def receive( @@ -194,6 +201,20 @@ def receive( self._trace("RX", envelope) return envelope + def after_frame(self, action: Callable[[], None]) -> None: + """Run ``action`` once the frame being sent has been traced, or now if none is. + + What a wrapped adapter reports *about* a frame while it is being sent (the + bus fault injector's ``FAULT`` lines) happens before this adapter has traced + that frame; deferring it keeps the trace in reading order: the frame first, + then what was done to it. + """ + + if self._sending: + self._after_send.append(action) + else: + action() + def new_sender(self, *, max_packet_bytes: int = 256) -> PacketSender: """Ask the wrapped adapter: how a packet becomes frames depends on the link.""" @@ -214,6 +235,11 @@ def close(self) -> None: self._inner.close() + def _run_after_send(self) -> None: + actions, self._after_send = self._after_send, [] + for action in actions: + action() + def _trace(self, direction: str, envelope: TransportEnvelope) -> None: record = describe_frame( direction, diff --git a/tests/test_bus_faults.py b/tests/test_bus_faults.py new file mode 100644 index 0000000..c0d2601 --- /dev/null +++ b/tests/test_bus_faults.py @@ -0,0 +1,760 @@ +"""Bus fault injection: rules on the in-memory bus, deterministic and visible.""" + +from __future__ import annotations + +import io +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from cubesat_testbed.config import ( + BusFaultRule, + ScenarioReferenceError, + parse_scenario, + parse_testbed_config, +) +from cubesat_testbed.main import main +from cubesat_testbed.protocol.csp_v2 import ( + CspCanFrame, + CspFields, + decode_frame, + fragment_packet, +) +from cubesat_testbed.report import RunRecorder, render_report +from cubesat_testbed.scenario import run_scenario +from cubesat_testbed.transport import ( + BusFaultEvent, + FaultInjectingTransport, + InMemoryBusAdapter, + format_bus_fault, +) + +# --- a hand-driven virtual clock for the transport on its own --------------------------- + + +class _Clock: + """Virtual time and the callbacks scheduled on it, run by hand.""" + + def __init__(self) -> None: + self.now = 0 + self._pending: list[tuple[int, int, object]] = [] + self._counter = 0 + + def schedule(self, delay: int, action: object) -> None: + self._pending.append((self.now + delay, self._counter, action)) + self._counter += 1 + + def advance(self, to: int) -> None: + while True: + due = sorted(item for item in self._pending if item[0] <= to) + if not due: + break + item = due[0] + self._pending.remove(item) + self.now = item[0] + item[2]() # type: ignore[operator] + self.now = to + + +def _bus( + *rules: dict[str, object], seed: int = 0 +) -> tuple[FaultInjectingTransport, _Clock, InMemoryBusAdapter]: + clock = _Clock() + inner = InMemoryBusAdapter() + bus = FaultInjectingTransport(inner, now=lambda: clock.now, schedule=clock.schedule, seed=seed) + for rule in rules: + bus.add_rule(BusFaultRule.model_validate(rule)) + return bus, clock, inner + + +def _frames( + port: int, payload: bytes = b"\x01", *, source: int = 1, count: int = 0 +) -> tuple[CspCanFrame, ...]: + fields = CspFields( + priority=2, + source=source, + destination=2, + destination_port=port, + source_port=5, + ) + return fragment_packet(fields, payload, packet_count=count) + + +def _send(bus: FaultInjectingTransport, port: int, payload: bytes = b"\x01") -> int: + (frame,) = _frames(port, payload) + return bus.send(frame, source=1).sequence + + +def _ports_received(bus: FaultInjectingTransport) -> list[int]: + return [decode_frame(env.frame).fields.destination_port for env in bus.drain()] # type: ignore[arg-type] + + +# --- the schema --------------------------------------------------------------------------- + + +def test_a_rule_needs_exactly_the_parameters_of_its_type() -> None: + with pytest.raises(ValidationError, match="positive delay"): + BusFaultRule(type="delay") + with pytest.raises(ValidationError, match="only valid for a delay"): + BusFaultRule(type="drop", delay="1s") # type: ignore[arg-type] + with pytest.raises(ValidationError, match="only valid for a reorder"): + BusFaultRule(type="drop", hold="1s") # type: ignore[arg-type] + with pytest.raises(ValidationError, match="only valid for a corrupt"): + BusFaultRule(type="drop", byte=1) + with pytest.raises(ValidationError, match="either nth or probability"): + BusFaultRule(type="drop", nth=1, probability=0.5) + with pytest.raises(ValidationError): + BusFaultRule(type="explode") # type: ignore[arg-type] + with pytest.raises(ValidationError): + BusFaultRule(type="drop", nth=0) + with pytest.raises(ValidationError): + BusFaultRule(type="drop", port=64) + + +def test_bus_faults_are_refused_on_a_transport_that_runs_on_wall_clock_time() -> None: + text = """ +[transport] +type = "kiss" +device = "/dev/null" + +[bus_faults] +seed = 1 + +[nodes.a] +mode = "software" +address = 1 +""" + with pytest.raises(ValidationError, match="requires transport.type='in-memory'"): + parse_testbed_config(text) + + +def test_an_inject_bus_fault_step_is_refused_on_a_real_transport() -> None: + setup = parse_testbed_config( + """ +[transport] +type = "socketcan" +interface = "vcan0" + +[nodes.a] +mode = "software" +address = 1 +""" + ) + scenario = parse_scenario( + "name: s\nsteps:\n - action: inject_bus_fault\n type: drop\n port: 3\n" + ) + from cubesat_testbed.config import validate_scenario_references + + with pytest.raises(ScenarioReferenceError, match="in-memory"): + validate_scenario_references(scenario, setup) + + +# --- drop, by the Nth frame on a port -------------------------------------------------------- + + +def test_drop_the_nth_frame_on_a_port_and_only_that_one() -> None: + bus, _clock, _inner = _bus({"type": "drop", "port": 7, "nth": 2}) + + for port in (7, 9, 7, 9, 7): + _send(bus, port) + + # Frames to port 9 are not counted, the second frame to port 7 is lost. + assert _ports_received(bus) == [7, 9, 9, 7] + (event,) = bus.events + assert (event.type, event.direction, event.match, event.destination_port) == ( + "drop", + "TX", + 2, + 7, + ) + + +def test_nth_with_count_drops_consecutive_frames() -> None: + bus, _clock, _inner = _bus({"type": "drop", "port": 7, "nth": 2, "count": 2}) + + for _ in range(5): + _send(bus, 7) + + assert len(_ports_received(bus)) == 3 + assert [event.match for event in bus.events] == [2, 3] + + +def test_a_rule_without_nth_fires_on_every_matching_frame_up_to_its_count() -> None: + bus, _clock, _inner = _bus({"type": "drop", "port": 7, "count": 2}) + + for _ in range(4): + _send(bus, 7) + + assert len(_ports_received(bus)) == 2 + assert len(bus.events) == 2 + + +def test_selectors_match_source_destination_and_source_port() -> None: + bus, _clock, _inner = _bus( + {"type": "drop", "source": 9}, + {"type": "drop", "destination": 4}, + {"type": "drop", "source_port": 6}, + ) + other = CspFields(priority=2, source=1, destination=2, destination_port=3, source_port=5) + from_nine = CspFields(priority=2, source=9, destination=2, destination_port=3, source_port=5) + to_four = CspFields(priority=2, source=1, destination=4, destination_port=3, source_port=5) + sport_six = CspFields(priority=2, source=1, destination=2, destination_port=3, source_port=6) + + for fields in (other, from_nine, to_four, sport_six): + (frame,) = fragment_packet(fields, b"\x01") + bus.send(frame, source=1) + + delivered = [decode_frame(env.frame).fields for env in bus.drain()] # type: ignore[arg-type] + assert delivered == [other] + assert [event.rule for event in bus.events] == ["drop-1", "drop-2", "drop-3"] + + +def test_a_port_rule_reaches_every_fragment_of_a_multi_frame_packet() -> None: + bus, _clock, _inner = _bus({"type": "drop", "port": 7, "nth": 2}) + packet = _frames(7, bytes(range(20))) # first frame + two continuation frames + assert len(packet) == 3 + + for frame in packet: + bus.send(frame, source=1) + + # The second frame on port 7 is the first continuation fragment, which carries no + # port of its own: it is matched by the header of the packet it belongs to. + assert len(list(bus.drain())) == 2 + (event,) = bus.events + assert event.destination_port == 7 + assert event.match == 2 + + +def test_the_first_matching_rule_wins_and_later_rules_count_only_what_reaches_them() -> None: + bus, _clock, _inner = _bus( + {"type": "drop", "port": 7, "nth": 1}, + {"type": "duplicate", "port": 7, "nth": 1}, + ) + + for _ in range(3): + _send(bus, 7) + + # Frame 1 is dropped by rule one. Rule two counts frames 2 and 3 as its 1st and 2nd, + # so it duplicates frame 2. + assert _ports_received(bus) == [7, 7, 7] + assert [(e.rule, e.match) for e in bus.events] == [("drop-1", 1), ("duplicate-2", 1)] + + +# --- delay, in virtual time ------------------------------------------------------------------ + + +def test_delay_holds_a_frame_for_a_virtual_duration() -> None: + bus, clock, _inner = _bus({"type": "delay", "port": 7, "nth": 1, "delay": "200ms"}) + released: list[int] = [] + bus.set_release_hook(lambda: released.append(clock.now)) + + _send(bus, 7) + _send(bus, 9) + + assert _ports_received(bus) == [9] # the frame to port 7 is still in flight + assert bus.events[0].detail == "until=200000" + + clock.advance(199_999) + assert _ports_received(bus) == [] + clock.advance(200_000) + assert _ports_received(bus) == [7] + assert released == [200_000] + + +def test_a_delayed_frame_keeps_the_sequence_its_sender_was_given() -> None: + bus, clock, _inner = _bus({"type": "delay", "nth": 1, "delay": "1s"}) + + delayed = _send(bus, 7) + later = _send(bus, 9) + clock.advance(1_000_000) + + # The sender was handed `delayed` before the frame reached the bus; the same number + # comes back when it arrives, behind the frame that was sent after it. + assert [env.sequence for env in bus.drain()] == [later, delayed] + + +# --- corrupt --------------------------------------------------------------------------------- + + +def test_corrupt_flips_the_named_byte() -> None: + bus, _clock, _inner = _bus({"type": "corrupt", "nth": 1, "byte": 4, "mask": 0x0F}) + (frame,) = _frames(7, b"\xa0") + + bus.send(frame, source=1) + + (envelope,) = bus.drain() + assert envelope.frame.data[:4] == frame.data[:4] # type: ignore[union-attr] + assert envelope.frame.data[4] == 0xAF # type: ignore[union-attr] + assert bus.events[0].detail == "byte=4 mask=0x0F a0->af" + assert bus.events[0].data == frame.data # the event keeps the frame as it was sent + + +def test_corrupt_without_a_byte_picks_one_from_the_seed() -> None: + def corrupted_byte(seed: int) -> int: + bus, _clock, _inner = _bus({"type": "corrupt"}, seed=seed) + (frame,) = _frames(7, b"\x01\x02\x03\x04") + bus.send(frame, source=1) + (envelope,) = bus.drain() + return next( + index + for index, (a, b) in enumerate(zip(frame.data, envelope.frame.data, strict=True)) # type: ignore[union-attr] + if a != b + ) + + assert corrupted_byte(1) == corrupted_byte(1) + assert len({corrupted_byte(seed) for seed in range(20)}) > 1 + + +def test_corrupting_a_byte_a_frame_does_not_have_is_reported_not_raised() -> None: + bus, _clock, _inner = _bus({"type": "corrupt", "nth": 1, "byte": 7}) + (frame,) = _frames(7, b"\x01") + + bus.send(frame, source=1) + + (envelope,) = bus.drain() + assert envelope.frame == frame + assert "skipped" in bus.events[0].detail + + +# --- duplicate and reorder ------------------------------------------------------------------- + + +def test_duplicate_delivers_the_frame_twice_under_one_sequence() -> None: + bus, _clock, _inner = _bus({"type": "duplicate", "port": 7, "nth": 1}) + + sequence = _send(bus, 7) + _send(bus, 9) + + envelopes = list(bus.drain()) + assert [decode_frame(e.frame).fields.destination_port for e in envelopes] == [7, 7, 9] # type: ignore[arg-type] + assert envelopes[0].sequence == envelopes[1].sequence == sequence + + +def test_reorder_delivers_the_frame_after_the_next_one() -> None: + bus, _clock, _inner = _bus({"type": "reorder", "port": 7, "nth": 1}) + + _send(bus, 7) + assert _ports_received(bus) == [] # held back + _send(bus, 9) + _send(bus, 8) + + assert _ports_received(bus) == [9, 7, 8] + + +def test_a_reordered_frame_is_released_after_its_hold_if_nothing_follows() -> None: + bus, clock, _inner = _bus({"type": "reorder", "nth": 1, "hold": "500ms"}) + + _send(bus, 7) + clock.advance(499_999) + assert _ports_received(bus) == [] + clock.advance(500_000) + assert _ports_received(bus) == [7] + + +# --- the receive side ------------------------------------------------------------------------ + + +def test_rx_drop_loses_a_frame_that_was_on_the_bus() -> None: + bus, _clock, inner = _bus({"type": "drop", "direction": "rx", "port": 7, "nth": 1}) + + _send(bus, 7) + _send(bus, 7) + + assert inner.pending_count() == 2 # both frames crossed the bus + assert len(list(bus.drain())) == 1 + assert bus.events[0].direction == "RX" + + +def test_rx_delay_duplicate_corrupt_and_reorder() -> None: + bus, clock, _inner = _bus( + {"type": "delay", "direction": "rx", "port": 1, "nth": 1, "delay": "1s"}, + {"type": "duplicate", "direction": "rx", "port": 2, "nth": 1}, + {"type": "corrupt", "direction": "rx", "port": 3, "nth": 1, "byte": 4, "mask": 0x10}, + {"type": "reorder", "direction": "rx", "port": 4, "nth": 1}, + ) + released: list[int] = [] + bus.set_release_hook(lambda: released.append(clock.now)) + + for port in (1, 2, 3, 4, 5): + _send(bus, port, bytes([port])) + + received = list(bus.drain()) + # Port 1 is delayed, 2 comes twice, 3 comes with a flipped byte, 4 is held back + # until 5 has gone by. + assert [e.frame.data[4] for e in received] == [2, 2, 3 ^ 0x10, 5, 4] # type: ignore[union-attr] + + clock.advance(1_000_000) + (late,) = bus.drain() + assert decode_frame(late.frame).fields.destination_port == 1 # type: ignore[arg-type] + assert released == [1_000_000] + + +def test_endpoint_streams_pass_through_untouched() -> None: + clock = _Clock() + inner = InMemoryBusAdapter(endpoints=(3,)) + bus = FaultInjectingTransport(inner, now=lambda: clock.now, schedule=clock.schedule) + bus.add_rule(BusFaultRule(type="drop", direction="rx")) + + _send(bus, 7) + + assert bus.pending_count(endpoint=3) == 1 + assert bus.receive(endpoint=3) is not None + assert bus.events == () + + +# --- determinism ------------------------------------------------------------------------------- + + +def _probabilistic_run(seed: int) -> tuple[list[int], list[BusFaultEvent]]: + bus, _clock, _inner = _bus({"type": "drop", "probability": 0.5}, seed=seed) + for port in range(40): + _send(bus, port % 64) + return _ports_received(bus), list(bus.events) + + +def test_the_same_seed_gives_the_same_faults_and_another_seed_does_not() -> None: + first = _probabilistic_run(11) + + assert _probabilistic_run(11) == first + assert _probabilistic_run(12) != first + delivered, events = first + assert 0 < len(events) < 40 + assert len(delivered) + len(events) == 40 + + +def test_adding_a_rule_does_not_change_the_draws_of_another() -> None: + def drops(extra: bool) -> list[int]: + rules = [{"type": "drop", "probability": 0.5, "port": 1}] + if extra: + rules.append({"type": "duplicate", "probability": 0.5, "port": 2}) + bus, _clock, _inner = _bus(*rules, seed=3) + for _ in range(30): + _send(bus, 1) + _send(bus, 2) + return [e.match for e in bus.events if e.type == "drop"] + + assert drops(extra=False) == drops(extra=True) + + +def test_take_lost_sequences_reports_dropped_frames_once() -> None: + bus, _clock, _inner = _bus({"type": "drop", "nth": 1}) + + lost = _send(bus, 7) + _send(bus, 7) + + assert bus.take_lost_sequences() == (lost,) + assert bus.take_lost_sequences() == () + + +def test_without_rules_the_bus_is_a_plain_pass_through() -> None: + bus, _clock, inner = _bus() + + sequences = [_send(bus, 7), _send(bus, 8)] + + assert sequences == [0, 1] + assert [e.sequence for e in bus.drain()] == [0, 1] + assert inner.pending_count() == 0 + assert not bus.has_rules + + +def test_format_bus_fault_is_a_single_greppable_line() -> None: + bus, _clock, _inner = _bus({"type": "drop", "port": 7, "nth": 1, "name": "lose-first"}) + _send(bus, 7) + + line = format_bus_fault(bus.events[0]) + + assert line.startswith("FAULT t=0 rule=lose-first action=drop dir=TX match=1 can_id=0x") + assert " dport=7 " in line and "\n" not in line + + +# --- end to end through the runner ----------------------------------------------------------- + +SETUP = """ +[transport] +type = "in-memory" + +[nodes.obc] +mode = "simulated" +module_type = "obc_peer" +address = 1 + +[nodes.eps] +mode = "simulated" +module_type = "generic_eps" +address = 2 + +[nodes.payload] +mode = "simulated" +module_type = "simple_payload" +address = 3 + +[nodes.obc.commands.payload_power_off] +target = "payload" +destination_port = 10 +source_port = 10 +payload_hex = "{command_payload}" + +[nodes.eps.telemetry.battery_percent] +source_port = 20 +destination_port = 20 +units = "percent" +min = 0 +max = 100 +offset = 0 +length = 4 +type = "float" + +[nodes.payload.telemetry.power_status] +source_port = 21 +destination_port = 21 +offset = 0 +length = 1 +type = "uint" + +[nodes.payload.telemetry.power_status.enum] +0 = "offline" +1 = "online" + +[nodes.obc.rules.low_battery_shed_payload] +signal = "eps.telemetry.battery_percent" +op = "<" +threshold = 30.0 +for = "3s" + +[[nodes.obc.rules.low_battery_shed_payload.actions]] +type = "send_command" +command = "payload_power_off" +""" + +SCENARIO = """ +name: Shed the payload +steps: + - action: inject_fault + type: state_override + target: eps.model.battery_percent + value: 25 + duration: 10s + - action: wait + virtual_time: 3s + - action: assert + signal: payload.telemetry.power_status + op: "==" + value: offline + timeout: 1s + - action: wait + virtual_time: 3s +""" + +LOSE_THE_COMMAND = """ +[bus_faults] +seed = 42 + +[[bus_faults.rules]] +type = "drop" +port = 10 +nth = 1 +""" + + +def _run( + setup_extra: str = "", + scenario_text: str = SCENARIO, + *, + command_payload: str = "00", + recorder: RunRecorder | None = None, +) -> tuple[object, str]: + setup = parse_testbed_config(SETUP.format(command_payload=command_payload) + setup_extra) + scenario = parse_scenario(scenario_text, setup=setup) + trace = io.StringIO() + result = run_scenario(scenario, setup, output=io.StringIO(), trace=trace, recorder=recorder) + return result, trace.getvalue() + + +def test_a_run_without_bus_faults_reports_none() -> None: + result, trace = _run() + + assert result.passed # type: ignore[attr-defined] + assert result.bus_faults == () # type: ignore[attr-defined] + assert result.bus_fault_seed is None # type: ignore[attr-defined] + assert "FAULT" not in trace + + +def test_a_dropped_command_is_visible_in_the_trace_and_the_result() -> None: + result, trace = _run(LOSE_THE_COMMAND) + + assert result.bus_fault_seed == 42 # type: ignore[attr-defined] + (fault,) = result.bus_faults # type: ignore[attr-defined] + assert (fault.type, fault.direction, fault.destination_port, fault.match) == ( + "drop", + "TX", + 10, + 1, + ) + + lines = trace.splitlines() + fault_index = next(i for i, line in enumerate(lines) if " FAULT " in line) + # The frame the OBC tried to send comes first, what the bus did to it right after, + # and that frame never shows up as received. + assert " TX " in lines[fault_index - 1] and "dport=10" in lines[fault_index - 1] + assert lines[fault_index].startswith(f"trace FAULT t={fault.time} rule=drop-1 action=drop") + assert not any( + " RX " in line and f"t={fault.time} " in line and "dport=10" in line for line in lines + ) + + +def test_the_rule_in_setup_and_the_same_rule_as_a_scenario_step_do_the_same_thing() -> None: + step = """ + - action: inject_bus_fault + type: drop + port: 10 + nth: 1 +""" + in_setup, _ = _run(LOSE_THE_COMMAND) + in_scenario, _ = _run(scenario_text=SCENARIO.replace("steps:\n", "steps:" + step, 1)) + + def shape(result: object) -> list[tuple[int, str, int]]: + return [(f.time, f.type, f.match) for f in result.bus_faults] # type: ignore[attr-defined] + + assert shape(in_scenario) == shape(in_setup) + assert in_scenario.bus_fault_seed == 0 # type: ignore[attr-defined] + + +def test_two_runs_with_one_seed_have_byte_identical_traces() -> None: + probabilistic = """ +[bus_faults] +seed = 5 + +[[bus_faults.rules]] +type = "drop" +probability = 0.4 +port = 20 + +[[bus_faults.rules]] +type = "corrupt" +probability = 0.3 +""" + first_result, first_trace = _run(probabilistic) + second_result, second_trace = _run(probabilistic) + + assert first_trace == second_trace + assert first_result.bus_faults == second_result.bus_faults # type: ignore[attr-defined] + assert first_result.bus_faults # type: ignore[attr-defined] + + _, other_trace = _run(probabilistic.replace("seed = 5", "seed = 6")) + assert other_trace != first_trace + + +def test_a_dropped_fragment_makes_the_packet_time_out_visibly() -> None: + long_payload = "00 01 02 03 04 05 06 07 08 09 0a 0b" + result, trace = _run( + """ +[[bus_faults.rules]] +type = "drop" +port = 10 +nth = 2 +""", + command_payload=long_payload, + ) + + # The second frame on port 10 is the first continuation fragment of the command. + reasons = {d.reason for d in result.dropped_packets} # type: ignore[attr-defined] + assert reasons & {"timeout", "frame_lost", "orphan_fragment"} + assert any(" FAULT " in line and "dport=10" in line for line in trace.splitlines()) + assert any(" DROP " in line for line in trace.splitlines()) + + +def test_a_delayed_command_arrives_late_with_its_payload_intact() -> None: + delayed = """ +[[bus_faults.rules]] +type = "delay" +port = 10 +nth = 1 +delay = "700ms" +""" + result, trace = _run(delayed) + + (fault,) = result.bus_faults # type: ignore[attr-defined] + assert fault.detail == f"until={fault.time + 700_000}" + received = [ + line + for line in trace.splitlines() + if " RX " in line and "dport=10" in line and f"t={fault.time + 700_000} " in line + ] + # Released in the very instant its delay ran out, not at the next physical step. + assert received + assert result.passed # type: ignore[attr-defined] + + +def test_a_dropped_command_does_not_leave_its_payload_behind() -> None: + from cubesat_testbed.scenario.runner import ScenarioRunner, build_runtime + + setup = parse_testbed_config(SETUP.format(command_payload="00") + LOSE_THE_COMMAND) + scenario = parse_scenario( + SCENARIO.replace( + " - action: wait\n virtual_time: 3s\n - action: assert", + " - action: send_command\n command: payload_power_off\n payload: 1\n" + " - action: wait\n virtual_time: 3s\n - action: assert", + 1, + ), + setup=setup, + ) + runtime = build_runtime(setup) + runner = ScenarioRunner(runtime, output=io.StringIO()) + + runner.run(scenario) + + assert runtime.bus_faults is not None + assert runner._semantic_payloads_by_envelope == {} + + +def test_cli_reports_bus_faults_in_text_json_and_html( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + setup_path = tmp_path / "setup.toml" + scenario_path = tmp_path / "scenario.yaml" + html_path = tmp_path / "report.html" + setup_path.write_text(SETUP.format(command_payload="00") + LOSE_THE_COMMAND) + scenario_path.write_text(SCENARIO) + base = ["run", "-c", str(setup_path), "-s", str(scenario_path)] + + assert main([*base, "--html", str(html_path)]) == 1 + text = capsys.readouterr().out + assert "FAULT t=" in text and "action=drop" in text + assert "bus_faults=1 bus_fault_seed=42" in text + assert "bus faults" in html_path.read_text() and 'class="fault"' in html_path.read_text() + + assert main([*base, "--json"]) == 1 + payload = json.loads(capsys.readouterr().out) + assert payload["bus_fault_seed"] == 42 + assert [(f["type"], f["direction"], f["destination_port"]) for f in payload["bus_faults"]] == [ + ("drop", "TX", 10) + ] + + +def test_the_html_report_lists_each_fault_and_the_seed() -> None: + recorder = RunRecorder() + result, _trace = _run(LOSE_THE_COMMAND, recorder=recorder) + + html = render_report(recorder, result) # type: ignore[arg-type] + + assert html.count('data-kind="fault"') == 1 + assert "seed 42" in html + + +def test_a_corrupted_telemetry_payload_is_a_reported_drop_not_an_aborted_run() -> None: + corrupt_status = """ +[[bus_faults.rules]] +type = "corrupt" +port = 21 +nth = 1 +byte = 4 +""" + result, trace = _run(corrupt_status) + + drops = result.dropped_packets # type: ignore[attr-defined] + assert [(d.reason, d.destination, d.sender) for d in drops] == [("undecodable", 3, 3)] + assert "payload.telemetry.power_status" in drops[0].detail + assert any(" DROP " in line and "reason=undecodable" in line for line in trace.splitlines()) + assert len(result.bus_faults) == 1 # type: ignore[attr-defined] diff --git a/tests/test_html_report.py b/tests/test_html_report.py index aefeab7..e072219 100644 --- a/tests/test_html_report.py +++ b/tests/test_html_report.py @@ -413,7 +413,7 @@ def test_cli_unwritable_report_path_is_an_execution_error( addEventListener(type, fn) { this.listeners[type] = fn; }}, extra || {}); } const rows = data.rows.map((r) => ({hidden: false, textContent: r.text, getAttribute: attr(r)})); const ids = {'f-dir': el(), 'f-node': el(), 'f-port': el(), 'f-text': el(), 'f-count': el(), - 'k-frame': el(), 'k-drop': el(), 'k-step': el()}; + 'k-frame': el(), 'k-drop': el(), 'k-fault': el(), 'k-step': el()}; for (const r of rows) {} const byId = {}; data.rows.forEach((r, i) => { if (r.id) byId[r.id] = rows[i]; }); From e2aed711e29600beef59b0066bdf596d05b5194a Mon Sep 17 00:00:00 2001 From: glocker Date: Mon, 16 Nov 2026 16:00:18 +0000 Subject: [PATCH 2/2] Document bus fault injection --- CHANGELOG.md | 19 +++++++++ README.md | 19 +++++++++ docs/architecture.md | 8 ++++ docs/roadmap.md | 5 +++ docs/v1-scope.md | 94 +++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 143 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d24269..062b13f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,21 @@ Versioning follows [Semantic Versioning](https://semver.org/). - `TransportAdapter.new_sender()` and `take_drops()`: how a packet becomes frames, and what a link discarded, are now the transport's business, so the runner and OBC Peer work unchanged over CAN and serial. +- Bus fault injection. Faults used to stop at the module boundary; now the in-memory + bus can fail too. `[bus_faults]` in the setup (a `seed` and `[[bus_faults.rules]]`) and + `inject_bus_fault` scenario steps apply `drop`, `delay`, `corrupt`, `duplicate` and + `reorder` rules to CAN frames, on send (`direction = "tx"`) or on receive (`"rx"`), + selected by `port`, `source_port`, `source`, `destination` and the `nth` matching + frame (with `count`) or a `probability`. A rule on a port reaches every fragment of a + multi-frame packet, so `nth` can drop exactly one fragment and the receiver reports + the loss as `frame_lost`/`timeout`. Delays are virtual-time events; every random + choice comes from a generator seeded from `seed` and the rule's position, so the same + seed gives a byte-identical `--trace`. Every injected fault is reported: `FAULT` + lines in `--trace` (right after the `TX` line of the frame concerned) and in the text + output, `bus_faults` and `bus_fault_seed` in `--json` and on the run result, rows in + the HTML report. In-memory transport only: the setup is rejected on `socketcan`, `kiss` + and `zmq`, which run on wall-clock time. See + [`docs/v1-scope.md`](docs/v1-scope.md#bus-faults). ### Changed (internal) @@ -91,6 +106,10 @@ Versioning follows [Semantic Versioning](https://semver.org/). ### Changed +- A telemetry payload the signal's codec cannot read (an enum value with no label, say a + byte a bus fault corrupted) is now a reported dropped packet (`reason=undecodable`, + in `--trace`, the text output, `--json` and the HTML report) and the run goes on, instead + of ending with an execution error. - A command `payload_hex` or telemetry layout longer than 4 bytes is accepted; the former 4-byte limit is replaced by the sending node's `max_packet_bytes`. - `ObcPeerNamedCommand.encode()` returns the tuple of frames carrying the command diff --git a/README.md b/README.md index 84bdcb7..395824e 100644 --- a/README.md +++ b/README.md @@ -276,6 +276,25 @@ It is written even when assertions fail or the run aborts, and it composes with `--json`, `--quiet` and `--trace`. The file contains no wall-clock time, so the same run produces the same bytes. +**9. Break the bus itself** + +`inject_fault` breaks a module. The bus can fail too: drop the Nth frame on a port, +delay a frame by a virtual duration, flip a byte, duplicate a frame or deliver it after +the next one, on send or on receive. Add the rule to the setup, or as a scenario step: + +```yaml +- action: inject_bus_fault + type: drop + port: 10 # destination port; the 2nd frame addressed to it is lost + nth: 2 +``` + +The randomness (`probability`, the byte a `corrupt` rule picks) is seeded +(`[bus_faults] seed = 42`), delays are virtual time, so two runs with one seed give +byte-identical traces, and every fault is marked in `--trace`, the text and `--json` +output and the HTML report, never silent. In-memory transport only. See +[`docs/v1-scope.md`](docs/v1-scope.md#bus-faults). + ## Hardware in the loop Point the same scenario at a real bus: switch the node under test to diff --git a/docs/architecture.md b/docs/architecture.md index 88a5f09..2c23c5f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -20,6 +20,7 @@ src/cubesat_testbed/ transport/base.py # Bus adapter interface transport/factory.py # Config-to-adapter construction transport/in_memory.py # CI/local in-memory bus + transport/faults.py # bus fault decorator: drop/delay/corrupt/duplicate/reorder frames transport/socketcan.py # Linux SocketCAN adapter transport/kiss.py # serial KISS adapter (whole CSP packets, local delivery) transport/serial_line.py # termios serial line (tty or pty), standard library only @@ -125,6 +126,13 @@ the next event time instead of spinning in idle loops. - `signal_override` targets `.telemetry.` paths and spoofs outgoing telemetry. - `named_fault` activates a module-defined fault flag. +`FaultInjectingTransport` (`transport/faults.py`) is the same kind of passive +executor one level down: it breaks the *bus* instead of a module, applying explicit +`[bus_faults]` rules and `inject_bus_fault` steps to the frames of the in-memory +transport (drop the Nth frame on a port, delay, corrupt, duplicate, reorder). Its +randomness is seeded and its delays are virtual-time events. See +[Bus faults](v1-scope.md#bus-faults). + The OBC Peer is the active decision maker. It listens to telemetry and applies rules such as: diff --git a/docs/roadmap.md b/docs/roadmap.md index d3d1934..bb0691e 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -98,6 +98,11 @@ verified in CI before any codec or adapter code. 4. [x] **Self-contained HTML run report.** `run --html OUT.html` writes one offline file with the assertions linked to the frames they were judged on, the decoded frame timeline with filters, signal plots and dropped packets. +5. [x] **Bus fault injection.** Faults no longer stop at the module boundary: on the + in-memory bus a scenario can drop the Nth frame on a port, delay a frame by a + virtual duration, flip a byte, duplicate a frame or deliver it after the next + one, on send or on receive, with a seed. Two runs with one seed give + byte-identical traces and `--trace` marks every injected fault. ## Next diff --git a/docs/v1-scope.md b/docs/v1-scope.md index bb32c57..8cf8d4a 100644 --- a/docs/v1-scope.md +++ b/docs/v1-scope.md @@ -132,6 +132,92 @@ setup against several different FDIR rule sets without editing it. A rules file reuses the exact same schema, keyed by node name at the top level (see [`configs/schema/module_schema.md`](../configs/schema/module_schema.md#obc-peer-rules)). +### Bus faults + +`FaultInjectionEngine` breaks a *module*. Bus faults break the *bus*: the in-memory +transport is wrapped in a decorator (`transport/faults.py`, +`FaultInjectingTransport`) that applies explicit rules to the CAN frames passing +through it. It is passive like the engine above: it decides nothing, it applies the +rules it was given. A rule says which frames, when, and what happens: + +```toml +[bus_faults] +seed = 42 # every random choice derives from this; default 0 + +[[bus_faults.rules]] +type = "drop" # drop | delay | corrupt | duplicate | reorder +port = 10 # destination port; also source_port, source, destination +nth = 2 # the 2nd matching frame (1-based) +``` + +The same fields work as a scenario step, which puts the rule in force from that +instant on and starts counting `nth` from there: + +```yaml +- action: inject_bus_fault + type: delay + port: 20 + nth: 1 + delay: 300ms +``` + +| field | meaning | +| --- | --- | +| `type` | `drop` loses the frame; `delay` holds it for `delay` of virtual time; `corrupt` XORs `mask` (default `0xFF`) into data byte `byte` (default: one drawn from the seed); `duplicate` delivers it twice; `reorder` holds it until the next frame in that direction has gone by, or for `hold` (default `1s`) if none comes | +| `direction` | `tx` (default) acts when a node sends, before the frame reaches the bus; `rx` acts on what the receiver reads off it. Write two rules for both | +| `port`, `source_port`, `source`, `destination` | selectors; a frame must match all that are given. `port` is the CSP destination port | +| `nth`, `count` | fire on the `nth` matching frame and the next `count - 1` (default 1). Without `nth` the rule fires on every matching frame, at most `count` times if given | +| `probability` | fire on each matching frame with this probability, drawn from the seeded generator; not combinable with `nth` | +| `name` | label in the trace and the result; default `-` | + +Things worth knowing: + +- **Frames, not packets.** A multi-frame packet is several frames, so `nth` can drop + exactly one fragment. Only a packet's first frame carries its ports and source + address, so the decorator remembers each first frame's header and matches the + continuation fragments by the packet they belong to; a rule on `port = 10` therefore + reaches every fragment of a packet to port 10. The receiver then reports the loss + like any other (`frame_lost`, `timeout`, ...), as dropped packets. +- **`byte` indexes the frame's CAN data field**, as `--trace` prints it in `data=` + (the first frame's first four bytes are the CSP header). Flipping a header byte + can make the frame unroutable or undecodable, which the trace then says. A `byte` + past the end of a short frame corrupts nothing; the fault is still reported, with + `skipped` in its detail. +- **One rule per frame.** Rules are tried in the order they were added (setup first, + then scenario steps); the first one that fires acts, and a rule counts only the + frames that reached it. +- **Delay is virtual.** A held frame is released by a scheduled event at its + virtual time and delivered in that same instant; nothing sleeps. +- **Determinism.** Each rule has its own generator, seeded from `seed` and the rule's + position, and nothing reads the wall clock, so the same seed, rules and scenario + give a byte-identical `--trace`. Adding a rule does not change the draws of the + others. The seed is part of the result (`bus_fault_seed`). +- **Visible.** `--trace` prints a `FAULT` line right after the `TX` line of the frame + it concerns (an `rx` fault, which has no such line, is printed where it happens): + + ```text + trace t=3000000 TX can_id=0x10006083 dlc=5 pri=2 src=1 dst=3 dport=10 ... command obc.payload_power_off->payload + trace FAULT t=3000000 rule=drop-1 action=drop dir=TX match=1 can_id=0x10006083 src=1 dst=3 dport=10 sport=10 data=0004a28000 + ``` + + A `delay` fault carries `until=`, a `corrupt` fault the byte, mask and + before/after values, a `reorder` fault its `hold`. The same events are `FAULT` lines + in the text output, `bus_faults`/`bus_fault_seed` in `--json`, rows in the HTML + report, and `bus_faults` on the run result. +- **What `TX` and `RX` mean once rules exist.** The tracing adapter sits above the + decorator, so `TX` is a frame a node *tried* to send and `RX` a frame that + *reached* the receiver. A dropped frame is a `TX` with a `FAULT drop` and no `RX`; + a duplicated one is one `TX` and two `RX`. +- **`rx` faults act on the bus monitor stream**, which is where the runner decodes + every frame for every node; per-endpoint streams pass through unchanged. + +**In-memory transport only.** A real bus, a serial line and a ZeroMQ hub deliver on +wall-clock time, so a delay or an ordering there could not be reproduced from a seed, +and a fault can only touch what the testbed itself sends or reads, not traffic between +two real devices. `[bus_faults]` and `inject_bus_fault` are therefore rejected when +the setup's transport is not `in-memory`. The decorator is always present on the +in-memory bus and is a plain pass-through until a rule exists. + ## Config and scenario files Static CubeSat/testbed setup uses TOML; scenario scripts use YAML. Both are @@ -234,8 +320,9 @@ A receiving node discards a packet, as libcsp does, when a frame is missing or out of order (`frame_lost`), the packet outgrows the node's limit (`overflow`), a continuation frame has no packet in progress (`orphan_fragment`), a new first frame arrives mid-packet (`restarted`), the frame or header is invalid -(`malformed`), or no frame arrives for one virtual second (`timeout`). Nothing is -discarded silently: +(`malformed`), or no frame arrives for one virtual second (`timeout`). A complete +packet whose telemetry payload the signal's codec cannot read (a value with no enum +label, a corrupted byte) is discarded as `undecodable`. Nothing is discarded silently: - `run` prints a `DROP t=... node=... reason=... detail=...` line per dropped packet before the summary (`SUMMARY` gains `dropped_packets=N`); @@ -651,6 +738,9 @@ consequences worth knowing: decodes it. That round trip is precisely what makes an assertion an observation of the wire rather than of module state (see [Telemetry wire encoding](#telemetry-wire-encoding)). +- With bus fault rules in force, `TX` means a frame a node tried to send and `RX` + one that reached the receiver; injected faults are `FAULT` lines between them + (see [Bus faults](#bus-faults)). - On SocketCAN the adapter does not receive its own messages, so outgoing frames appear only as `TX` and everything the peer sends appears as `RX`. This is the only place an outgoing HIL command is visible at all.