diff --git a/CHANGELOG.md b/CHANGELOG.md index bf2c136..1d24269 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,15 @@ Versioning follows [Semantic Versioning](https://semver.org/). [`docs/v1-scope.md`](docs/v1-scope.md#zeromq-transport). - The whole-packet links (KISS, ZMQ) share one routing and delivery implementation (`transport/whole_packet.py`); KISS behaviour is unchanged. +- `run --html PATH`: a self-contained HTML report of the run, as one offline file for a CI + artifact or a colleague. It links each assertion to the frame its value was decoded + from (a failing assertion leads straight to the frame that carried the wrong value), + shows the decoded frame timeline with the scenario steps and dropped packets and filters + by direction, node, port and text, and plots every numeric signal over virtual time with + the assertions marked. It needs no server and no network, is readable with scripting + off, is written even when assertions fail or the run aborts, composes with `--json`, + `--quiet` and `--trace`, and is byte-identical for the same run. See + [`docs/v1-scope.md`](docs/v1-scope.md#html-report). - `--trace` shows a whole-packet frame as one line (`link=serial|zmq|local`). - `run` warns on stderr, as for SocketCAN, when a `kiss` or `zmq` transport is used without `--realtime`. @@ -53,6 +62,13 @@ Versioning follows [Semantic Versioning](https://semver.org/). what a link discarded, are now the transport's business, so the runner and OBC Peer work unchanged over CAN and serial. +### Changed (internal) + +- The `--trace` line is now rendered from a structured `FrameRecord` + (`transport.tracing.describe_frame` / `format_frame_record`) instead of being built as + text directly; the output is unchanged. `TracingTransportAdapter` takes an optional + `observer`, and its `stream` is now optional. + ### Fixed - A configured command whose target is a `software`/`hardware` node no longer aborts diff --git a/README.md b/README.md index 3b18ac6..84bdcb7 100644 --- a/README.md +++ b/README.md @@ -253,6 +253,29 @@ it, not a parallel report. It goes to `stderr`, so it composes with `--json` and `--quiet`, and it works on SocketCAN too, where it is the only way to see what your testbed actually put on the wire. +**8. A report you can attach to a CI run** + +Add `--html report.html` and the run also writes one self-contained HTML file: + +```sh +cubesat-testbed run --example default --html report.html +``` + +It needs nothing to open (no server, no network, no installed testbed) and shows what +`--trace` shows, in a form you can send to a colleague or keep as a CI artifact: + +- the **assertions**, each linked to the frame its value was decoded from, so a failing + assertion takes you straight to the frame that carried the wrong value; +- the **frame timeline**, decoded as in the trace, interleaved with the scenario steps + and any dropped packets, with filters by direction, node, port and text; +- **plots** of every numeric signal over virtual time, with the assertions marked on them; +- the setup (transport and nodes) and, if the run aborted, the error and everything + recorded up to it. + +It is written even when assertions fail or the run aborts, and it composes with +`--json`, `--quiet` and `--trace`. The file contains no wall-clock time, so the same run +produces the same bytes. + ## Hardware in the loop Point the same scenario at a real bus: switch the node under test to @@ -300,8 +323,9 @@ sudo ip link set up vcan0 USB-serial or a pty) and ZeroMQ (a libcsp `zmqproxy` hub; needs the `zmq` extra: `pip install 'cubesat-testbed[zmq]'`). - Deterministic virtual-time engine, TOML setup + YAML scenarios, PASS/FAIL - CLI report with CI-friendly exit codes, JUnit XML output and a decoded - wire-level frame trace (`--trace`). + CLI report with CI-friendly exit codes, JUnit XML output, a decoded + wire-level frame trace (`--trace`) and a self-contained HTML report + (`--html`). Full detail, constraints, and what's deliberately out of scope for v1: [`docs/v1-scope.md`](docs/v1-scope.md). diff --git a/docs/architecture.md b/docs/architecture.md index bbefaed..88a5f09 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -24,6 +24,8 @@ src/cubesat_testbed/ transport/kiss.py # serial KISS adapter (whole CSP packets, local delivery) transport/serial_line.py # termios serial line (tty or pty), standard library only transport/whole_packet.py # routing shared by the whole-packet links (KISS, ZMQ) + report/recorder.py # observer that records a run for the HTML report + report/html_report.py # renders the recording as one self-contained HTML file transport/zmq.py # ZeroMQ adapter: a client of a libcsp zmqproxy hub transport/zmq_link.py # pyzmq PUB/SUB connection (optional `zmq` extra) protocol/csp_v2_packet.py # whole-packet frame and 6-byte header shared by KISS and ZMQ diff --git a/docs/roadmap.md b/docs/roadmap.md index 50c1ae6..d3d1934 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -84,7 +84,7 @@ unblocked that user, highest first. ## Landed since v1.1.0 (unreleased) On `main`, not yet in a published release; see the `[Unreleased]` section of -[`CHANGELOG.md`](../CHANGELOG.md). Each item had its golden vectors committed and +[`CHANGELOG.md`](../CHANGELOG.md). Items 1-3 had their golden vectors committed and verified in CI before any codec or adapter code. 1. [x] **CFP2 fragmentation and reassembly.** A packet longer than one CAN frame is @@ -95,6 +95,9 @@ verified in CI before any codec or adapter code. 3. [x] **ZeroMQ transport.** The testbed as a client of a libcsp `zmqproxy` hub, so an unmodified libcsp process can exchange packets with a simulated node; needs the optional `zmq` extra. +4. [x] **Self-contained HTML run report.** `run --html OUT.html` writes one offline + file with the assertions linked to the frames they were judged on, the decoded + frame timeline with filters, signal plots and dropped packets. ## Next diff --git a/docs/v1-scope.md b/docs/v1-scope.md index 4e66f75..bb32c57 100644 --- a/docs/v1-scope.md +++ b/docs/v1-scope.md @@ -661,6 +661,58 @@ touch the assertion path, and enabling it does not change scenario results or determinism. From the Python API, pass a stream as `trace=` to `run_scenario`, `run_scenario_files` or `build_runtime`. +### HTML report + +`--html PATH` writes one self-contained HTML file for the run, in addition to the +normal output: + +```sh +cubesat-testbed run --example default --html report.html +``` + +**What it contains.** + +- **Assertions.** Result, expected and actual value, when it was evaluated and an + *evidence* link: the frame (the last fragment, for a multi-frame packet) from which the + signal's latest value was decoded when the assertion was evaluated. For a passing + assertion that is the frame that satisfied it, for a failing one the frame whose value + did not match; a signal never seen on the bus says so and has no link. Failed + assertions are repeated in a list at the top. +- **Setup.** The transport and every node (address, mode, module, `max_packet_bytes`). +- **Signals.** A step plot of each numeric telemetry signal over virtual time, with a + line where an assertion on it was evaluated; a table of value changes for the others + (enumerations, booleans). +- **Timeline.** Every frame that crossed the transport boundary, decoded exactly as + `--trace` prints it (both are rendered from the same `FrameRecord`), interleaved with + the scenario steps and the dropped packets. On the in-memory bus a frame appears twice, + `TX` when sent and `RX` when read back. Filters: direction, node, port, the three kinds + of row, and free text. A multi-frame packet shows every fragment, the earlier ones + dimmed, and the last one with the reassembled payload. + +**Properties.** + +- *Self-contained:* inline stylesheet, script and SVG; no font, image or network fetch. + Everything is readable with scripting off: only the filters need it, and the evidence + links are plain `#f` anchors. Following one to a row a filter has hidden clears the + filters. +- *Passive:* it is an observer on the same transport wrapper as `--trace`. It changes + neither scenario semantics, ordering nor determinism, and a run gives the same result + with and without it. +- *Deterministic:* the file is a pure function of the run, with no wall-clock time, host + name, absolute path or random id, so the same inputs give the same bytes. +- *Always written:* also when assertions fail (exit code 1) and when the run aborts + (exit code 2), the latter with the error and everything recorded up to it. If the file + cannot be written the run is an execution error (exit code 2). +- *Composes* with `--json`, `--quiet` and `--trace`; the only extra output is a note on + `stderr` unless `--quiet`. + +The timestamps in it are the virtual ones described under +[Timestamps of frames from a real peer](#timestamps-of-frames-from-a-real-peer). + +From Python, pass a `cubesat_testbed.report.RunRecorder` as `recorder=` to +`run_scenario`, `run_scenario_files` or `build_runtime`, then +`cubesat_testbed.report.write_report(path, recorder, result)`. + ### Hardware-in-the-loop from the CLI `--realtime` runs the scenario on a `RealTimeClock` instead of the default diff --git a/src/cubesat_testbed/main.py b/src/cubesat_testbed/main.py index 4544834..d385777 100644 --- a/src/cubesat_testbed/main.py +++ b/src/cubesat_testbed/main.py @@ -28,6 +28,7 @@ available_examples, get_example, ) +from cubesat_testbed.report import RunRecorder, write_report from cubesat_testbed.scenario import ( ScenarioRunResult, build_obc_rules_from_file, @@ -123,6 +124,17 @@ def _build_parser() -> argparse.ArgumentParser: "CSP header fields, raw bytes and the decoded command/telemetry signal" ), ) + run_parser.add_argument( + "--html", + type=Path, + metavar="PATH", + help=( + "write a self-contained HTML report of the run to PATH: assertions linked to the " + "frames they were judged on, the decoded frame timeline with filters, signal " + "plots over virtual time, dropped packets and the scenario steps. One offline " + "file, written even when the run aborts; composes with --json and --quiet" + ), + ) run_parser.add_argument( "--quiet", action="store_true", @@ -189,6 +201,7 @@ def _run_command(args: argparse.Namespace) -> int: suppress_assertion_output = bool(args.quiet or args.json) output = io.StringIO() if suppress_assertion_output else None clock: Clock | None = RealTimeClock() if args.realtime else None + recorder = RunRecorder() if args.html is not None else None started = time.perf_counter() try: @@ -211,9 +224,17 @@ def _run_command(args: argparse.Namespace) -> int: # The trace goes to stderr precisely so it stays out of the way of # --json/--quiet stdout, which is why it ignores both flags. trace=sys.stderr if args.trace else None, + recorder=recorder, ) except Exception as exc: # noqa: BLE001 # CLI contract: execution errors map to 2; assertion failures map to 1 via result.passed. + if recorder is not None and args.html is not None: + # A run that aborts is when the report is most wanted; it shows what was + # recorded up to the error. Failing to write it must not hide the real error. + try: + write_report(args.html, recorder, None, error=str(exc)) + except OSError as write_error: + print(f"warning: could not write the HTML report: {write_error}", file=sys.stderr) if args.junit_xml is not None: _write_junit_error_xml( args.junit_xml, str(exc), kind="execution_error", started=started @@ -232,6 +253,20 @@ def _run_command(args: argparse.Namespace) -> int: format_junit_xml(result, wall_time_seconds=elapsed), encoding="utf-8" ) + if recorder is not None and args.html is not None: + try: + write_report(args.html, recorder, result) + except OSError as exc: + _print_error( + f"could not write the HTML report to {args.html}: {exc}", + json_output=args.json, + exit_code=_EXIT_EXECUTION_ERROR, + kind="report_error", + ) + return _EXIT_EXECUTION_ERROR + if not args.quiet: + print(f"html report written to {args.html}", file=sys.stderr) + if not result.assertions: print("warning: 0 assertions in scenario", file=sys.stderr) diff --git a/src/cubesat_testbed/report/__init__.py b/src/cubesat_testbed/report/__init__.py new file mode 100644 index 0000000..35b794a --- /dev/null +++ b/src/cubesat_testbed/report/__init__.py @@ -0,0 +1,24 @@ +"""Run recording and the self-contained HTML report built from it.""" + +from cubesat_testbed.report.html_report import render_report, write_report +from cubesat_testbed.report.recorder import ( + RecordedAssertion, + RecordedDrop, + RecordedFrame, + RecordedStep, + RunRecorder, + describe_step, + format_virtual_time, +) + +__all__ = [ + "RecordedAssertion", + "RecordedDrop", + "RecordedFrame", + "RecordedStep", + "RunRecorder", + "describe_step", + "format_virtual_time", + "render_report", + "write_report", +] diff --git a/src/cubesat_testbed/report/html_report.py b/src/cubesat_testbed/report/html_report.py new file mode 100644 index 0000000..9dd46b3 --- /dev/null +++ b/src/cubesat_testbed/report/html_report.py @@ -0,0 +1,609 @@ +"""Renders a recorded run as one self-contained HTML file. + +The file needs nothing at view time: the stylesheet and the few lines of script are +inline, the signal plots are inline SVG, and there is no font, image or network +fetch. Everything is readable with scripting off; the script only adds the timeline +filters. An assertion links to the frame its value was decoded from with a plain +``#f`` anchor, so that works without scripting too. + +The output is a pure function of the recording: no wall-clock time, host name, +absolute path or random id goes into it, so the same run produces the same bytes. +""" + +from __future__ import annotations + +from html import escape +from importlib import metadata +from pathlib import Path +from typing import TYPE_CHECKING + +from cubesat_testbed.report.recorder import ( + RecordedAssertion, + RecordedDrop, + RecordedFrame, + RecordedStep, + RunRecorder, + format_virtual_time, +) + +if TYPE_CHECKING: # pragma: no cover - typing only + from cubesat_testbed.scenario.runner import ScenarioRunResult + +_PLOT_WIDTH = 720 +_PLOT_HEIGHT = 170 +_MARGIN_LEFT = 62 +_MARGIN_RIGHT = 14 +_MARGIN_TOP = 12 +_MARGIN_BOTTOM = 26 + + +def _esc(value: object) -> str: + return escape(str(value), quote=True) + + +def _version() -> str: + try: + return metadata.version("cubesat_testbed") + except metadata.PackageNotFoundError: # pragma: no cover - not installed + return "unknown" + + +def _hex(data: bytes | None) -> str: + if data is None: + return "" + return data.hex(" ") if data else "—" + + +def _display(value: object) -> str: + if isinstance(value, float): + return repr(value) + return str(value) + + +# --- node names ------------------------------------------------------------------------ + + +def _node_names(recorder: RunRecorder) -> dict[int, str]: + if recorder.setup is None: + return {} + return {node.address: name for name, node in recorder.setup.nodes.items()} + + +def _node_label(address: int | None, names: dict[int, str]) -> str: + if address is None: + return "?" + name = names.get(address) + return f"{name} ({address})" if name else str(address) + + +# --- plots ----------------------------------------------------------------------------- + + +def _is_number(value: object) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def _collapse(points: list[tuple[int, object]]) -> list[tuple[int, float]]: + """Keep the first point of every run of equal values, plus the very last point. + + A held value draws as a flat step, so the points in the middle of a run add bytes + and no information; the last point keeps the plot reaching the end of the run. + """ + + kept: list[tuple[int, float]] = [] + previous: object = object() + for time, value in points: + number = float(value) # type: ignore[arg-type] + if number != previous: + kept.append((time, number)) + previous = number + if points and (not kept or kept[-1][0] != points[-1][0]): + kept.append((points[-1][0], float(points[-1][1]))) # type: ignore[arg-type] + return kept + + +def _tick_label(value: float) -> str: + text = f"{value:.4g}" + return text + + +def _plot_svg( + signal: str, + points: list[tuple[int, float]], + marks: list[RecordedAssertion], + *, + start: int, + end: int, +) -> str: + """A step plot of one numeric signal with a marker per assertion on that signal.""" + + low = min(value for _, value in points) + high = max(value for _, value in points) + if low == high: + pad = abs(low) * 0.1 or 1.0 + low, high = low - pad, high + pad + span_x = max(end - start, 1) + plot_w = _PLOT_WIDTH - _MARGIN_LEFT - _MARGIN_RIGHT + plot_h = _PLOT_HEIGHT - _MARGIN_TOP - _MARGIN_BOTTOM + + def x_of(time: int) -> float: + return _MARGIN_LEFT + (time - start) / span_x * plot_w + + def y_of(value: float) -> float: + return _MARGIN_TOP + (1 - (value - low) / (high - low)) * plot_h + + path: list[str] = [] + previous_y = 0.0 + for index, (time, value) in enumerate(points): + x, y = x_of(time), y_of(value) + if index == 0: + path.append(f"M{x:.2f},{y:.2f}") + else: + path.append(f"L{x:.2f},{previous_y:.2f}L{x:.2f},{y:.2f}") + previous_y = y + path.append(f"L{x_of(end):.2f},{previous_y:.2f}") + + parts = [ + ( + f'' + ), + f"{_esc(signal)} over virtual time", + ( + f'' + ), + ] + for fraction in (0.0, 0.5, 1.0): + value = low + (high - low) * (1 - fraction) + y = _MARGIN_TOP + fraction * plot_h + parts.append( + f'' + f'{_esc(_tick_label(value))}' + ) + for fraction in (0.0, 0.5, 1.0): + time = start + round(span_x * fraction) + x = _MARGIN_LEFT + fraction * plot_w + anchor = "start" if fraction == 0 else "end" if fraction == 1 else "middle" + parts.append( + f'{_esc(format_virtual_time(time))}' + ) + for mark in marks: + x = x_of(min(max(mark.result.evaluated_at, start), end)) + kind = "pass" if mark.result.passed else "fail" + parts.append( + f'' + f"{_esc(mark.result.name)}: {kind.upper()} at " + f"{_esc(format_virtual_time(mark.result.evaluated_at))}" + ) + parts.append(f'') + parts.append("") + return "".join(parts) + + +def _signals_section(recorder: RunRecorder, *, start: int, end: int) -> str: + if not recorder.signals: + return ( + '

Signals

' + '

No telemetry was decoded from the bus during this run.

' + "
" + ) + cards: list[str] = [] + for signal in sorted(recorder.signals): + values = recorder.signals[signal] + marks = [a for a in recorder.assertions if a.result.signal == signal] + if all(_is_number(value) for _, value in values): + points = _collapse(values) + body = _plot_svg(signal, points, marks, start=start, end=end) + last = _display(values[-1][1]) + note = f"{len(values)} samples, last {_esc(last)}" + else: + changes: list[tuple[int, object]] = [] + for time, value in values: + if not changes or changes[-1][1] != value: + changes.append((time, value)) + rows = "".join( + f"{_esc(format_virtual_time(time))}{_esc(_display(value))}" + for time, value in changes + ) + body = ( + '' + f"{rows}
virtual timevalue
" + ) + note = f"{len(values)} samples, {len(changes)} value change(s)" + cards.append( + f'

{_esc(signal)}

{note}

{body}
' + ) + legend = ( + '

Vertical lines mark when an assertion on the signal was evaluated: ' + 'PASS FAIL.

' + ) + return f'

Signals

{legend}{"".join(cards)}
' + + +# --- assertions ------------------------------------------------------------------------ + + +def _frame_link(frame_id: int | None) -> str: + if frame_id is None: + return 'signal never seen on the bus' + return f'frame #{frame_id}' + + +def _assertions_section(recorder: RunRecorder) -> str: + if not recorder.assertions: + return ( + '

Assertions

' + '

The scenario made no assertions.

' + ) + failed = [a for a in recorder.assertions if not a.result.passed] + lines: list[str] = [] + if failed: + items = "".join( + f"
  • {_esc(a.result.name)} " + f"({_esc(a.result.signal)}) → {_frame_link(a.evidence_frame)}
  • " + for a in failed + ) + lines.append(f'

    Failed assertions:

      {items}
    ') + rows: list[str] = [] + for number, a in enumerate(recorder.assertions, start=1): + result = a.result + kind = "pass" if result.passed else "fail" + rows.append( + f'{number}' + f'{"PASS" if result.passed else "FAIL"}' + f"{_esc(result.name)}" + f"{_esc(result.signal)} {_esc(result.operator.value)} " + f"{_esc(_display(result.expected))}" + f"{_esc(_display(result.actual))}" + f"{_esc(format_virtual_time(result.evaluated_at))}" + f"{_frame_link(a.evidence_frame)}" + f"{_esc(result.detail or '')}" + ) + return ( + '

    Assertions

    ' + + "".join(lines) + + '' + "" + f"{''.join(rows)}
    #resultnameexpectedactualevaluated atevidencedetail
    " + ) + + +# --- setup ----------------------------------------------------------------------------- + + +def _setup_section(recorder: RunRecorder) -> str: + setup = recorder.setup + if setup is None: + return "" + transport = setup.transport + options = ", ".join( + f"{key}={value}" for key, value in transport.model_dump().items() if key != "type" + ) + node_rows = "".join( + f"{_esc(name)}{node.address}{_esc(node.mode.value)}" + f"{_esc(node.module_type or '—')}{node.max_packet_bytes}" + for name, node in setup.nodes.items() + ) + return ( + '

    Setup

    ' + f"

    Transport: {_esc(transport.type)}" + f"{' (' + _esc(options) + ')' if options else ''}

    " + "" + "" + f"{node_rows}
    nodeCSP addressmodemodulemax_packet_bytes
    " + ) + + +# --- timeline -------------------------------------------------------------------------- + + +def _frame_row(entry: RecordedFrame, names: dict[int, str]) -> str: + record = entry.record + nodes = " ".join(str(a) for a in (record.source, record.destination) if a is not None) + ports = " ".join(str(p) for p in (record.destination_port, record.source_port) if p is not None) + classes = ["frame", f"dir-{record.direction.lower()}"] + if record.kind == "fragment" and record.frames is None: + classes.append("partial") + if record.kind == "undecodable": + classes.append("bad") + if record.kind == "packet": + wire = f"{record.link}" + else: + wire = f"CAN 0x{record.can_id:08X}" if record.can_id is not None else "" + if record.cfp is not None: + wire += f" cfp={record.cfp} pc={record.packet_count} fc={record.frame_count}" + if record.has_header: + route = ( + f"{_esc(_node_label(record.source, names))} → " + f"{_esc(_node_label(record.destination, names))}" + ) + port_text = f"{record.destination_port} ← {record.source_port}" + flags = f"0x{record.flags:02X}" + else: + route = ( + f'sender {_esc(record.sender)} → ' + f"{_esc(_node_label(record.destination, names))}" + if record.kind != "undecodable" + else "" + ) + port_text = "" + flags = "" + notes: list[str] = [] + if record.frames is not None: + notes.append(f"reassembled {record.frames} frames") + if record.annotation: + notes.append(_esc(record.annotation)) + if record.undecodable: + notes.append(f"undecodable: {_esc(record.undecodable)}") + time = record.time if record.time is not None else 0 + return ( + f'' + f'#{entry.frame_id}' + f"{_esc(format_virtual_time(time))}{_esc(record.direction)}" + f"{_esc(wire)}{route}{_esc(port_text)}{_esc(flags)}" + f'{_esc(_hex(record.data))}' + f'{_esc(_hex(record.payload))}{"; ".join(notes)}' + ) + + +def _drop_row(entry: RecordedDrop, names: dict[int, str], number: int) -> str: + nodes = " ".join(str(a) for a in (entry.sender, entry.destination) if a is not None) + where = f"at {_node_label(entry.destination, names)}" if entry.destination is not None else "" + limit = f" (limit {entry.limit})" if entry.limit is not None else "" + return ( + f'' + f"drop{_esc(format_virtual_time(entry.time))}" + f'DROP {_esc(entry.reason)} ' + f"{_esc(where)}{_esc(limit)}: {_esc(entry.detail)}" + ) + + +def _step_row(entry: RecordedStep) -> str: + return ( + f'step {entry.number}' + f"{_esc(format_virtual_time(entry.time))}" + f'{_esc(entry.text)}' + ) + + +def _timeline_section(recorder: RunRecorder, names: dict[int, str]) -> str: + rows: list[str] = [] + drops = 0 + addresses: set[int] = set() + ports: set[int] = set() + for entry in recorder.timeline: + if isinstance(entry, RecordedFrame): + rows.append(_frame_row(entry, names)) + for address in (entry.record.source, entry.record.destination): + if address is not None: + addresses.add(address) + for port in (entry.record.destination_port, entry.record.source_port): + if port is not None: + ports.add(port) + elif isinstance(entry, RecordedDrop): + drops += 1 + rows.append(_drop_row(entry, names, drops)) + for address in (entry.sender, entry.destination): + if address is not None: + addresses.add(address) + else: + rows.append(_step_row(entry)) + node_options = "".join( + f'' for a in sorted(addresses) + ) + port_options = "".join(f'' for p in sorted(ports)) + controls = ( + '
    ' + '' + f'' + f'' + '' + '' + '' + '' + '
    ' + ) + table = ( + '' + "" + "" + f"{''.join(rows)}
    #virtual timedirlink / frameroutedport ← sportflagsdatapayloadnote
    " + ) + return ( + '

    Timeline

    ' + '

    Every frame that crossed the transport boundary, in order, with the ' + "scenario steps and dropped packets. On the in-memory bus a frame appears twice, " + "TX when sent and RX when read back: that round trip is what an assertion observes." + f"

    {controls}{table}
    " + ) + + +# --- page ------------------------------------------------------------------------------ + +_STYLE = """ +:root{color-scheme:light dark;--bg:#fff;--fg:#1b1f24;--muted:#5b6672;--line:#d0d7de; +--card:#f6f8fa;--pass:#1a7f37;--fail:#c4272d;--accent:#0b5cad;--warn:#fff1f0} +@media (prefers-color-scheme:dark){:root{--bg:#0f1318;--fg:#e6edf3;--muted:#9aa7b4; +--line:#30363d;--card:#161b22;--pass:#56d364;--fail:#ff7b72;--accent:#79c0ff;--warn:#2d1517}} +*{box-sizing:border-box}body{margin:0;padding:1.2rem clamp(.8rem,3vw,2rem);background:var(--bg); +color:var(--fg);font:14px/1.45 system-ui,-apple-system,"Segoe UI",Roboto,sans-serif} +h1{margin:0 0 .2rem;font-size:1.5rem}h2{margin:1.8rem 0 .5rem;font-size:1.15rem; +border-bottom:1px solid var(--line);padding-bottom:.25rem}h3{margin:.2rem 0;font-size:.95rem} +a{color:var(--accent)}code,.hex{font:12px/1.4 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace} +.muted{color:var(--muted)}.badge{display:inline-block;padding:.15rem .7rem;border-radius:1rem; +font-weight:700;color:#fff}.badge.pass{background:var(--pass)}.badge.fail,.badge.error{background:var(--fail)} +.chip{display:inline-block;padding:0 .45rem;border-radius:.3rem;font-weight:700;font-size:.8rem; +border:1px solid currentColor}.chip.pass{color:var(--pass)}.chip.fail{color:var(--fail)} +.summary{display:flex;flex-wrap:wrap;gap:.4rem 1.4rem;margin:.6rem 0} +.errorbox,.failures{border:1px solid var(--fail);background:var(--warn);padding:.5rem .9rem; +border-radius:.4rem;margin:.6rem 0}.failures ul{margin:.2rem 0 .2rem 1.2rem;padding:0} +table{border-collapse:collapse;width:100%}th,td{border-bottom:1px solid var(--line); +padding:.25rem .5rem;text-align:left;vertical-align:top}thead th{position:sticky;top:0; +background:var(--card)}.hex{word-break:break-all;max-width:22rem} +tr.a-fail td{background:var(--warn)}tr.partial{opacity:.7}tr.bad td{background:var(--warn)} +tr.step td{background:var(--card);font-weight:600}tr.drop td{background:var(--warn)} +tr.frame:target td,tr.frame.hit td{outline:2px solid var(--accent);background:var(--card)} +tr[hidden]{display:none}.filters{display:flex;flex-wrap:wrap;gap:.4rem 1rem;align-items:center; +margin:.4rem 0 .6rem;padding:.4rem .6rem;background:var(--card);border-radius:.4rem} +.filters select,.filters input[type=search]{font:inherit;color:inherit;background:var(--bg); +border:1px solid var(--line);border-radius:.25rem;padding:.1rem .3rem} +.card{background:var(--card);border:1px solid var(--line);border-radius:.5rem; +padding:.5rem .8rem;margin:.6rem 0}.plot{width:100%;max-width:760px;height:auto} +.plot-bg{fill:var(--bg);stroke:var(--line)}.grid{stroke:var(--line);stroke-dasharray:3 3} +.tick{fill:var(--muted);font-size:11px}.series{fill:none;stroke:var(--accent);stroke-width:2} +.mark{stroke-width:1.5;stroke-dasharray:4 3}.mark-pass{stroke:var(--pass)}.mark-fail{stroke:var(--fail)} +table.changes{width:auto}footer{margin-top:2rem;color:var(--muted);font-size:.85rem} +""" + +_SCRIPT = """ +(function(){ +var rows=[].slice.call(document.querySelectorAll('#timeline tbody tr')); +var dir=document.getElementById('f-dir'),node=document.getElementById('f-node'), +port=document.getElementById('f-port'),text=document.getElementById('f-text'), +count=document.getElementById('f-count'); +var kinds={frame:document.getElementById('k-frame'),drop:document.getElementById('k-drop'), +step:document.getElementById('k-step')}; +if(!rows.length||!dir){return;} +function has(list,value){return (' '+list+' ').indexOf(' '+value+' ')>=0;} +function apply(){ +var shown=0,needle=text.value.toLowerCase(); +rows.forEach(function(row){ +var kind=row.getAttribute('data-kind'),show=kinds[kind].checked; +if(show&&kind==='frame'){ +if(dir.value&&row.getAttribute('data-dir')!==dir.value){show=false;} +if(show&&node.value&&!has(row.getAttribute('data-nodes'),node.value)){show=false;} +if(show&&port.value&&!has(row.getAttribute('data-ports'),port.value)){show=false;} +}else if(show&&kind==='drop'&&node.value&&!has(row.getAttribute('data-nodes'),node.value)){show=false;} +if(show&&needle&&row.textContent.toLowerCase().indexOf(needle)<0){show=false;} +row.hidden=!show;if(show){shown++;} +}); +count.textContent='showing '+shown+' of '+rows.length; +} +function reveal(id){ +var row=id&&document.getElementById(id); +if(!row||!row.hidden){return;} +dir.value='';node.value='';port.value='';text.value=''; +Object.keys(kinds).forEach(function(k){kinds[k].checked=true;}); +apply(); +} +[dir,node,port,text].forEach(function(el){el.addEventListener('input',apply);}); +Object.keys(kinds).forEach(function(k){kinds[k].addEventListener('change',apply);}); +window.addEventListener('hashchange',function(){reveal(location.hash.slice(1));}); +document.addEventListener('click',function(e){ +var a=e.target&&e.target.closest&&e.target.closest('a[href^="#"]'); +if(a){reveal(a.getAttribute('href').slice(1));} +}); +apply();reveal(location.hash.slice(1)); +})(); +""" + + +def render_report( + recorder: RunRecorder, + result: ScenarioRunResult | None, + *, + error: str | None = None, +) -> str: + """Render the recording as a complete HTML document (a string, ending in a newline). + + ``result`` is ``None`` when the run aborted before producing one; ``error`` is the + message then, and the report shows what was recorded up to that point. + """ + + names = _node_names(recorder) + scenario_name = ( + result.scenario_name + if result is not None + else (recorder.scenario.name if recorder.scenario is not None else "scenario") + ) + if error is not None: + status, label = "error", "ERROR" + elif result is not None and result.passed: + status, label = "pass", "PASS" + else: + status, label = "fail", "FAIL" + + last_time = max( + [entry.time for entry in recorder.timeline if not isinstance(entry, RecordedFrame)] + + [ + entry.record.time + for entry in recorder.timeline + if isinstance(entry, RecordedFrame) and entry.record.time is not None + ] + + [time for series in recorder.signals.values() for time, _ in series] + + [0] + ) + start = result.started_at if result is not None else 0 + end = max(result.finished_at if result is not None else last_time, start + 1) + + passed = sum(1 for a in recorder.assertions if a.result.passed) + failed = len(recorder.assertions) - passed + drops = sum(1 for entry in recorder.timeline if isinstance(entry, RecordedDrop)) + summary = [ + f"assertions: {passed} passed, {failed} failed", + f"frames traced: {len(recorder.frames)}", + f"dropped packets: {drops}", + ( + f"virtual time: {_esc(format_virtual_time(start))} → " + f"{_esc(format_virtual_time(end))}" + ), + ] + description = "" + if recorder.scenario is not None and recorder.scenario.description: + description = f'

    {_esc(recorder.scenario.description)}

    ' + error_box = ( + f'
    The run aborted: {_esc(error)}
    ' + 'The report shows what was recorded before the error.
    ' + if error is not None + else "" + ) + + body = "".join( + ( + f"

    {_esc(scenario_name)}

    ", + f'

    {label}

    ', + description, + f'
    {"".join(summary)}
    ', + error_box, + _assertions_section(recorder), + _setup_section(recorder), + _signals_section(recorder, start=start, end=end), + _timeline_section(recorder, names), + ( + f"
    cubesat-testbed {_esc(_version())}. Times are virtual; this report " + "contains no wall-clock time, so the same run renders the same file.
    " + ), + ) + ) + return ( + '\n' + '' + f"{_esc(scenario_name)} — {label} — CubeSat Testbed report" + f"{body}\n" + ) + + +def write_report( + path: Path, + recorder: RunRecorder, + result: ScenarioRunResult | None, + *, + error: str | None = None, +) -> None: + """Write :func:`render_report` to ``path`` as UTF-8, creating parent directories.""" + + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(render_report(recorder, result, error=error), encoding="utf-8") + + +__all__ = ["render_report", "write_report"] diff --git a/src/cubesat_testbed/report/recorder.py b/src/cubesat_testbed/report/recorder.py new file mode 100644 index 0000000..562bda0 --- /dev/null +++ b/src/cubesat_testbed/report/recorder.py @@ -0,0 +1,169 @@ +"""Records what happened in a run, for the HTML report. + +The recorder is an observer, in the same sense as ``--trace``: it is handed what the +run already produces (every frame crossing the transport boundary as a +:class:`~cubesat_testbed.transport.tracing.FrameRecord`, every dropped packet, every +decoded telemetry value, every assertion result) and never feeds anything back. It +holds no clock and no randomness, so two runs with the same inputs record the same +thing. It imports nothing from the scenario runner, which in turn depends on it. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from cubesat_testbed.transport.base import TransportEnvelope +from cubesat_testbed.transport.tracing import FrameRecord + +if TYPE_CHECKING: # pragma: no cover - typing only, avoids a runtime import cycle + from cubesat_testbed.config import ScenarioScript, TestbedConfig + from cubesat_testbed.scenario.assertions import AssertionResult + from cubesat_testbed.scenario.runner import DroppedPacket + + +@dataclass(frozen=True, slots=True) +class RecordedFrame: + """One traced frame, with the id the report links to (``f``).""" + + frame_id: int + record: FrameRecord + envelope_sequence: int + + +@dataclass(frozen=True, slots=True) +class RecordedDrop: + """A packet a node or a link discarded, at its virtual time.""" + + time: int + destination: int | None + sender: int | None + reason: str + detail: str + limit: int | None + + +@dataclass(frozen=True, slots=True) +class RecordedStep: + """A scenario step starting, so the timeline shows what the scenario was doing.""" + + time: int + number: int + text: str + + +@dataclass(frozen=True, slots=True) +class RecordedAssertion: + """An assertion result and the frame that carried the value it was judged on. + + ``evidence_frame`` is the id of the frame (the last fragment, for a multi-frame + packet) from which the signal's latest value was decoded when the assertion was + evaluated: for a passing assertion the frame that satisfied it, for a failing one + the frame whose value did not match. ``None`` means the signal was never seen on + the bus. + """ + + result: AssertionResult + evidence_frame: int | None + + +TimelineEntry = RecordedFrame | RecordedDrop | RecordedStep + + +class RunRecorder: + """Collects the frames, drops, steps, signal values and assertions of one run.""" + + def __init__(self) -> None: + self.setup: TestbedConfig | None = None + self.scenario: ScenarioScript | None = None + self.timeline: list[TimelineEntry] = [] + self.frames: list[RecordedFrame] = [] + self.assertions: list[RecordedAssertion] = [] + self.signals: dict[str, list[tuple[int, object]]] = {} + self._rx_frame_by_sequence: dict[int, int] = {} + + def begin(self, setup: TestbedConfig, scenario: ScenarioScript) -> None: + """Remember what is being run; the report describes both.""" + + self.setup = setup + self.scenario = scenario + + # --- frames --------------------------------------------------------------------- + + def observe_frame(self, record: FrameRecord, envelope: TransportEnvelope) -> None: + """Frame observer for :class:`~cubesat_testbed.transport.TracingTransportAdapter`.""" + + frame = RecordedFrame( + frame_id=len(self.frames) + 1, + record=record, + envelope_sequence=envelope.sequence, + ) + self.frames.append(frame) + self.timeline.append(frame) + if record.direction == "RX": + self._rx_frame_by_sequence[envelope.sequence] = frame.frame_id + + def rx_frame_id(self, envelope: TransportEnvelope) -> int | None: + """Id of the ``RX`` row for ``envelope``: the frame the runner is routing now.""" + + return self._rx_frame_by_sequence.get(envelope.sequence) + + # --- the rest of the run -------------------------------------------------------- + + def step(self, time: int, number: int, step: Any) -> None: + self.timeline.append(RecordedStep(time=time, number=number, text=describe_step(step))) + + def drop(self, dropped: DroppedPacket) -> None: + self.timeline.append( + RecordedDrop( + time=dropped.time, + destination=dropped.destination, + sender=dropped.sender, + reason=dropped.reason, + detail=dropped.detail, + limit=dropped.limit, + ) + ) + + def signal(self, time: int, signal: str, value: object) -> None: + self.signals.setdefault(signal, []).append((time, value)) + + def assertion(self, result: AssertionResult, evidence_frame: int | None) -> None: + self.assertions.append(RecordedAssertion(result=result, evidence_frame=evidence_frame)) + + +_DURATION_KEYS = frozenset({"virtual_time", "duration", "timeout"}) + + +def describe_step(step: Any) -> str: + """One line for a scenario step: ``action key=value ...``, durations in seconds.""" + + data = step.model_dump(exclude_none=True) + action = str(data.pop("action")) + parts = [action] + for key, value in data.items(): + if key in _DURATION_KEYS and isinstance(value, int): + parts.append(f"{key}={format_virtual_time(value)}") + else: + parts.append(f"{key}={getattr(value, 'value', value)}") + return " ".join(parts) + + +def format_virtual_time(microseconds: int) -> str: + """Virtual microseconds as seconds with six decimals, exact and locale-free.""" + + sign = "-" if microseconds < 0 else "" + whole, fraction = divmod(abs(microseconds), 1_000_000) + return f"{sign}{whole}.{fraction:06d} s" + + +__all__ = [ + "RecordedAssertion", + "RecordedDrop", + "RecordedFrame", + "RecordedStep", + "RunRecorder", + "TimelineEntry", + "describe_step", + "format_virtual_time", +] diff --git a/src/cubesat_testbed/scenario/runner.py b/src/cubesat_testbed/scenario/runner.py index 597e31c..12e312e 100644 --- a/src/cubesat_testbed/scenario/runner.py +++ b/src/cubesat_testbed/scenario/runner.py @@ -63,6 +63,7 @@ decode_telemetry_value, encode_telemetry_value, ) +from cubesat_testbed.report.recorder import RunRecorder from cubesat_testbed.scenario.assertions import ( AssertionResult, TelemetryAssertion, @@ -192,6 +193,8 @@ class ScenarioRuntime: clock: Clock = field(default_factory=VirtualClock) trace: TextIO | None = None """Stream ``DROP`` lines are written to when ``--trace`` is on.""" + recorder: RunRecorder | None = None + """Observer for the HTML report; never read back by the run.""" def module(self, name: str) -> SimulatedModule: """Return a configured simulated module by node name.""" @@ -330,6 +333,7 @@ def __init__( self._reassemblers: dict[int, CspReassembler] = {} self._senders: dict[int, PacketSender] = {} self._dropped_packets: list[DroppedPacket] = [] + self._signal_evidence: dict[str, int | None] = {} self.runtime.engine.add_handler(EventKind.TELEMETRY, self._record_telemetry) self._arm_physical_step() @@ -345,8 +349,13 @@ def run(self, scenario: ScenarioScript) -> ScenarioRunResult: started_at = self.runtime.engine.now self._assertion_results = [] self._dropped_packets = [] + recorder = self.runtime.recorder + if recorder is not None: + recorder.begin(self.runtime.setup, scenario) for index, step in enumerate(scenario.steps, start=1): + if recorder is not None: + recorder.step(self.runtime.engine.now, index, step) if isinstance(step, WaitStep): self.wait(step.virtual_time) elif isinstance(step, InjectFaultStep): @@ -513,6 +522,8 @@ def assert_step(self, step: AssertStep, *, index: int) -> AssertionResult: self.wait(next_stop - self.runtime.engine.now) self._assertion_results.append(result) + if self.runtime.recorder is not None: + self.runtime.recorder.assertion(result, self._signal_evidence.get(assertion.signal)) print(format_assertion_result(result), file=self._output) self.runtime.engine.schedule_assertion( result.name, passed=result.passed, detail=result.detail @@ -524,6 +535,10 @@ def _record_telemetry(self, event: SimulationEvent, _engine: DiscreteEventEngine if not isinstance(event.payload, TelemetryPayload): raise ScenarioRuntimeError("telemetry events must carry a TelemetryPayload") self._latest_telemetry[event.payload.signal] = event.payload.value + if self.runtime.recorder is not None: + self.runtime.recorder.signal( + self.runtime.engine.now, event.payload.signal, event.payload.value + ) def _arm_physical_step(self) -> None: """Schedule the recurring engine timer that drives ticks/cycles/telemetry. @@ -744,6 +759,8 @@ def _record_drop(self, event: ReassemblyEvent, destination: int | None) -> None: def _append_drop(self, dropped: DroppedPacket) -> None: self._dropped_packets.append(dropped) + if self.runtime.recorder is not None: + self.runtime.recorder.drop(dropped) if self.runtime.trace is not None: self.runtime.trace.write(format_drop_trace_line(dropped) + "\n") self.runtime.trace.flush() @@ -756,6 +773,12 @@ def _route_packet(self, packet: CspPacketView, envelope: TransportEnvelope) -> N telemetry_route = self._telemetry_route_for_packet(packet) if telemetry_route is not None: + if self.runtime.recorder is not None: + # The frame this value is decoded from, for the report to link an + # assertion to; it only describes the run, nothing reads it back. + self._signal_evidence[telemetry_route.signal] = self.runtime.recorder.rx_frame_id( + envelope + ) self._deliver_telemetry(telemetry_route, packet) return @@ -900,6 +923,7 @@ def build_runtime( obc_rules: Mapping[str, Iterable[ObcPeerRule]] | None = None, clock: Clock | None = None, trace: TextIO | None = None, + recorder: RunRecorder | None = None, ) -> ScenarioRuntime: """Build a runtime from a validated setup config. @@ -927,6 +951,10 @@ def build_runtime( transport in a :class:`~cubesat_testbed.transport.TracingTransportAdapter` and get one decoded line per frame crossing the bus. Purely observational: it changes neither scenario semantics nor determinism. + + Pass ``recorder`` (a :class:`~cubesat_testbed.report.RunRecorder`) to record the + run for the HTML report. It rides on the same transport wrapper as ``trace`` and + is equally passive; either, both or neither may be given. """ engine = DiscreteEventEngine() @@ -942,7 +970,7 @@ def build_runtime( ), max_packet_bytes=max(node.max_packet_bytes for node in setup.nodes.values()), ) - if trace is not None: + if trace is not None or recorder is not None: # Wrapping here, rather than tracing inside the runner, is what makes # frames an OBC Peer module sends straight to the transport visible too. transport = TracingTransportAdapter( @@ -950,6 +978,7 @@ def build_runtime( stream=trace, now=lambda: engine.now, annotate=_build_frame_annotator(command_routes, telemetry_routes), + observer=None if recorder is None else recorder.observe_frame, ) # Both the construction order and the per-tick order come from the module @@ -1007,6 +1036,7 @@ def build_runtime( tick_order=tick_order, clock=VirtualClock() if clock is None else clock, trace=trace, + recorder=recorder, ) @@ -1036,6 +1066,7 @@ def run_scenario( obc_rules: Mapping[str, Iterable[ObcPeerRule]] | None = None, clock: Clock | None = None, trace: TextIO | None = None, + recorder: RunRecorder | None = None, ) -> ScenarioRunResult: """Build a runtime (in-memory or SocketCAN, per setup) and run a scenario. @@ -1048,7 +1079,7 @@ def run_scenario( the same stream as ``output``, which carries the assertion report. """ - runtime = build_runtime(setup, obc_rules=obc_rules, clock=clock, trace=trace) + runtime = build_runtime(setup, obc_rules=obc_rules, clock=clock, trace=trace, recorder=recorder) try: return ScenarioRunner(runtime, output=output).run(scenario) finally: @@ -1063,6 +1094,7 @@ def run_scenario_files( obc_rules: Mapping[str, Iterable[ObcPeerRule]] | None = None, clock: Clock | None = None, trace: TextIO | None = None, + recorder: RunRecorder | None = None, ) -> ScenarioRunResult: """Load setup/scenario files, validate references, and execute them.""" @@ -1075,6 +1107,7 @@ def run_scenario_files( obc_rules=obc_rules, clock=clock, trace=trace, + recorder=recorder, ) diff --git a/src/cubesat_testbed/transport/tracing.py b/src/cubesat_testbed/transport/tracing.py index 5046025..533ae00 100644 --- a/src/cubesat_testbed/transport/tracing.py +++ b/src/cubesat_testbed/transport/tracing.py @@ -41,7 +41,8 @@ from __future__ import annotations -from collections.abc import Callable +from collections.abc import Callable, Mapping +from dataclasses import dataclass from typing import TextIO from cubesat_testbed.protocol.csp_v2 import ( @@ -74,6 +75,56 @@ VirtualTimeSource = Callable[[], int] """Return the current virtual time, in microseconds, for the trace timestamp.""" + +@dataclass(frozen=True, slots=True) +class FrameRecord: + """One frame as the trace sees it, decoded into fields instead of text. + + The ``--trace`` line and the HTML report are both rendered from this, so what the + report shows is exactly what the trace would print. ``kind`` is ``"packet"`` for + a packet a link carries whole, ``"frame"`` for a CAN frame that is a whole packet, + ``"fragment"`` for one CAN frame of a multi-frame packet and ``"undecodable"`` + for a frame the codec rejected. Header fields are ``None`` where the frame does + not carry them (a continuation fragment has only ``dst`` and ``sender``). + """ + + direction: str + kind: str + time: int | None = None + link: str | None = None + can_id: int | None = None + dlc: int | None = None + cfp: str | None = None + packet_count: int | None = None + frame_count: int | None = None + priority: int | None = None + source: int | None = None + destination: int | None = None + destination_port: int | None = None + source_port: int | None = None + flags: int | None = None + sender: int | None = None + data: bytes = b"" + payload: bytes | None = None + frames: int | None = None + """Frames in the reassembled packet, set on the fragment that completes it.""" + annotation: str | None = None + undecodable: str | None = None + + @property + def has_header(self) -> bool: + """Whether the full CSP header (source, ports, flags) is known for this frame.""" + + return self.source is not None and self.destination_port is not None + + +FrameObserver = Callable[[FrameRecord, TransportEnvelope], None] +"""Called with every traced frame, after it has crossed the transport boundary. + +Observability only, like the trace itself: it must not raise and must not change +delivery, ordering or virtual time. +""" + TRACE_PREFIX = "trace" """Leading token of every trace line, so a trace is greppable out of a log.""" @@ -83,21 +134,25 @@ class TracingTransportAdapter(TransportAdapter): ``stream`` receives one line per frame and should be ``stderr`` (or a file) rather than ``stdout``, so a trace composes with the CLI's machine-readable - ``--json`` output instead of corrupting it. + ``--json`` output instead of corrupting it. It may be ``None`` when only an + ``observer`` wants the frames (the HTML report): the frames are still decoded + the same way, there is just nowhere to print them. """ def __init__( self, inner: TransportAdapter, *, - stream: TextIO, + stream: TextIO | None = None, now: VirtualTimeSource | None = None, annotate: FrameAnnotator | None = None, + observer: FrameObserver | None = None, ) -> None: self._inner = inner self._stream = stream self._now = now self._annotate = annotate + self._observer = observer self._reassemblers = { "TX": CspReassembler(max_packet_bytes=None), "RX": CspReassembler(max_packet_bytes=None), @@ -160,17 +215,19 @@ def close(self) -> None: self._inner.close() def _trace(self, direction: str, envelope: TransportEnvelope) -> None: - self._stream.write( - format_trace_line( - direction, - envelope, - now=self._now_value(), - annotate=self._annotate, - reassembler=self._reassemblers[direction], - ) + record = describe_frame( + direction, + envelope, + now=self._now_value(), + annotate=self._annotate, + reassembler=self._reassemblers[direction], ) - self._stream.write("\n") - self._stream.flush() + if self._stream is not None: + self._stream.write(format_frame_record(record)) + self._stream.write("\n") + self._stream.flush() + if self._observer is not None: + self._observer(record, envelope) def _now_value(self) -> int | None: return None if self._now is None else self._now() @@ -197,124 +254,172 @@ def format_trace_line( different debugging jobs. """ + return format_frame_record( + describe_frame(direction, envelope, now=now, annotate=annotate, reassembler=reassembler) + ) + + +def describe_frame( + direction: str, + envelope: TransportEnvelope, + *, + now: int | None = None, + annotate: FrameAnnotator | None = None, + reassembler: CspReassembler | None = None, +) -> FrameRecord: + """Decode one frame into a :class:`FrameRecord`; never raises on a bad frame.""" + frame = envelope.frame - parts = [TRACE_PREFIX] - if now is not None: - parts.append(f"t={now}") - parts.append(direction) if isinstance(frame, CspPacketFrame): - return _format_packet_line(parts, frame, annotate=annotate) - parts.append(f"can_id=0x{frame.can_id:08X}") - parts.append(f"dlc={frame.dlc}") + fields = frame.fields + return FrameRecord( + direction=direction, + kind="packet", + time=now, + link=frame.link or "local", + priority=fields.priority, + source=fields.source, + destination=fields.destination, + destination_port=fields.destination_port, + source_port=fields.source_port, + flags=fields.flags, + data=frame.wire, + payload=frame.payload, + annotation=None if annotate is None else annotate(frame), + ) can_fields = decode_can_id(frame.can_id) if frame.is_extended_id else None + base = { + "direction": direction, + "time": now, + "can_id": frame.can_id, + "dlc": frame.dlc, + "data": frame.data, + } if can_fields is not None and not (can_fields.begin and can_fields.end): - return _format_fragment_line( - parts, frame, can_fields, now=now, annotate=annotate, reassembler=reassembler + return _describe_fragment( + frame, can_fields, base, now=now, annotate=annotate, reassembler=reassembler ) try: packet = _decode_frame(frame) except CspV2CodecError as exc: - parts.append(f"data={_hex(frame.data)}") - parts.append(f"undecodable={str(exc)!r}") - return " ".join(parts) + return FrameRecord(kind="undecodable", undecodable=str(exc), **base) # type: ignore[arg-type] fields = packet.fields - parts.extend( - ( - f"pri={fields.priority}", - f"src={fields.source}", - f"dst={fields.destination}", - f"dport={fields.destination_port}", - f"sport={fields.source_port}", - f"flags=0x{fields.flags:02X}", - f"data={_hex(frame.data)}", - f"payload={_hex(packet.payload)}", - ) + return FrameRecord( + kind="frame", + priority=fields.priority, + source=fields.source, + destination=fields.destination, + destination_port=fields.destination_port, + source_port=fields.source_port, + flags=fields.flags, + payload=packet.payload, + annotation=None if annotate is None else annotate(packet), + **base, # type: ignore[arg-type] ) - annotation = None if annotate is None else annotate(packet) - if annotation: - parts.append(annotation) - return " ".join(parts) - -def _format_packet_line( - parts: list[str], - frame: CspPacketFrame, - *, - annotate: FrameAnnotator | None, -) -> str: - """Describe a packet that a link carries whole; see the module docstring.""" - - fields = frame.fields - parts.append(f"link={frame.link or 'local'}") - parts.extend( - ( - f"pri={fields.priority}", - f"src={fields.source}", - f"dst={fields.destination}", - f"dport={fields.destination_port}", - f"sport={fields.source_port}", - f"flags=0x{fields.flags:02X}", - f"data={_hex(frame.wire)}", - f"payload={_hex(frame.payload)}", - ) - ) - annotation = None if annotate is None else annotate(frame) - if annotation: - parts.append(annotation) - return " ".join(parts) - - -def _format_fragment_line( - parts: list[str], +def _describe_fragment( frame: CspCanFrame, can_fields: CspCanIdFields, + base: Mapping[str, object], *, now: int | None, annotate: FrameAnnotator | None, reassembler: CspReassembler | None, -) -> str: +) -> FrameRecord: """Describe one frame of a multi-frame packet; see the module docstring.""" position = "begin" if can_fields.begin else "end" if can_fields.end else "more" - parts.append(f"cfp={position}") - parts.append(f"pc={can_fields.packet_count}") - parts.append(f"fc={can_fields.frame_count}") - + cfp = { + "cfp": position, + "packet_count": can_fields.packet_count, + "frame_count": can_fields.frame_count, + } + header: dict[str, object] = {} if can_fields.begin: try: - header = decode_first_frame_header(frame) + first = decode_first_frame_header(frame) except CspV2CodecError as exc: - parts.append(f"data={_hex(frame.data)}") - parts.append(f"undecodable={str(exc)!r}") - return " ".join(parts) - parts.extend( - ( - f"pri={header.priority}", - f"src={header.source}", - f"dst={header.destination}", - f"dport={header.destination_port}", - f"sport={header.source_port}", - f"flags=0x{header.flags:02X}", - ) - ) + return FrameRecord(kind="undecodable", undecodable=str(exc), **base, **cfp) # type: ignore[arg-type] + header = { + "priority": first.priority, + "source": first.source, + "destination": first.destination, + "destination_port": first.destination_port, + "source_port": first.source_port, + "flags": first.flags, + } else: - parts.append(f"dst={can_fields.destination}") - parts.append(f"sender={can_fields.sender}") - parts.append(f"data={_hex(frame.data)}") + header = {"destination": can_fields.destination, "sender": can_fields.sender} + frames: int | None = None + payload: bytes | None = None + annotation: str | None = None if reassembler is not None: for event in reassembler.feed(frame, now=0 if now is None else now): packet = event.packet if event.kind == "complete" and packet is not None: - parts.append(f"frames={packet.frame_count}") - parts.append(f"payload={_hex(packet.payload)}") + frames = packet.frame_count + payload = packet.payload annotation = None if annotate is None else annotate(packet) - if annotation: - parts.append(annotation) + return FrameRecord( + kind="fragment", + frames=frames, + payload=payload, + annotation=annotation, + **base, # type: ignore[arg-type] + **cfp, # type: ignore[arg-type] + **header, # type: ignore[arg-type] + ) + + +def format_frame_record(record: FrameRecord) -> str: + """Render a :class:`FrameRecord` as the single ``key=value`` ``--trace`` line.""" + + parts = [TRACE_PREFIX] + if record.time is not None: + parts.append(f"t={record.time}") + parts.append(record.direction) + + if record.kind == "packet": + parts.append(f"link={record.link}") + else: + parts.append(f"can_id=0x{record.can_id:08X}") + parts.append(f"dlc={record.dlc}") + if record.cfp is not None: + parts.append(f"cfp={record.cfp}") + parts.append(f"pc={record.packet_count}") + parts.append(f"fc={record.frame_count}") + + if record.kind == "undecodable": + parts.append(f"data={_hex(record.data)}") + parts.append(f"undecodable={record.undecodable!r}") + return " ".join(parts) + + if record.has_header: + parts.extend( + ( + f"pri={record.priority}", + f"src={record.source}", + f"dst={record.destination}", + f"dport={record.destination_port}", + f"sport={record.source_port}", + f"flags=0x{record.flags:02X}", + ) + ) + else: # a continuation fragment knows only where it is going and who sent it + parts.append(f"dst={record.destination}") + parts.append(f"sender={record.sender}") + parts.append(f"data={_hex(record.data)}") + if record.frames is not None: + parts.append(f"frames={record.frames}") + if record.payload is not None: + parts.append(f"payload={_hex(record.payload)}") + if record.annotation: + parts.append(record.annotation) return " ".join(parts) @@ -327,7 +432,11 @@ def _hex(data: bytes) -> str: __all__ = [ "TRACE_PREFIX", "FrameAnnotator", + "FrameObserver", + "FrameRecord", "TracingTransportAdapter", "VirtualTimeSource", + "describe_frame", + "format_frame_record", "format_trace_line", ] diff --git a/tests/test_html_report.py b/tests/test_html_report.py new file mode 100644 index 0000000..aefeab7 --- /dev/null +++ b/tests/test_html_report.py @@ -0,0 +1,565 @@ +"""The HTML report: structure, determinism, evidence links and the CLI flag.""" + +from __future__ import annotations + +import io +import json +import re +import shutil +import subprocess +from html.parser import HTMLParser +from pathlib import Path + +import pytest + +import cubesat_testbed.scenario.runner as runner_module +from cubesat_testbed.config import load_scenario, load_testbed_config, parse_scenario +from cubesat_testbed.main import main +from cubesat_testbed.report import RunRecorder, describe_step, format_virtual_time, render_report +from cubesat_testbed.scenario import run_scenario +from tests.example_paths import DEFAULT_SCENARIO, DEFAULT_SETUP +from tests.test_multi_frame_runtime import SCENARIO as MULTI_SCENARIO +from tests.test_multi_frame_runtime import _setup_text as multi_setup_text + +FAILING = """ +name: Deliberately failing scenario +steps: + - action: inject_fault + type: state_override + target: eps.model.battery_percent + value: 25 + duration: 5s + - action: wait + virtual_time: 3s + - action: assert + signal: eps.telemetry.battery_percent + op: "==" + value: 25.0 + timeout: 1s + - action: assert + signal: eps.telemetry.battery_percent + op: "==" + value: 99.0 + timeout: 1s +""" + +_VOID = {"meta", "link", "br", "hr", "img", "input", "col", "area", "base", "source", "wbr"} + + +def _record(setup_path: Path = DEFAULT_SETUP, scenario_text: str | None = None): # type: ignore[no-untyped-def] + setup = load_testbed_config(setup_path) + scenario = ( + load_scenario(DEFAULT_SCENARIO, setup=setup) + if scenario_text is None + else parse_scenario(scenario_text, setup=setup) + ) + recorder = RunRecorder() + result = run_scenario(scenario, setup, output=io.StringIO(), recorder=recorder) + return recorder, result + + +class _Rows(HTMLParser): + """Collects the report's timeline rows, anchors and script, and checks tag balance.""" + + def __init__(self) -> None: + super().__init__() + self.stack: list[str] = [] + self.rows: dict[str, dict[str, str]] = {} + self.links: list[str] = [] + self.script = "" + self._in_script = False + self._row: str | None = None + self._row_text: list[str] = [] + self.errors: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + attributes = {k: (v or "") for k, v in attrs} + if tag not in _VOID: + self.stack.append(tag) + if tag == "script": + self._in_script = True + if tag == "a" and attributes.get("href", "").startswith("#"): + self.links.append(attributes["href"][1:]) + if tag == "tr" and "data-kind" in attributes: + self._row = attributes.get("id", f"anon{len(self.rows)}") + self.rows[self._row] = {**attributes, "text": ""} + self._row_text = [] + + def handle_endtag(self, tag: str) -> None: + if tag == "script": + self._in_script = False + if tag in _VOID: + return + if not self.stack or self.stack[-1] != tag: + self.errors.append(f"unbalanced (open: {self.stack[-3:]})") + else: + self.stack.pop() + if tag in {"td", "th"} and self._row is not None: + self._row_text.append(" ") # cells read as separate words, as on screen + if tag == "tr" and self._row is not None: + self.rows[self._row]["text"] = " ".join("".join(self._row_text).split()) + self._row = None + + def handle_data(self, data: str) -> None: + if self._in_script: + self.script += data + if self._row is not None: + self._row_text.append(data) + + +def _parse(report: str) -> _Rows: + parser = _Rows() + parser.feed(report) + parser.close() + return parser + + +# --- structure ------------------------------------------------------------------------- + + +def test_a_passing_run_renders_a_well_formed_self_contained_page() -> None: + recorder, result = _record() + + report = render_report(recorder, result) + parsed = _parse(report) + + assert report.startswith("") and report.endswith("\n") + assert parsed.errors == [] and parsed.stack == [] + assert ">PASS" in report and "EPS Low Battery Protection Test" in report + # Nothing is fetched at view time: no external URL, stylesheet link, import or font. + assert not re.search(r"https?://", report) + assert " None: + setup = load_testbed_config(DEFAULT_SETUP) + scenario = load_scenario(DEFAULT_SCENARIO, setup=setup) + + plain = run_scenario(scenario, setup, output=io.StringIO()) + recorded = run_scenario(scenario, setup, output=io.StringIO(), recorder=RunRecorder()) + + assert recorded == plain + + +# --- the point of the feature: an assertion leads to its frame ------------------------- + + +def test_a_failed_assertion_links_to_the_frame_carrying_the_value_it_failed_on() -> None: + recorder, result = _record(scenario_text=FAILING) + parsed = _parse(render_report(recorder, result)) + + assert not result.passed + failed = [a for a in recorder.assertions if not a.result.passed] + assert len(failed) == 1 and failed[0].evidence_frame is not None + row = parsed.rows[f"f{failed[0].evidence_frame}"] + # The linked row is the RX of the telemetry frame whose decoded value was 25.0, not 99.0. + assert row["data-dir"] == "RX" + assert "telemetry eps.telemetry.battery_percent=25.0" in row["text"] + assert f"f{failed[0].evidence_frame}" in parsed.links + + +def test_a_passing_assertion_links_to_the_frame_that_satisfied_it() -> None: + recorder, result = _record() + (assertion,) = recorder.assertions + + assert result.passed and assertion.evidence_frame is not None + parsed = _parse(render_report(recorder, result)) + row = parsed.rows[f"f{assertion.evidence_frame}"] + assert "telemetry payload.telemetry.power_status='offline'" in row["text"] + + +def test_every_evidence_link_points_at_a_real_row() -> None: + recorder, result = _record(scenario_text=FAILING) + parsed = _parse(render_report(recorder, result)) + + assert parsed.links and all(target in parsed.rows for target in parsed.links) + + +def test_a_signal_never_seen_on_the_bus_has_no_link_and_says_so() -> None: + scenario = """ +name: Never observed +steps: + - action: wait + virtual_time: 1s + - action: assert + signal: payload.telemetry.power_status + op: "==" + value: offline + timeout: 1s +""" + setup_text = DEFAULT_SETUP.read_text(encoding="utf-8") + setup_text = setup_text.replace( + 'mode = "simulated"\nmodule_type = "simple_payload"', 'mode = "software"' + ) + path = DEFAULT_SETUP.parent / "never_seen_setup.toml" + try: + path.write_text(setup_text, encoding="utf-8") + recorder, result = _record(setup_path=path, scenario_text=scenario) + finally: + path.unlink(missing_ok=True) + + (assertion,) = recorder.assertions + report = render_report(recorder, result) + assert not result.passed and assertion.evidence_frame is None + assert "signal never seen on the bus" in report + + +# --- other content --------------------------------------------------------------------- + + +def test_a_multi_frame_packet_shows_each_fragment_and_the_reassembled_one() -> None: + from cubesat_testbed.config import parse_testbed_config + + setup = parse_testbed_config(multi_setup_text()) + scenario = parse_scenario(MULTI_SCENARIO, setup=setup) + recorder = RunRecorder() + result = run_scenario(scenario, setup, output=io.StringIO(), recorder=recorder) + + parsed = _parse(render_report(recorder, result)) + battery = [a for a in recorder.assertions if a.result.signal == "eps.telemetry.battery_percent"] + + assert result.passed and battery[0].evidence_frame is not None + last = parsed.rows[f"f{battery[0].evidence_frame}"] + first = parsed.rows[f"f{battery[0].evidence_frame - 1}"] + assert "reassembled 2 frames" in last["text"] and "cfp=end" in last["text"] + assert "cfp=begin" in first["text"] and "reassembled" not in first["text"] + assert "partial" in first["class"] + + +def test_dropped_packets_and_scenario_steps_are_on_the_timeline() -> None: + from cubesat_testbed.config import parse_testbed_config + + text = multi_setup_text("max_packet_bytes = 8") + setup = parse_testbed_config(text) + scenario = parse_scenario(MULTI_SCENARIO, setup=setup) + recorder = RunRecorder() + result = run_scenario(scenario, setup, output=io.StringIO(), recorder=recorder) + + report = render_report(recorder, result) + parsed = _parse(report) + kinds = [row["data-kind"] for row in parsed.rows.values()] + + assert result.dropped_packets + assert kinds.count("drop") == len(result.dropped_packets) + assert kinds.count("step") == len(scenario.steps) + assert "overflow" in report and "max_packet_bytes=8" in report + + +def test_numeric_signals_get_a_plot_and_enumerations_a_change_table() -> None: + recorder, result = _record() + + report = render_report(recorder, result) + + assert report.count('' in report # power_status goes online -> offline + assert "'online'" in report and "'offline'" in report + + +def test_everything_dynamic_is_escaped() -> None: + scenario = """ +name: +description: a bold & "quoted" description +steps: + - action: wait + virtual_time: 1s + - action: assert + signal: eps.telemetry.battery_percent + op: ">=" + value: 0 + timeout: 1s +""" + recorder, result = _record(scenario_text=scenario) + + report = render_report(recorder, result) + + assert '' not in report + assert "<script>alert("x")</script>" in report + assert "<b>bold</b> &" in report + assert _parse(report).errors == [] + + +def test_an_aborted_run_renders_an_error_page_with_what_was_recorded() -> None: + recorder, _result = _record() + + report = render_report(recorder, None, error="boom <1>") + + assert ">ERROR" in report + assert "The run aborted:" in report and "boom <1>" in report + assert _parse(report).errors == [] + + +def test_helpers() -> None: + assert format_virtual_time(0) == "0.000000 s" + assert format_virtual_time(3_500_001) == "3.500001 s" + assert format_virtual_time(-1_000_000) == "-1.000000 s" + scenario = parse_scenario(FAILING) + assert describe_step(scenario.steps[0]) == ( + "inject_fault type=state_override target=eps.model.battery_percent value=25 " + "duration=5.000000 s" + ) + assert describe_step(scenario.steps[1]) == "wait virtual_time=3.000000 s" + + +# --- the CLI flag ---------------------------------------------------------------------- + + +def _cli(args: list[str], capsys: pytest.CaptureFixture[str]) -> tuple[int, str, str]: + code = main(args) + captured = capsys.readouterr() + return code, captured.out, captured.err + + +def test_cli_writes_the_report_and_leaves_stdout_to_the_normal_summary( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + out = tmp_path / "nested" / "report.html" + + code, stdout, stderr = _cli(["run", "--example", "default", "--html", str(out)], capsys) + + assert code == 0 and out.is_file() + assert "SUMMARY" in stdout and "PASS" in out.read_text(encoding="utf-8") + + +def test_cli_html_composes_with_json_and_quiet( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + out = tmp_path / "report.html" + + code, stdout, stderr = _cli( + ["run", "--example", "default", "--html", str(out), "--json"], capsys + ) + assert code == 0 and json.loads(stdout)["passed"] is True and out.is_file() + + quiet = tmp_path / "quiet.html" + code, stdout, stderr = _cli( + ["run", "--example", "default", "--html", str(quiet), "--quiet"], capsys + ) + assert code == 0 and stdout == "" and quiet.is_file() + assert "html report written" not in stderr + + +def test_cli_html_composes_with_trace(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + out = tmp_path / "report.html" + + code, _stdout, stderr = _cli( + ["run", "--example", "default", "--html", str(out), "--trace"], capsys + ) + + assert code == 0 and out.is_file() + assert "trace t=0 TX " in stderr # the trace still prints, from the same frames + + +def test_cli_failing_run_still_writes_a_report_and_exits_1( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + scenario = tmp_path / "scenario.yaml" + scenario.write_text(FAILING, encoding="utf-8") + out = tmp_path / "report.html" + + code, _stdout, _stderr = _cli( + ["run", "-c", str(DEFAULT_SETUP), "-s", str(scenario), "--html", str(out)], capsys + ) + + assert code == 1 + assert ">FAIL" in out.read_text(encoding="utf-8") + + +def test_cli_aborted_run_writes_a_partial_report_and_exits_2( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + def explode(self: object, virtual_time: int) -> None: + raise runner_module.ScenarioRuntimeError("simulated failure mid-run") + + monkeypatch.setattr(runner_module.ScenarioRunner, "wait", explode) + out = tmp_path / "report.html" + + code, _stdout, stderr = _cli(["run", "--example", "default", "--html", str(out)], capsys) + + assert code == 2 and "simulated failure mid-run" in stderr + report = out.read_text(encoding="utf-8") + assert ">ERROR" in report and "simulated failure mid-run" in report + assert 'data-kind="step"' in report # the steps reached before the error are there + + +def test_cli_unwritable_report_path_is_an_execution_error( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + code, _stdout, stderr = _cli(["run", "--example", "default", "--html", str(tmp_path)], capsys) + + assert code == 2 and "could not write the HTML report" in stderr + + +# --- the script, run for real ---------------------------------------------------------- + +_NODE_HARNESS = r""" +const data = JSON.parse(process.argv[1]); +const attr = (row) => (name) => row[name]; +function el(extra) { return Object.assign({value: '', checked: true, listeners: {}, hidden: false, + addEventListener(type, fn) { this.listeners[type] = fn; }}, extra || {}); } +const rows = data.rows.map((r) => ({hidden: false, textContent: r.text, getAttribute: attr(r)})); +const ids = {'f-dir': el(), 'f-node': el(), 'f-port': el(), 'f-text': el(), 'f-count': el(), + 'k-frame': el(), 'k-drop': el(), 'k-step': el()}; +for (const r of rows) {} +const byId = {}; +data.rows.forEach((r, i) => { if (r.id) byId[r.id] = rows[i]; }); +global.window = {addEventListener() {}}; +global.location = {hash: data.hash || ''}; +const docListeners = {}; +global.document = { + querySelectorAll: () => rows, + getElementById: (id) => ids[id] || byId[id] || null, + addEventListener(type, fn) { docListeners[type] = fn; }, +}; +eval(data.script); +const out = []; +for (const step of data.steps) { + for (const [k, v] of Object.entries(step.set || {})) { + if (k.startsWith('k-')) ids[k].checked = v; else ids[k].value = v; + } + if (step.fire) ids[step.fire].listeners[step.type || 'input'](); + if (step.click) { + const href = step.click; + docListeners.click({target: {closest: () => ({getAttribute: () => href})}}); + } + out.push({hidden: data.rows.map((r, i) => rows[i].hidden), count: ids['f-count'].textContent, + value: Object.fromEntries(['f-dir','f-node','f-port','f-text'].map(k => [k, ids[k].value]))}); +} +console.log(JSON.stringify(out)); +""" + + +def _run_script(parsed: _Rows, steps: list[dict[str, object]], hash_: str = "") -> list[dict]: # type: ignore[type-arg] + rows = [{**row, "id": key} for key, row in parsed.rows.items()] + payload = json.dumps({"script": parsed.script, "rows": rows, "steps": steps, "hash": hash_}) + done = subprocess.run( + ["node", "-e", _NODE_HARNESS, payload], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert done.returncode == 0, done.stderr + return json.loads(done.stdout) # type: ignore[no-any-return] + + +@pytest.mark.skipif(shutil.which("node") is None, reason="node is not installed") +def test_the_timeline_filters_work() -> None: + recorder, result = _record(scenario_text=FAILING) + parsed = _parse(render_report(recorder, result)) + keys = list(parsed.rows) + + def shown(state: dict) -> list[str]: # type: ignore[type-arg] + return [k for k, h in zip(keys, state["hidden"], strict=True) if not h] + + states = _run_script( + parsed, + [ + {}, + {"set": {"f-dir": "TX"}, "fire": "f-dir"}, + {"set": {"f-dir": "", "k-frame": False}, "fire": "k-frame", "type": "change"}, + {"set": {"k-frame": True, "f-port": "21"}, "fire": "f-port"}, + {"set": {"f-port": "", "f-text": "99"}, "fire": "f-text"}, + ], + ) + + everything = shown(states[0]) + assert everything == keys and states[0]["count"] == f"showing {len(keys)} of {len(keys)}" + tx = shown(states[1]) + assert tx and all(parsed.rows[k]["data-dir"] == "TX" for k in tx if k.startswith("f")) + assert all(parsed.rows[k]["data-kind"] != "frame" for k in shown(states[2])) + on_port = [k for k in shown(states[3]) if parsed.rows[k]["data-kind"] == "frame"] + assert on_port and all("21" in parsed.rows[k]["data-ports"].split() for k in on_port) + assert shown(states[4]) == [k for k in keys if "99" in parsed.rows[k]["text"].lower()] + + +@pytest.mark.skipif(shutil.which("node") is None, reason="node is not installed") +def test_following_an_evidence_link_to_a_filtered_out_row_clears_the_filters() -> None: + """Clicking works even when the hash already equals the target, which fires no event.""" + + recorder, result = _record(scenario_text=FAILING) + parsed = _parse(render_report(recorder, result)) + failed = next(a for a in recorder.assertions if not a.result.passed) + target = f"f{failed.evidence_frame}" + keys = list(parsed.rows) + assert parsed.rows[target]["data-dir"] == "RX" + + states = _run_script( + parsed, + [ + {"set": {"f-dir": "TX"}, "fire": "f-dir"}, # hides the RX evidence row + {"click": f"#{target}"}, # follow the assertion's link + ], + hash_=f"#{target}", + ) + + assert states[0]["hidden"][keys.index(target)] # the filter really hid it + assert not states[1]["hidden"][keys.index(target)] # following the link brought it back + assert states[1]["value"]["f-dir"] == "" + + +@pytest.mark.skipif(shutil.which("node") is None, reason="node is not installed") +def test_opening_the_report_on_a_filtered_out_anchor_shows_that_row() -> None: + recorder, result = _record(scenario_text=FAILING) + parsed = _parse(render_report(recorder, result)) + target = next(k for k, r in parsed.rows.items() if r["data-dir"] == "RX") + + states = _run_script(parsed, [{}], hash_=f"#{target}") + + assert not states[0]["hidden"][list(parsed.rows).index(target)] + + +# --- the whole-packet links ------------------------------------------------------------ + + +def test_a_report_over_a_serial_line_shows_whole_packet_frames_and_a_bad_frame() -> None: + import os + import tty + + from cubesat_testbed.config import parse_testbed_config + from tests.test_kiss_transport import SCENARIO as KISS_SCENARIO + from tests.test_kiss_transport import SETUP as KISS_SETUP + from tests.test_kiss_transport import Pty, _eps_telemetry_frame + + line = Pty() + os.set_blocking(line.master, False) + try: + tty.setraw(line.master) + bad = bytearray(_eps_telemetry_frame(41.0)) + bad[-3] ^= 0xFF + line.push(bytes(bad)) + line.push(_eps_telemetry_frame(42.0)) + setup = parse_testbed_config(KISS_SETUP.format(device=line.path)) + scenario = parse_scenario(KISS_SCENARIO, setup=setup) + recorder = RunRecorder() + result = run_scenario(scenario, setup, output=io.StringIO(), recorder=recorder) + finally: + line.close() + + report = render_report(recorder, result) + parsed = _parse(report) + texts = [row["text"] for row in parsed.rows.values()] + + assert result.passed + assert any(" RX serial " in t and "eps (2) → eps (2)" in t for t in texts), texts + assert any("telemetry eps.telemetry.battery_percent=42.0" in t for t in texts) + assert any( + row["data-kind"] == "drop" and "crc_error" in row["text"] for row in parsed.rows.values() + ) + (assertion,) = recorder.assertions + assert assertion.evidence_frame is not None + assert parsed.errors == []