From aa1f06a9b5d213a4cb1e54237cd6e0e2a61bd400 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Wed, 9 Sep 2026 21:42:12 -0500 Subject: [PATCH 01/11] feat(ota): pure-Python OTA-over-USB host tool + `idf.py ota-usb` build target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `espp_ota`, a host CLI/library that updates an espp device over USB using the existing stream_frame framing + OTA stream protocol (dispatcher module 0) — the same protocol the ota example serves and ota_console.html drives from the browser. No device-side changes; this is the missing host counterpart. Package (components/ota/python/espp_ota/), layered like espp_odrive: - frame.py — stream_frame v2 codec (stdlib only): build_frame + a resynchronizing StreamParser. Verified against the C++ golden CRC (crc32("123456789")==0xCBF43926). - protocol.py — OTA opcodes (Begin/Data/Finish/Abort -> Ok/Error/Progress) + dispatcher discovery request. - transport.py — pyusb vendor-interface transport (discovers the 0xFF interface and its bulk IN/OUT endpoints; VID/PID default 0x1209:0x0d32). pyusb is imported lazily so the codec/protocol stay stdlib-only. - client.py — the session driver (BEGIN -> chunked DATA -> FINISH), one request in flight, progress callback, PROGRESS/ERROR handling. - cli.py — `python -m espp_ota {flash,list,discover}`; VID/PID/serial via flags or ESPP_OTA_* env vars. Seamless build -> OTA: components/ota/project_include.cmake registers an `ota-usb` build target (the OTA counterpart to `idf.py flash`), so any project using the ota component can run `idf.py ota-usb` to build the app and flash it over USB in one step. The build dependency on gen_project_binary is wired via a deferred call (that target is created after project_include runs); on CMake <3.19 `idf.py build ota-usb` is the explicit form. Packaging: espp_ota is added to the espp wheel (wheel.packages) with a `usb` extra (pyusb) and an `espp-ota` console script, mirroring espp_odrive. Verified: codec + full-OTA-against-a-mock-device host tests pass with plain python3; the CMake target + deferred app-build dependency validated in an isolated harness (ninja runs the build step before the flash step); graceful error when pyusb is absent. On-device USB flashing needs hardware. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01DFjjnq3XCTRAXENSxsCxJU --- components/ota/README.md | 14 ++ components/ota/project_include.cmake | 49 +++++ components/ota/python/README.md | 85 ++++++++ components/ota/python/espp_ota/__init__.py | 35 ++++ components/ota/python/espp_ota/__main__.py | 6 + components/ota/python/espp_ota/cli.py | 181 +++++++++++++++++ components/ota/python/espp_ota/client.py | 122 ++++++++++++ components/ota/python/espp_ota/frame.py | 180 +++++++++++++++++ components/ota/python/espp_ota/protocol.py | 108 +++++++++++ components/ota/python/espp_ota/transport.py | 192 +++++++++++++++++++ components/ota/python/tests/test_ota_host.py | 111 +++++++++++ pyproject.toml | 12 ++ 12 files changed, 1095 insertions(+) create mode 100644 components/ota/project_include.cmake create mode 100644 components/ota/python/README.md create mode 100644 components/ota/python/espp_ota/__init__.py create mode 100644 components/ota/python/espp_ota/__main__.py create mode 100644 components/ota/python/espp_ota/cli.py create mode 100644 components/ota/python/espp_ota/client.py create mode 100644 components/ota/python/espp_ota/frame.py create mode 100644 components/ota/python/espp_ota/protocol.py create mode 100644 components/ota/python/espp_ota/transport.py create mode 100644 components/ota/python/tests/test_ota_host.py diff --git a/components/ota/README.md b/components/ota/README.md index 2eff367d29..85826ee03e 100644 --- a/components/ota/README.md +++ b/components/ota/README.md @@ -103,6 +103,20 @@ is the routing id (OTA is **module 0**). OTA layers its message types on it. The [espp OTA Console](https://esp-cpp.github.io/espp/apps/ota_console.html) (`web/ota_console.html`) implements this protocol over WebUSB in the browser. +### Command line: build → OTA + +The [`python/espp_ota`](python/) tool speaks the same protocol from a terminal. +Because this component ships a `project_include.cmake`, any project using it gets +a build-and-flash-over-USB target — the OTA counterpart to `idf.py flash`: + +```sh +pip install pyusb # once (needs a libusb backend) +idf.py ota-usb # builds the app, then OTAs it over USB +``` + +Or drive it directly: `python -m espp_ota flash build/.bin` (see +[`python/README.md`](python/README.md)). + ## Rollback With `CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE=y`, a freshly-installed app boots diff --git a/components/ota/project_include.cmake b/components/ota/project_include.cmake new file mode 100644 index 0000000000..66c9b4a581 --- /dev/null +++ b/components/ota/project_include.cmake @@ -0,0 +1,49 @@ +# espp `ota` component — build-system integration for OTA-over-USB. +# +# Included automatically by ESP-IDF (in project scope) for any project that uses +# the `ota` component. It registers an `ota-usb` build target so you can build +# and OTA-flash your app over USB in one step, the same way `idf.py flash` works +# for the serial bootloader: +# +# idf.py ota-usb # builds the app, then OTAs it over USB +# idf.py build ota-usb # equivalent explicit form (also works pre-CMake 3.19) +# +# Device/port overrides are read from the environment by the tool, e.g.: +# ESPP_OTA_PID=0x1234 idf.py ota-usb +# +# The work is done by the pure-Python `espp_ota` tool shipped alongside this file +# (components/ota/python/). It needs `pyusb` at flash time (not at build time): +# pip install pyusb +# +# For full control (a specific serial, chunk size, discovery probe, ...) run the +# tool directly: python -m espp_ota flash build/.bin --help + +if(NOT TARGET ota-usb) + idf_build_get_property(python PYTHON) + set(__espp_ota_pkg_dir "${CMAKE_CURRENT_LIST_DIR}/python") + # CMAKE_PROJECT_NAME is already set here (the real project() runs before + # idf_build_process includes this file); the app .bin lands in the build dir. + set(__espp_ota_bin "${CMAKE_BINARY_DIR}/${CMAKE_PROJECT_NAME}.bin") + + add_custom_target(ota-usb + COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${__espp_ota_pkg_dir}" + ${python} -m espp_ota flash "${__espp_ota_bin}" + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" + VERBATIM + USES_TERMINAL + COMMENT "OTA-flashing ${__espp_ota_bin} over USB (espp_ota)") + + # `gen_project_binary` (the target that produces the app .bin) is defined + # later in project.cmake, so add the build dependency once this directory + # scope has finished processing. On CMake < 3.19 (no cmake_language(DEFER)) + # the target still works via the explicit `idf.py build ota-usb` form. + function(__espp_ota_link_build_dependency) + if(TARGET gen_project_binary) + add_dependencies(ota-usb gen_project_binary) + endif() + endfunction() + if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.19") + cmake_language(DEFER DIRECTORY "${CMAKE_SOURCE_DIR}" + CALL __espp_ota_link_build_dependency) + endif() +endif() diff --git a/components/ota/python/README.md b/components/ota/python/README.md new file mode 100644 index 0000000000..335d7bac00 --- /dev/null +++ b/components/ota/python/README.md @@ -0,0 +1,85 @@ +# espp_ota — OTA over USB from the command line + +A small, pure-Python host tool that updates an espp device over USB using the +espp `stream_frame` framing + OTA stream protocol (dispatcher **module 0**) — the +same protocol the on-device [`ota` example](../example/) serves and +[`ota_console.html`](../web/ota_console.html) drives from the browser. + +It talks to the device's USB **vendor (WebUSB)** interface (`bInterfaceClass +0xFF`, one bulk IN + one bulk OUT endpoint). The frame codec and OTA protocol are +standard-library only; the USB transport uses [`pyusb`](https://pypi.org/project/pyusb/), +imported lazily. + +## Seamless: build → OTA with `idf.py` + +If your project uses the espp `ota` component, its `project_include.cmake` +registers an `ota-usb` build target, so you can build and flash over USB in one +step (just like `idf.py flash` does over the serial bootloader): + +```sh +pip install pyusb # once (libusb backend: `brew install libusb`, `apt install libusb-1.0-0`) +idf.py ota-usb # builds the app, then OTAs it over USB +# or, equivalently / on CMake < 3.19: +idf.py build ota-usb +``` + +Override the target device without editing anything (the tool reads these): + +```sh +ESPP_OTA_PID=0x1234 idf.py ota-usb +``` + +## Standalone CLI + +Run it directly for full control (or when you already have a `.bin`): + +```sh +python -m espp_ota flash build/my_app.bin # BEGIN -> stream -> FINISH +python -m espp_ota flash build/my_app.bin --pid 0x1234 --chunk-size 2048 +python -m espp_ota list # list matching USB devices +python -m espp_ota discover # probe the device's dispatcher +``` + +Installed with the espp wheel it's also available as the `espp-ota` command +(`pip install "espp[usb]"`). + +## Library use + +```python +from espp_ota import OtaClient, UsbVendorTransport + +with open("build/my_app.bin", "rb") as f: + image = f.read() + +with UsbVendorTransport() as t: # default VID/PID 0x1209:0x0d32 + OtaClient(t, progress=lambda w, tot: print(w, "/", tot)).flash(image) +``` + +## Protocol + +`module = 0`; requests are host→device, replies device→host (reply flag set). +Flow control is one request in flight — each request waits for its OK/ERROR +reply before the next is sent. + +| type | name | dir | payload | +|------|------|-----|---------| +| 0x01 | BEGIN | host→dev | u32 image_size (0 = unknown/streaming) | +| 0x02 | DATA | host→dev | image bytes (1..4096) | +| 0x03 | FINISH | host→dev | — (validate + activate) | +| 0x04 | ABORT | host→dev | — | +| 0x05 | OK | dev→host | u32 bytes_received | +| 0x06 | ERROR | dev→host | u32 code + utf-8 message | +| 0x07 | PROGRESS | dev→host | u32 written, u32 total | + +The wire framing is `espp::stream_frame` v2 (magic `0x4F54`, CRC-32); see +`espp_ota/frame.py`. Host tests (codec + a full OTA against a mock device) live +in `tests/test_ota_host.py` and run with plain `python3`. + +## Requirements + +- Python 3.8+ +- `pyusb` + a libusb backend (only for the actual USB transport): + - macOS: `brew install libusb` + - Linux: `apt install libusb-1.0-0` (add a udev rule for non-root access) + - Windows: the device advertises WebUSB + MS-OS-2.0, so WinUSB binds + automatically; otherwise bind it once with [Zadig](https://zadig.akeo.ie/). diff --git a/components/ota/python/espp_ota/__init__.py b/components/ota/python/espp_ota/__init__.py new file mode 100644 index 0000000000..9e8aae9c28 --- /dev/null +++ b/components/ota/python/espp_ota/__init__.py @@ -0,0 +1,35 @@ +"""espp_ota — pure-Python host tool to OTA-update an espp device over USB. + +Speaks the espp ``stream_frame`` framing + OTA stream protocol (dispatcher +module 0) over the device's USB vendor (WebUSB) interface — the same protocol +``components/ota/web/ota_console.html`` implements in the browser and the +``ota`` example serves on-device. + +The codec (:mod:`espp_ota.frame`) and protocol (:mod:`espp_ota.protocol`) are +standard-library only; the USB transport (:mod:`espp_ota.transport`) needs +`pyusb`, imported lazily. + +Typical use:: + + from espp_ota import OtaClient, UsbVendorTransport + with UsbVendorTransport() as t: + OtaClient(t, progress=lambda w, tot: ...).flash(open("app.bin", "rb").read()) +""" + +from .client import OtaClient +from .protocol import ErrorInfo, MessageType, OtaError, ProgressInfo +from .transport import DEFAULT_PID, DEFAULT_VID, TransportError, UsbVendorTransport + +__all__ = [ + "OtaClient", + "UsbVendorTransport", + "TransportError", + "OtaError", + "MessageType", + "ErrorInfo", + "ProgressInfo", + "DEFAULT_VID", + "DEFAULT_PID", +] + +__version__ = "0.1.0" diff --git a/components/ota/python/espp_ota/__main__.py b/components/ota/python/espp_ota/__main__.py new file mode 100644 index 0000000000..dbdd066172 --- /dev/null +++ b/components/ota/python/espp_ota/__main__.py @@ -0,0 +1,6 @@ +import sys + +from .cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/components/ota/python/espp_ota/cli.py b/components/ota/python/espp_ota/cli.py new file mode 100644 index 0000000000..c316ad991e --- /dev/null +++ b/components/ota/python/espp_ota/cli.py @@ -0,0 +1,181 @@ +"""Command-line interface: ``python -m espp_ota ``. + +Commands: + flash BEGIN -> stream DATA -> FINISH an image over USB. + list List matching USB devices. + discover Probe the device (dispatcher ListModules) and report reply. + +VID/PID default to the espp UsbDevice default (0x1209:0x0d32) but can be +overridden (also via the ESPP_OTA_VID / ESPP_OTA_PID env vars, which the CMake +``ota-usb`` target forwards). +""" + +from __future__ import annotations + +import argparse +import os +import sys +import time +from typing import Optional + +from . import __version__ +from .client import OtaClient +from .protocol import OtaError +from .transport import DEFAULT_PID, DEFAULT_VID, TransportError, UsbVendorTransport, list_devices + + +def _auto_int(text: str) -> int: + return int(text, 0) # accepts 0x1209, 4617, etc. + + +def _env_int(name: str, default: int) -> int: + val = os.environ.get(name) + return _auto_int(val) if val else default + + +def _add_device_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--vid", type=_auto_int, default=_env_int("ESPP_OTA_VID", DEFAULT_VID), + help="USB vendor id (default 0x%04x)" % DEFAULT_VID) + p.add_argument("--pid", type=_auto_int, default=_env_int("ESPP_OTA_PID", DEFAULT_PID), + help="USB product id (default 0x%04x; pass -1 to match any)" % DEFAULT_PID) + p.add_argument("--serial", default=os.environ.get("ESPP_OTA_SERIAL"), + help="match a specific device serial number") + p.add_argument("--interface", type=_auto_int, default=None, + help="force a specific vendor interface number") + + +def _open_transport(args) -> UsbVendorTransport: + pid = None if args.pid is not None and args.pid < 0 else args.pid + return UsbVendorTransport(vid=args.vid, pid=pid, serial=args.serial, + interface=args.interface).open() + + +class _ProgressBar: + def __init__(self, quiet: bool) -> None: + self._quiet = quiet + self._last = 0.0 + + def __call__(self, written: int, total: int) -> None: + if self._quiet: + return + now = time.monotonic() + done = total and written >= total + if now - self._last < 0.1 and not done: + return + self._last = now + if total: + pct = min(100.0, 100.0 * written / total) + bar = "#" * int(pct / 2.5) + sys.stderr.write(f"\r [{bar:<40}] {pct:5.1f}% {written}/{total} B") + else: + sys.stderr.write(f"\r {written} B") + if done: + sys.stderr.write("\n") + sys.stderr.flush() + + +def _cmd_flash(args) -> int: + with open(args.binary, "rb") as fh: + image = fh.read() + if not image: + print("error: image is empty", file=sys.stderr) + return 2 + size = 0 if args.unknown_size else len(image) + t = _open_transport(args) + try: + if not args.quiet: + print(f"Connected to {t.description}; flashing {args.binary} " + f"({len(image)} bytes)...", file=sys.stderr) + client = OtaClient( + t, + chunk_size=args.chunk_size, + progress=_ProgressBar(args.quiet), + begin_timeout_ms=args.begin_timeout, + data_timeout_ms=args.data_timeout, + finish_timeout_ms=args.finish_timeout, + ) + start = time.monotonic() + client.flash(image, image_size=size) + if not args.quiet: + dt = time.monotonic() - start + rate = len(image) / dt / 1024 if dt else 0 + print(f"OTA complete in {dt:.1f}s ({rate:.0f} KiB/s). The device " + f"activates the new image and reboots per its own policy.", file=sys.stderr) + finally: + t.close() + return 0 + + +def _cmd_list(args) -> int: + pid = None if args.pid is not None and args.pid < 0 else args.pid + found = list_devices(vid=args.vid, pid=pid) + if not found: + print("no matching USB devices found", file=sys.stderr) + return 1 + for vid, pid_, desc in found: + print(f"0x{vid:04x}:0x{pid_:04x} {desc}") + return 0 + + +def _cmd_discover(args) -> int: + t = _open_transport(args) + try: + frames = OtaClient(t).discover(timeout_ms=args.timeout) + if not frames: + print("no discovery reply (device may not run a Dispatcher on the " + "vendor interface)", file=sys.stderr) + return 1 + for fr in frames: + print(f"reply module=0x{fr.module:02x} type=0x{fr.type:02x} " + f"reply={fr.is_reply} payload={len(fr.payload)} bytes") + finally: + t.close() + return 0 + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(prog="espp_ota", description=__doc__.split("\n")[0]) + p.add_argument("--version", action="version", version=f"espp_ota {__version__}") + sub = p.add_subparsers(dest="command", required=True) + + f = sub.add_parser("flash", help="OTA-update a binary over USB") + f.add_argument("binary", help="path to the app .bin to flash") + _add_device_args(f) + f.add_argument("--chunk-size", type=_auto_int, default=4096, + help="DATA payload bytes per frame (1..4096, default 4096)") + f.add_argument("--unknown-size", action="store_true", + help="stream with size 0 (device erases the whole partition)") + f.add_argument("--begin-timeout", type=int, default=60000, help="ms (default 60000)") + f.add_argument("--data-timeout", type=int, default=5000, help="ms (default 5000)") + f.add_argument("--finish-timeout", type=int, default=60000, help="ms (default 60000)") + f.add_argument("-q", "--quiet", action="store_true", help="suppress progress output") + f.set_defaults(func=_cmd_flash) + + lst = sub.add_parser("list", help="list matching USB devices") + _add_device_args(lst) + lst.set_defaults(func=_cmd_list) + + d = sub.add_parser("discover", help="probe the device's dispatcher (ListModules)") + _add_device_args(d) + d.add_argument("--timeout", type=int, default=2000, help="ms (default 2000)") + d.set_defaults(func=_cmd_discover) + return p + + +def main(argv: Optional[list] = None) -> int: + args = build_parser().parse_args(argv) + try: + return args.func(args) + except (OtaError, TransportError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + except FileNotFoundError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + except KeyboardInterrupt: + print("\ninterrupted", file=sys.stderr) + return 130 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/components/ota/python/espp_ota/client.py b/components/ota/python/espp_ota/client.py new file mode 100644 index 0000000000..bedb2574a7 --- /dev/null +++ b/components/ota/python/espp_ota/client.py @@ -0,0 +1,122 @@ +"""The OTA session driver: BEGIN -> DATA* -> FINISH over a byte transport. + +Transport-agnostic: it needs an object with ``write(bytes, timeout_ms)`` and +``read(max_len, timeout_ms) -> bytes`` (``b""`` on timeout), e.g. +:class:`espp_ota.transport.UsbVendorTransport`. Flow control is one request in +flight — each request waits for its OK/ERROR reply before the next is sent — +matching the device and ``ota_console.html``. +""" + +from __future__ import annotations + +import time +from collections import deque +from typing import Callable, Deque, List, Optional + +from . import frame as _f +from . import protocol as _p +from .protocol import MessageType, OtaError + +ProgressFn = Callable[[int, int], None] # (written, total) -> None + + +class OtaClient: + def __init__( + self, + transport, + chunk_size: int = _f.MAX_PAYLOAD_SIZE, + progress: Optional[ProgressFn] = None, + begin_timeout_ms: int = 60000, + data_timeout_ms: int = 5000, + finish_timeout_ms: int = 60000, + ) -> None: + if not (1 <= chunk_size <= _f.MAX_PAYLOAD_SIZE): + raise ValueError(f"chunk_size must be 1..{_f.MAX_PAYLOAD_SIZE}") + self._t = transport + self._chunk = chunk_size + self._progress = progress + self._begin_to = begin_timeout_ms + self._data_to = data_timeout_ms + self._finish_to = finish_timeout_ms + self._parser = _f.StreamParser() + self._pending: Deque[_f.Frame] = deque() + + # -- reply plumbing ------------------------------------------------------- + def _next_frame(self, deadline: float) -> _f.Frame: + while True: + if self._pending: + return self._pending.popleft() + remaining = deadline - time.monotonic() + if remaining <= 0: + raise OtaError("timed out waiting for a device reply") + data = self._t.read(_f.MAX_FRAME_SIZE, timeout_ms=max(1, int(remaining * 1000))) + if data: + self._pending.extend(self._parser.feed(data)) + + def _transact(self, request: bytes, timeout_ms: int) -> _f.Frame: + """Send one request and return the matching OK reply (module 0). + + PROGRESS frames are surfaced to the callback and skipped; an ERROR reply + raises :class:`OtaError`; frames for other modules are ignored.""" + self._t.write(request, timeout_ms=self._data_to) + deadline = time.monotonic() + timeout_ms / 1000.0 + while True: + fr = self._next_frame(deadline) + if fr.module != _p.MODULE: + continue # discovery / other module chatter + if fr.type == MessageType.PROGRESS: + info = _p.parse_progress(fr) + if info and self._progress: + self._progress(info.written, info.total) + continue + if fr.type == MessageType.ERROR: + info = _p.parse_error(fr) + if info: + raise OtaError(f"device error: {info.message}", info.code) + raise OtaError("device error (unparseable ERROR reply)") + if fr.type == MessageType.OK: + return fr + raise OtaError(f"unexpected reply type 0x{fr.type:02x}") + + # -- public API ----------------------------------------------------------- + def flash(self, image: bytes, image_size: Optional[int] = None) -> None: + """Run a full OTA: BEGIN(size) -> DATA chunks -> FINISH. + + ``image_size`` defaults to ``len(image)``; pass 0 for an unknown-size + (streaming) session (the device erases the whole partition).""" + if not image: + raise OtaError("empty image") + size = len(image) if image_size is None else image_size + + self._transact(_p.make_begin(size), self._begin_to) + + total = len(image) + sent = 0 + for off in range(0, total, self._chunk): + chunk = image[off : off + self._chunk] + ok = self._transact(_p.make_data(chunk), self._data_to) + sent += len(chunk) + # OK carries bytes_received; prefer it, fall back to our own count. + received = _p.parse_u32(ok) + if self._progress: + self._progress(received if received is not None else sent, total) + + self._transact(_p.make_finish(), self._finish_to) + + def abort(self) -> None: + try: + self._transact(_p.make_abort(), self._data_to) + except OtaError: + pass # best-effort + + def discover(self, timeout_ms: int = 2000) -> List[_f.Frame]: + """Send a dispatcher ListModules request; return the reply frame(s). + + Useful as a connectivity probe before flashing. Returns raw frames (the + discovery TLV is not decoded here).""" + self._t.write(_p.make_discovery_request(), timeout_ms=self._data_to) + deadline = time.monotonic() + timeout_ms / 1000.0 + try: + return [self._next_frame(deadline)] + except OtaError: + return [] diff --git a/components/ota/python/espp_ota/frame.py b/components/ota/python/espp_ota/frame.py new file mode 100644 index 0000000000..0cd7a3d770 --- /dev/null +++ b/components/ota/python/espp_ota/frame.py @@ -0,0 +1,180 @@ +"""espp ``stream_frame`` v2 codec — pure Python, standard library only. + +This mirrors ``components/stream_frame/include/stream_frame.hpp`` so a host tool +can speak the exact same wire protocol the device does, without building the +espp Python bindings. + +Wire format (all multi-byte fields little-endian):: + + [magic u16 = 0x4F54 "OT"][flags u8][module u8][type u8] + {[correlation u16] iff flags bit1}[len u32][payload][crc32 u32] + +``crc32`` is the standard zlib CRC-32 over every byte from the magic through the +payload (i.e. the whole frame except the trailing CRC field). Python's +``zlib.crc32`` is that exact algorithm, so ``crc32(b"123456789") == 0xCBF43926`` +matches the C++ golden value. +""" + +from __future__ import annotations + +import struct +import zlib +from dataclasses import dataclass +from typing import List, Optional, Tuple + +# ---- constants (kept in step with stream_frame.hpp) ------------------------- +MAGIC = 0x4F54 +MAGIC_BYTES = struct.pack(" int: + """Standard zlib/IEEE CRC-32 (matches ``espp::stream_frame::crc32``).""" + return zlib.crc32(data) & 0xFFFFFFFF + + +def make_flags(reply: bool, version: int = VERSION) -> int: + return ((version & 0x0F) << 4) | (FLAG_REPLY if reply else 0) + + +def flags_is_reply(flags: int) -> bool: + return bool(flags & FLAG_REPLY) + + +def flags_version(flags: int) -> int: + return (flags >> 4) & 0x0F + + +def flags_has_correlation(flags: int) -> bool: + return bool(flags & FLAG_CORRELATION) + + +def build_frame( + module: int, + type_: int, + payload: bytes = b"", + reply: bool = False, + correlation: Optional[int] = None, +) -> bytes: + """Encode one frame. Raises ``ValueError`` if the payload is too large.""" + if len(payload) > MAX_PAYLOAD_SIZE: + raise ValueError( + f"payload {len(payload)} bytes exceeds MAX_PAYLOAD_SIZE ({MAX_PAYLOAD_SIZE})" + ) + flags = make_flags(reply) + if correlation is not None: + flags |= FLAG_CORRELATION + out = bytearray(MAGIC_BYTES) + out += bytes((flags & 0xFF, module & 0xFF, type_ & 0xFF)) + if correlation is not None: + out += struct.pack(" bool: + return flags_is_reply(self.flags) + + @property + def version(self) -> int: + return flags_version(self.flags) + + +class StreamParser: + """Incremental, resynchronizing frame parser. + + Mirrors ``espp::stream_frame::StreamParser``: feed arbitrary chunks (USB bulk + transfers may split or coalesce frames) and it yields the complete, CRC-valid + frames they contain, resynchronizing past a bad magic, an oversized length, + or a CRC mismatch by dropping one byte and retrying. + """ + + def __init__(self) -> None: + self._buf = bytearray() + self.dropped_bytes = 0 + + def reset(self) -> None: + self._buf.clear() + + def buffered(self) -> int: + return len(self._buf) + + def feed(self, data: bytes) -> List[Frame]: + self._buf += data + frames: List[Frame] = [] + while True: + frame, consumed = self._try_one() + if consumed == 0: + break # need more bytes + del self._buf[:consumed] + if frame is not None: + frames.append(frame) + return frames + + def _try_one(self) -> Tuple[Optional[Frame], int]: + """Return (frame|None, bytes_to_consume). consumed==0 means "need more". + + A non-None frame with consumed>0 is a good frame; a None frame with + consumed==1 is a resync (drop one byte and keep scanning).""" + buf = self._buf + n = len(buf) + # Find the magic. Need at least 2 bytes to check it. + if n < 2: + # Could a single trailing byte be the start of the magic? Keep it. + if n == 1 and buf[0] == MAGIC_BYTES[0]: + return None, 0 + if n == 1: + self.dropped_bytes += 1 + return None, 1 + return None, 0 + if buf[0] != MAGIC_BYTES[0] or buf[1] != MAGIC_BYTES[1]: + self.dropped_bytes += 1 + return None, 1 # resync: drop one byte + # Enough for the fixed header? + if n < HEADER_SIZE: + return None, 0 + flags = buf[2] + module = buf[3] + type_ = buf[4] + offset = 5 + correlation: Optional[int] = None + if flags_has_correlation(flags): + if n < HEADER_SIZE + CORRELATION_SIZE: + return None, 0 + correlation = struct.unpack_from(" MAX_PAYLOAD_SIZE: + self.dropped_bytes += 1 + return None, 1 # bogus length -> resync + total = offset + length + CRC_SIZE + if n < total: + return None, 0 # wait for the rest of the frame + want_crc = struct.unpack_from(" resync + payload = bytes(buf[offset : offset + length]) + return Frame(flags, module, type_, payload, correlation), total diff --git a/components/ota/python/espp_ota/protocol.py b/components/ota/python/espp_ota/protocol.py new file mode 100644 index 0000000000..25236cce2d --- /dev/null +++ b/components/ota/python/espp_ota/protocol.py @@ -0,0 +1,108 @@ +"""espp OTA stream protocol (dispatcher module 0). + +Mirrors ``components/ota/include/detail/ota_stream_protocol.hpp``: the OTA +message-type enum, frame builders (``make_*``) and reply parsers (``parse_*``) +layered on the :mod:`espp_ota.frame` codec. + +Requests are host->device (reply flag = 0); replies are device->host +(reply flag = 1). Flow control is one-frame-in-flight: the host sends a request +and waits for the matching OK/ERROR reply before sending the next. +""" + +from __future__ import annotations + +import struct +from dataclasses import dataclass +from enum import IntEnum +from typing import Optional + +from . import frame as _f + +#: OTA occupies dispatcher module id 0. +MODULE = 0 + +#: Discovery meta-module (see components/dispatcher). ListModules == 0x00. +DISCOVERY_MODULE = 0xFF +DISCOVERY_LIST_MODULES = 0x00 + + +class MessageType(IntEnum): + BEGIN = 0x01 # host->device: u32 image_size (0 = unknown / streaming) + DATA = 0x02 # host->device: raw image bytes (1..MAX_PAYLOAD_SIZE) + FINISH = 0x03 # host->device: validate + activate (no payload) + ABORT = 0x04 # host->device: discard the session (no payload) + OK = 0x05 # device->host: u32 bytes_received so far + ERROR = 0x06 # device->host: u32 code + utf-8 message + PROGRESS = 0x07 # device->host: u32 written, u32 total (0 if unknown) + + +_REPLY_TYPES = {MessageType.OK, MessageType.ERROR, MessageType.PROGRESS} + + +def _build(type_: MessageType, payload: bytes = b"") -> bytes: + return _f.build_frame(MODULE, int(type_), payload, reply=type_ in _REPLY_TYPES) + + +# ---- request builders (host -> device) -------------------------------------- +def make_begin(image_size: int) -> bytes: + return _build(MessageType.BEGIN, struct.pack(" bytes: + return _build(MessageType.DATA, chunk) + + +def make_finish() -> bytes: + return _build(MessageType.FINISH) + + +def make_abort() -> bytes: + return _build(MessageType.ABORT) + + +def make_discovery_request() -> bytes: + """A dispatcher discovery (ListModules) request on module 0xFF.""" + return _f.build_frame(DISCOVERY_MODULE, DISCOVERY_LIST_MODULES, b"", reply=False) + + +# ---- reply parsers (device -> host) ----------------------------------------- +@dataclass +class ErrorInfo: + code: int + message: str + + +@dataclass +class ProgressInfo: + written: int + total: int # 0 if unknown + + +def parse_u32(fr: _f.Frame) -> Optional[int]: + """The single-u32 payload of a BEGIN echo or an OK (bytes_received).""" + if len(fr.payload) != 4: + return None + return struct.unpack(" Optional[ErrorInfo]: + if len(fr.payload) < 4: + return None + code = struct.unpack_from(" Optional[ProgressInfo]: + if len(fr.payload) != 8: + return None + written, total = struct.unpack(" None: + super().__init__(message if code is None else f"{message} (code {code})") + self.code = code diff --git a/components/ota/python/espp_ota/transport.py b/components/ota/python/espp_ota/transport.py new file mode 100644 index 0000000000..5e6406e9c3 --- /dev/null +++ b/components/ota/python/espp_ota/transport.py @@ -0,0 +1,192 @@ +"""USB vendor-interface transport for the OTA host tool. + +Talks to the device's WebUSB / vendor interface (``bInterfaceClass == 0xFF``, +one bulk IN + one bulk OUT endpoint) — the same interface ``ota_console.html`` +uses from the browser. Uses `pyusb` (libusb); it is imported lazily so the +:mod:`espp_ota.frame` / :mod:`espp_ota.protocol` layers stay stdlib-only. + +Default device id is the espp ``UsbDevice`` default, ``0x1209:0x0d32``; both are +overridable (a project may set its own VID/PID). +""" + +from __future__ import annotations + +from typing import List, Optional, Tuple + +DEFAULT_VID = 0x1209 +DEFAULT_PID = 0x0D32 +VENDOR_CLASS = 0xFF + + +class TransportError(RuntimeError): + pass + + +def _import_usb(): + try: + import usb.core # noqa: F401 + import usb.util # noqa: F401 + except ImportError as exc: # pragma: no cover - environment dependent + raise TransportError( + "pyusb is required for the USB transport. Install it with " + "`pip install pyusb` (and a libusb backend: `brew install libusb` on " + "macOS, `apt install libusb-1.0-0` on Linux; on Windows bind WinUSB " + "with Zadig if the device is not already driverless)." + ) from exc + import usb.core as core + import usb.util as util + + return core, util + + +def list_devices(vid: int = DEFAULT_VID, pid: Optional[int] = None) -> List[Tuple[int, int, str]]: + """Return (vid, pid, description) for candidate devices matching the filter.""" + core, util = _import_usb() + kwargs = {"find_all": True, "idVendor": vid} + if pid is not None: + kwargs["idProduct"] = pid + out: List[Tuple[int, int, str]] = [] + for dev in core.find(**kwargs): + try: + desc = util.get_string(dev, dev.iProduct) or "" + except Exception: + desc = "" + out.append((dev.idVendor, dev.idProduct, desc)) + return out + + +class UsbVendorTransport: + """Open the vendor bulk pipe of an espp device and read/write raw frames. + + Use as a context manager:: + + with UsbVendorTransport() as t: + t.write(frame_bytes) + reply = t.read(4111, timeout_ms=5000) + """ + + def __init__( + self, + vid: int = DEFAULT_VID, + pid: Optional[int] = DEFAULT_PID, + serial: Optional[str] = None, + interface: Optional[int] = None, + ) -> None: + self._vid = vid + self._pid = pid + self._serial = serial + self._want_itf = interface + self._core, self._util = _import_usb() + self._dev = None + self._itf_num: Optional[int] = None + self._ep_in = None + self._ep_out = None + self._claimed = False + + # -- lifecycle ------------------------------------------------------------ + def open(self) -> "UsbVendorTransport": + core, util = self._core, self._util + + def _match(dev): + if self._serial is None: + return True + try: + return util.get_string(dev, dev.iSerialNumber) == self._serial + except Exception: + return False + + kwargs = {"idVendor": self._vid} + if self._pid is not None: + kwargs["idProduct"] = self._pid + dev = core.find(custom_match=_match, **kwargs) + if dev is None: + raise TransportError( + f"no device found matching vid=0x{self._vid:04x}" + + (f" pid=0x{self._pid:04x}" if self._pid is not None else "") + + (f" serial={self._serial!r}" if self._serial else "") + ) + self._dev = dev + + cfg = dev.get_active_configuration() + itf, ep_in, ep_out = self._find_vendor_interface(cfg) + self._itf_num = itf.bInterfaceNumber + self._ep_in, self._ep_out = ep_in, ep_out + + # Detach a kernel driver if one grabbed the interface (rare for a pure + # vendor class, but be safe on Linux). + try: + if dev.is_kernel_driver_active(self._itf_num): + dev.detach_kernel_driver(self._itf_num) + except (NotImplementedError, self._core.USBError): + pass + + self._util.claim_interface(dev, self._itf_num) + self._claimed = True + return self + + def _find_vendor_interface(self, cfg): + util = self._util + for itf in cfg: + if self._want_itf is not None and itf.bInterfaceNumber != self._want_itf: + continue + if itf.bInterfaceClass != VENDOR_CLASS: + continue + ep_in = ep_out = None + for ep in itf: + is_bulk = util.endpoint_type(ep.bmAttributes) == util.ENDPOINT_TYPE_BULK + if not is_bulk: + continue + if util.endpoint_direction(ep.bEndpointAddress) == util.ENDPOINT_IN: + ep_in = ep + else: + ep_out = ep + if ep_in is not None and ep_out is not None: + return itf, ep_in, ep_out + raise TransportError( + "no vendor interface (class 0xFF with a bulk IN+OUT endpoint pair) found; " + "is the device running the OTA example over its WebUSB/vendor interface?" + ) + + def close(self) -> None: + if self._dev is None: + return + try: + if self._claimed and self._itf_num is not None: + self._util.release_interface(self._dev, self._itf_num) + except Exception: + pass + try: + self._util.dispose_resources(self._dev) + except Exception: + pass + self._dev = None + self._claimed = False + + def __enter__(self) -> "UsbVendorTransport": + return self.open() + + def __exit__(self, *exc) -> None: + self.close() + + # -- I/O ------------------------------------------------------------------ + def write(self, data: bytes, timeout_ms: int = 5000) -> None: + n = self._ep_out.write(data, timeout_ms) + if n != len(data): + raise TransportError(f"short write: {n}/{len(data)} bytes") + + def read(self, max_len: int, timeout_ms: int = 5000) -> bytes: + """Read up to ``max_len`` bytes; return ``b""`` on timeout (not an error).""" + try: + arr = self._ep_in.read(max_len, timeout_ms) + except self._core.USBError as exc: + # errno 110 == ETIMEDOUT; pyusb>=1.1 raises the USBTimeoutError subclass. + if getattr(exc, "errno", None) in (110, None) or "timeout" in str(exc).lower(): + return b"" + raise + return bytes(arr) + + @property + def description(self) -> str: + if self._dev is None: + return "" + return f"0x{self._dev.idVendor:04x}:0x{self._dev.idProduct:04x} itf {self._itf_num}" diff --git a/components/ota/python/tests/test_ota_host.py b/components/ota/python/tests/test_ota_host.py new file mode 100644 index 0000000000..8901ee2f69 --- /dev/null +++ b/components/ota/python/tests/test_ota_host.py @@ -0,0 +1,111 @@ +"""Host tests for espp_ota: codec round-trips + a full OTA against a mock device. + +Runs with plain ``python3`` (no hardware, no pyusb). Also importable by pytest. +""" + +import os +import struct +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from espp_ota import frame as F # noqa: E402 +from espp_ota import protocol as P # noqa: E402 +from espp_ota.client import OtaClient # noqa: E402 +from espp_ota.protocol import MessageType, OtaError # noqa: E402 + + +class MockDevice: + """Loopback transport implementing the OTA device side in memory. + + The host writes request frames; ``read()`` returns the device's replies. + Set ``fail_after`` to make the device answer a DATA frame with ERROR.""" + + def __init__(self, fail_after=None, emit_progress=False): + self._parser = F.StreamParser() + self._out = bytearray() + self.image = bytearray() + self.received = 0 + self.data_frames = 0 + self.finished = False + self._fail_after = fail_after + self._emit_progress = emit_progress + + def write(self, data, timeout_ms=0): + for fr in self._parser.feed(data): + self._handle(fr) + + def read(self, max_len, timeout_ms=0): + if not self._out: + return b"" + chunk = bytes(self._out[:max_len]) + del self._out[: len(chunk)] + return chunk + + def _reply(self, b): + self._out += b + + def _handle(self, fr): + if fr.module != P.MODULE: + return + t = fr.type + if t == MessageType.BEGIN: + self.received = 0 + self.image = bytearray() + self._reply(P._build(MessageType.OK, struct.pack(" self._fail_after: + self._reply(P._build(MessageType.ERROR, struct.pack(" 0 and seen[-1][0] == len(image)) + + +def test_error_reply(): + dev = MockDevice(fail_after=1) + raised = False + try: + OtaClient(dev).flash(bytes(4096 * 3)) + except OtaError as exc: + raised = True + _ok("error carries device code", exc.code == 22) + _ok("error path raises OtaError", raised) + + +def test_small_image_one_chunk(): + dev = MockDevice() + OtaClient(dev).flash(b"\xe9tiny firmware") + _ok("single-chunk image", bytes(dev.image) == b"\xe9tiny firmware" and dev.data_frames == 1) + + +if __name__ == "__main__": + test_full_flash() + test_error_reply() + test_small_image_one_chunk() + print("all host tests passed") diff --git a/pyproject.toml b/pyproject.toml index bb454efef3..577630b772 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,14 @@ Issues = "https://github.com/esp-cpp/espp/issues" # the shipped packages is stdlib-only. `pip install espp[serial]` enables the # espp_odrive serial/UART transports. serial = ["pyserial"] +# espp_ota's USB transport imports pyusb lazily; the frame codec + OTA protocol +# are stdlib-only. `pip install espp[usb]` enables the OTA-over-USB host tool. +usb = ["pyusb"] + +[project.scripts] +# Standalone CLI (also runnable as `python -m espp_ota`). The `ota` component's +# CMake integration wires `idf.py ota-usb` to the same tool for a build->OTA flow. +espp-ota = "espp_ota.cli:main" [tool.scikit-build] # The CMake project for the host (PC) build lives in lib/; the repo root is the @@ -48,6 +56,10 @@ cmake.source-dir = "lib" wheel.packages = [ "lib/python_bindings/espp", "components/odrive_native/python/espp_odrive", + # espp_ota: pure-python OTA-over-USB host tool (stdlib-only except a + # lazily-imported pyusb; see the `usb` extra). Single source of truth lives in + # the ota component. + "components/ota/python/espp_ota", ] wheel.exclude = ["**/.mypy_cache", "**/__pycache__"] # Persistent CMake build dir (gitignored) so repeated local builds - notably From 44e99deb4ef3da27aa569f6d35e6c31fceec7321 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Thu, 10 Sep 2026 08:48:46 -0500 Subject: [PATCH 02/11] fix(ota): address PR review on the OTA host tool client.py: - _transact now requires the reply flag (fr.is_reply) on module-0 frames, so a device-originated request whose type collides with OK/ERROR/PROGRESS can't be mistaken for a reply (matches the browser probe). - flash() wraps the BEGIN/DATA/FINISH session in best-effort ABORT cleanup on any failure (device ERROR, timeout, Ctrl-C, transport error) before re-raising, so the device doesn't stay "busy" and reject a retry's BEGIN. abort() now swallows all errors (the link may already be gone). - discover() reads until the actual discovery reply (module 0xFF, reply flag, ListModules type) instead of returning the first frame seen. cli.py: open the transport with a `with` statement (context manager) instead of try/finally in the flash and discover commands. transport.py: comment the three best-effort empty-except blocks (kernel-driver detach, interface release, resource dispose). tests: add byte-level golden fixtures (CRC vector 0xCBF43926, exact BEGIN(0) and discovery request frames) and StreamParser resync fixtures (leading garbage, a CRC-corrupted frame, a split frame) so wire-encoding / resync correctness is asserted independently of the mock loopback. Host tests pass with python3. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01DFjjnq3XCTRAXENSxsCxJU --- components/ota/python/espp_ota/cli.py | 15 ++--- components/ota/python/espp_ota/client.py | 70 +++++++++++++------- components/ota/python/espp_ota/transport.py | 6 +- components/ota/python/tests/test_ota_host.py | 39 +++++++++++ 4 files changed, 92 insertions(+), 38 deletions(-) diff --git a/components/ota/python/espp_ota/cli.py b/components/ota/python/espp_ota/cli.py index c316ad991e..e49ee6aeb8 100644 --- a/components/ota/python/espp_ota/cli.py +++ b/components/ota/python/espp_ota/cli.py @@ -44,10 +44,11 @@ def _add_device_args(p: argparse.ArgumentParser) -> None: help="force a specific vendor interface number") -def _open_transport(args) -> UsbVendorTransport: +def _make_transport(args) -> UsbVendorTransport: + """Build an (unopened) transport; use it as a context manager (`with`).""" pid = None if args.pid is not None and args.pid < 0 else args.pid return UsbVendorTransport(vid=args.vid, pid=pid, serial=args.serial, - interface=args.interface).open() + interface=args.interface) class _ProgressBar: @@ -81,8 +82,7 @@ def _cmd_flash(args) -> int: print("error: image is empty", file=sys.stderr) return 2 size = 0 if args.unknown_size else len(image) - t = _open_transport(args) - try: + with _make_transport(args) as t: if not args.quiet: print(f"Connected to {t.description}; flashing {args.binary} " f"({len(image)} bytes)...", file=sys.stderr) @@ -101,8 +101,6 @@ def _cmd_flash(args) -> int: rate = len(image) / dt / 1024 if dt else 0 print(f"OTA complete in {dt:.1f}s ({rate:.0f} KiB/s). The device " f"activates the new image and reboots per its own policy.", file=sys.stderr) - finally: - t.close() return 0 @@ -118,8 +116,7 @@ def _cmd_list(args) -> int: def _cmd_discover(args) -> int: - t = _open_transport(args) - try: + with _make_transport(args) as t: frames = OtaClient(t).discover(timeout_ms=args.timeout) if not frames: print("no discovery reply (device may not run a Dispatcher on the " @@ -128,8 +125,6 @@ def _cmd_discover(args) -> int: for fr in frames: print(f"reply module=0x{fr.module:02x} type=0x{fr.type:02x} " f"reply={fr.is_reply} payload={len(fr.payload)} bytes") - finally: - t.close() return 0 diff --git a/components/ota/python/espp_ota/client.py b/components/ota/python/espp_ota/client.py index bedb2574a7..67e8a61753 100644 --- a/components/ota/python/espp_ota/client.py +++ b/components/ota/python/espp_ota/client.py @@ -62,8 +62,12 @@ def _transact(self, request: bytes, timeout_ms: int) -> _f.Frame: deadline = time.monotonic() + timeout_ms / 1000.0 while True: fr = self._next_frame(deadline) - if fr.module != _p.MODULE: - continue # discovery / other module chatter + # Only device->host replies on module 0 are ours. Skip other modules' + # traffic and any device-originated *request* on module 0 (reply flag + # clear) so a request whose type collides with OK/ERROR/PROGRESS can + # never be mistaken for a reply (as the browser probe also enforces). + if fr.module != _p.MODULE or not fr.is_reply: + continue if fr.type == MessageType.PROGRESS: info = _p.parse_progress(fr) if info and self._progress: @@ -88,35 +92,51 @@ def flash(self, image: bytes, image_size: Optional[int] = None) -> None: raise OtaError("empty image") size = len(image) if image_size is None else image_size - self._transact(_p.make_begin(size), self._begin_to) - - total = len(image) - sent = 0 - for off in range(0, total, self._chunk): - chunk = image[off : off + self._chunk] - ok = self._transact(_p.make_data(chunk), self._data_to) - sent += len(chunk) - # OK carries bytes_received; prefer it, fall back to our own count. - received = _p.parse_u32(ok) - if self._progress: - self._progress(received if received is not None else sent, total) - - self._transact(_p.make_finish(), self._finish_to) + # The device keeps its OTA session across a host disconnect, so a failed + # or interrupted flash would leave it "busy" and reject a retry's BEGIN. + # On any failure (device ERROR, timeout, Ctrl-C, transport error) send a + # best-effort ABORT to release the session before propagating. + try: + self._transact(_p.make_begin(size), self._begin_to) + + total = len(image) + sent = 0 + for off in range(0, total, self._chunk): + chunk = image[off : off + self._chunk] + ok = self._transact(_p.make_data(chunk), self._data_to) + sent += len(chunk) + # OK carries bytes_received; prefer it, fall back to our own count. + received = _p.parse_u32(ok) + if self._progress: + self._progress(received if received is not None else sent, total) + + self._transact(_p.make_finish(), self._finish_to) + except BaseException: + self.abort() # best-effort; swallows its own errors + raise def abort(self) -> None: + # Best-effort: called from flash()'s failure path where the transport may + # already be gone, so swallow every error (OtaError, transport, etc.). try: self._transact(_p.make_abort(), self._data_to) - except OtaError: - pass # best-effort + except Exception: + pass def discover(self, timeout_ms: int = 2000) -> List[_f.Frame]: - """Send a dispatcher ListModules request; return the reply frame(s). + """Send a dispatcher ListModules request; return the matching reply. - Useful as a connectivity probe before flashing. Returns raw frames (the - discovery TLV is not decoded here).""" + Useful as a connectivity probe before flashing. Reads until the actual + discovery reply arrives (module 0xFF, reply flag set, ListModules type), + ignoring unrelated / device-initiated frames; returns [] on timeout. The + discovery TLV payload is not decoded here.""" self._t.write(_p.make_discovery_request(), timeout_ms=self._data_to) deadline = time.monotonic() + timeout_ms / 1000.0 - try: - return [self._next_frame(deadline)] - except OtaError: - return [] + while True: + try: + fr = self._next_frame(deadline) + except OtaError: + return [] # timed out + if (fr.module == _p.DISCOVERY_MODULE and fr.is_reply + and fr.type == _p.DISCOVERY_LIST_MODULES): + return [fr] diff --git a/components/ota/python/espp_ota/transport.py b/components/ota/python/espp_ota/transport.py index 5e6406e9c3..2b3cf761fa 100644 --- a/components/ota/python/espp_ota/transport.py +++ b/components/ota/python/espp_ota/transport.py @@ -118,7 +118,7 @@ def _match(dev): if dev.is_kernel_driver_active(self._itf_num): dev.detach_kernel_driver(self._itf_num) except (NotImplementedError, self._core.USBError): - pass + pass # no kernel driver bound (or the platform can't detach) -> nothing to do self._util.claim_interface(dev, self._itf_num) self._claimed = True @@ -154,11 +154,11 @@ def close(self) -> None: if self._claimed and self._itf_num is not None: self._util.release_interface(self._dev, self._itf_num) except Exception: - pass + pass # teardown is best-effort (device may already be gone/unplugged) try: self._util.dispose_resources(self._dev) except Exception: - pass + pass # ditto: free libusb handles best-effort, never raise from close() self._dev = None self._claimed = False diff --git a/components/ota/python/tests/test_ota_host.py b/components/ota/python/tests/test_ota_host.py index 8901ee2f69..3be2a99187 100644 --- a/components/ota/python/tests/test_ota_host.py +++ b/components/ota/python/tests/test_ota_host.py @@ -76,6 +76,43 @@ def _ok(name, cond): raise SystemExit(1) +def test_frame_golden(): + """Byte-level fixtures independent of the mock loopback: any change to the + wire encoding (or a divergence from the C++ codec) breaks these.""" + # zlib/IEEE CRC-32 golden vector, same as espp::stream_frame::crc32. + _ok("golden crc vector", F.crc32(b"123456789") == 0xCBF43926) + # A request's flags byte is version 1 << 4, reply bit clear. + _ok("request flags 0x10", F.make_flags(False) == 0x10) + # Whole-frame goldens (magic "TO", flags, module, type, len LE, payload, crc LE). + _ok("BEGIN(0) bytes", + P.make_begin(0) == bytes.fromhex("544f100001040000000000000096ed77b9")) + _ok("discovery request bytes", + P.make_discovery_request() == bytes.fromhex("544f10ff000000000097e310ba")) + + +def test_parser_resync(): + """The StreamParser must skip leading garbage and a bad-CRC frame and still + yield the valid frame that follows (the interoperability-critical behavior).""" + good = P.make_begin(12345) + # leading garbage (no 0x54 magic byte) before a valid frame + p = F.StreamParser() + _ok("garbage holds", p.feed(b"\x00\xffJUNK") == []) + frames = p.feed(good) + _ok("resync past garbage", len(frames) == 1 and frames[0].type == 1 and p.dropped_bytes == 6) + # a CRC-corrupted frame followed by a good one -> only the good one survives + bad = bytearray(good) + bad[-1] ^= 0xFF + p2 = F.StreamParser() + frames2 = p2.feed(bytes(bad) + good) + _ok("skip bad CRC, keep good", + len(frames2) == 1 and frames2[0].type == 1 and p2.dropped_bytes >= 1) + # a frame split across two feeds + p3 = F.StreamParser() + half = len(good) // 2 + _ok("partial holds", p3.feed(good[:half]) == []) + _ok("completes on rest", len(p3.feed(good[half:])) == 1) + + def test_full_flash(): image = bytes(bytearray((i * 7) & 0xFF for i in range(4096 * 2 + 123))) # 2+ chunks dev = MockDevice(emit_progress=True) @@ -105,6 +142,8 @@ def test_small_image_one_chunk(): if __name__ == "__main__": + test_frame_golden() + test_parser_resync() test_full_flash() test_error_reply() test_small_image_one_chunk() From 770405ab14c26748dc9bfbc7cf465214517f6c0e Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Thu, 10 Sep 2026 08:59:22 -0500 Subject: [PATCH 03/11] fix(ota): live progress under idf.py + prominent colored errors Feedback from on-device testing: - Progress only appeared after the flash finished when run via `idf.py ota-usb`, because idf.py captures the child's stderr (not a TTY) and the carriage-return in-place bar never flushed until the final newline. _ProgressBar now detects a non-TTY stderr and emits throttled newline-terminated "OTA NN%" lines that flush live through the capturing parent; it still draws the in-place bar on a real TTY. - Errors were a plain hard-to-spot line. They now print with a distinct "espp_ota ERROR:" prefix (stands out even in idf.py's plain capture) and in red+bold on a color terminal (respects NO_COLOR / FORCE_COLOR / CLICOLOR_FORCE). No protocol/transport change; host tests still pass. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01DFjjnq3XCTRAXENSxsCxJU --- components/ota/python/espp_ota/cli.py | 81 ++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 15 deletions(-) diff --git a/components/ota/python/espp_ota/cli.py b/components/ota/python/espp_ota/cli.py index e49ee6aeb8..45b0495466 100644 --- a/components/ota/python/espp_ota/cli.py +++ b/components/ota/python/espp_ota/cli.py @@ -51,35 +51,78 @@ def _make_transport(args) -> UsbVendorTransport: interface=args.interface) +def _safe_isatty() -> bool: + try: + return sys.stderr.isatty() + except Exception: + return False + + +def _color_enabled() -> bool: + """Whether to emit ANSI color on stderr (respects NO_COLOR / *COLOR_FORCE).""" + if os.environ.get("NO_COLOR") is not None: + return False + if os.environ.get("CLICOLOR_FORCE") or os.environ.get("FORCE_COLOR"): + return True + return _safe_isatty() + + +def _paint(text: str, code: str) -> str: + return f"\033[{code}m{text}\033[0m" if _color_enabled() else text + + class _ProgressBar: + """Progress reporter that adapts to its output. + + On a TTY it draws an in-place carriage-return bar. When stderr is a pipe + (e.g. run through ``idf.py ota-usb``, whose capture buffers until a newline) + it prints throttled newline-terminated lines instead, so progress shows live + rather than all at once when the flash completes. + """ + def __init__(self, quiet: bool) -> None: self._quiet = quiet + self._tty = _safe_isatty() self._last = 0.0 + self._last_pct = -100 def __call__(self, written: int, total: int) -> None: if self._quiet: return now = time.monotonic() - done = total and written >= total - if now - self._last < 0.1 and not done: + done = bool(total) and written >= total + if self._tty: + if now - self._last < 0.1 and not done: + return + self._last = now + if total: + pct = min(100.0, 100.0 * written / total) + bar = "#" * int(pct / 2.5) + sys.stderr.write(f"\r [{bar:<40}] {pct:5.1f}% {written}/{total} B") + else: + sys.stderr.write(f"\r {written} B") + if done: + sys.stderr.write("\n") + sys.stderr.flush() return - self._last = now + # Piped: newline-terminated updates (throttled to every ~2%) so each line + # flushes through the capturing parent immediately. if total: - pct = min(100.0, 100.0 * written / total) - bar = "#" * int(pct / 2.5) - sys.stderr.write(f"\r [{bar:<40}] {pct:5.1f}% {written}/{total} B") - else: - sys.stderr.write(f"\r {written} B") - if done: - sys.stderr.write("\n") - sys.stderr.flush() + pct = int(100 * written / total) + if done or pct >= self._last_pct + 2: + self._last_pct = 100 if done else pct + print(f" OTA {self._last_pct:3d}% {written}/{total} B", + file=sys.stderr, flush=True) + elif done or now - self._last >= 0.5: + self._last = now + print(f" OTA {written} B", file=sys.stderr, flush=True) def _cmd_flash(args) -> int: with open(args.binary, "rb") as fh: image = fh.read() if not image: - print("error: image is empty", file=sys.stderr) + _error("image is empty") return 2 size = 0 if args.unknown_size else len(image) with _make_transport(args) as t: @@ -157,18 +200,26 @@ def build_parser() -> argparse.ArgumentParser: return p +def _error(msg: str) -> None: + """Print a prominent error. Red+bold on a color terminal; always a distinct + 'espp_ota ERROR:' prefix so it stands out even through idf.py's plain capture.""" + label = _paint("espp_ota ERROR:", "1;31") + sys.stderr.write(f"\n{label} {_paint(str(msg), '31')}\n") + sys.stderr.flush() + + def main(argv: Optional[list] = None) -> int: args = build_parser().parse_args(argv) try: return args.func(args) except (OtaError, TransportError) as exc: - print(f"error: {exc}", file=sys.stderr) + _error(exc) return 1 except FileNotFoundError as exc: - print(f"error: {exc}", file=sys.stderr) + _error(exc) return 2 except KeyboardInterrupt: - print("\ninterrupted", file=sys.stderr) + sys.stderr.write("\n" + _paint("interrupted", "33") + "\n") return 130 From c41ca939e6b887a34d74abc580c83a9c478cd033 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Thu, 10 Sep 2026 09:10:17 -0500 Subject: [PATCH 04/11] feat(ota): rich progress bar + colorized output for the host tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the plain per-line progress with a proper UI (new ui.py), mirroring the `rich` library that ships in the ESP-IDF Python env: - On a real terminal: a rich progress bar (spinner, bar, %, bytes, transfer speed, ETA) and colorized status/success/error lines. - Under `idf.py ota-usb` (output captured, not a TTY): rich's live display can't animate through idf.py's line capture, so we emit throttled lines ending in "(NN %)" — the pattern idf.py re-renders in place, exactly how esptool's progress shows under `idf.py flash`. Result is a live in-place text bar there. - No rich installed: degrades to plain text. Errors keep the distinct "espp_ota ERROR:" prefix and are red+bold when color is supported. `rich` added to the `usb` extra (`pip install "espp[usb]"`); it's already present in the IDF env so `idf.py ota-usb` has it for free. The frame/protocol/transport layers are unchanged; codec + resync + full-OTA host tests still pass. (Protocol speed-up is a separate, backwards-compatible follow-up.) Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01DFjjnq3XCTRAXENSxsCxJU --- components/ota/python/README.md | 10 ++ components/ota/python/espp_ota/cli.py | 124 +++++--------------- components/ota/python/espp_ota/ui.py | 160 ++++++++++++++++++++++++++ pyproject.toml | 5 +- 4 files changed, 200 insertions(+), 99 deletions(-) create mode 100644 components/ota/python/espp_ota/ui.py diff --git a/components/ota/python/README.md b/components/ota/python/README.md index 335d7bac00..03291b6cca 100644 --- a/components/ota/python/README.md +++ b/components/ota/python/README.md @@ -75,6 +75,16 @@ The wire framing is `espp::stream_frame` v2 (magic `0x4F54`, CRC-32); see `espp_ota/frame.py`. Host tests (codec + a full OTA against a mock device) live in `tests/test_ota_host.py` and run with plain `python3`. +## Output + +On a real terminal the tool draws a [`rich`](https://pypi.org/project/rich/) +progress bar (spinner, bar, %, bytes, transfer speed, ETA) and colorizes status / +error lines. Run through `idf.py ota-usb`, whose output capture re-renders lines +ending in `(NN %)` in place (the same way esptool's progress shows under +`idf.py flash`), it emits a live in-place text bar. `rich` is optional — without +it the output degrades to plain text. It ships in the ESP-IDF Python environment +(so `idf.py ota-usb` already has it) and is pulled in by `pip install "espp[usb]"`. + ## Requirements - Python 3.8+ diff --git a/components/ota/python/espp_ota/cli.py b/components/ota/python/espp_ota/cli.py index 45b0495466..6171b902e8 100644 --- a/components/ota/python/espp_ota/cli.py +++ b/components/ota/python/espp_ota/cli.py @@ -18,11 +18,13 @@ import time from typing import Optional -from . import __version__ +from . import __version__, ui from .client import OtaClient from .protocol import OtaError from .transport import DEFAULT_PID, DEFAULT_VID, TransportError, UsbVendorTransport, list_devices +CON = ui.Console() + def _auto_int(text: str) -> int: return int(text, 0) # accepts 0x1209, 4617, etc. @@ -51,99 +53,33 @@ def _make_transport(args) -> UsbVendorTransport: interface=args.interface) -def _safe_isatty() -> bool: - try: - return sys.stderr.isatty() - except Exception: - return False - - -def _color_enabled() -> bool: - """Whether to emit ANSI color on stderr (respects NO_COLOR / *COLOR_FORCE).""" - if os.environ.get("NO_COLOR") is not None: - return False - if os.environ.get("CLICOLOR_FORCE") or os.environ.get("FORCE_COLOR"): - return True - return _safe_isatty() - - -def _paint(text: str, code: str) -> str: - return f"\033[{code}m{text}\033[0m" if _color_enabled() else text - - -class _ProgressBar: - """Progress reporter that adapts to its output. - - On a TTY it draws an in-place carriage-return bar. When stderr is a pipe - (e.g. run through ``idf.py ota-usb``, whose capture buffers until a newline) - it prints throttled newline-terminated lines instead, so progress shows live - rather than all at once when the flash completes. - """ - - def __init__(self, quiet: bool) -> None: - self._quiet = quiet - self._tty = _safe_isatty() - self._last = 0.0 - self._last_pct = -100 - - def __call__(self, written: int, total: int) -> None: - if self._quiet: - return - now = time.monotonic() - done = bool(total) and written >= total - if self._tty: - if now - self._last < 0.1 and not done: - return - self._last = now - if total: - pct = min(100.0, 100.0 * written / total) - bar = "#" * int(pct / 2.5) - sys.stderr.write(f"\r [{bar:<40}] {pct:5.1f}% {written}/{total} B") - else: - sys.stderr.write(f"\r {written} B") - if done: - sys.stderr.write("\n") - sys.stderr.flush() - return - # Piped: newline-terminated updates (throttled to every ~2%) so each line - # flushes through the capturing parent immediately. - if total: - pct = int(100 * written / total) - if done or pct >= self._last_pct + 2: - self._last_pct = 100 if done else pct - print(f" OTA {self._last_pct:3d}% {written}/{total} B", - file=sys.stderr, flush=True) - elif done or now - self._last >= 0.5: - self._last = now - print(f" OTA {written} B", file=sys.stderr, flush=True) - - def _cmd_flash(args) -> int: with open(args.binary, "rb") as fh: image = fh.read() if not image: - _error("image is empty") + CON.error("image is empty") return 2 size = 0 if args.unknown_size else len(image) with _make_transport(args) as t: if not args.quiet: - print(f"Connected to {t.description}; flashing {args.binary} " - f"({len(image)} bytes)...", file=sys.stderr) - client = OtaClient( - t, - chunk_size=args.chunk_size, - progress=_ProgressBar(args.quiet), - begin_timeout_ms=args.begin_timeout, - data_timeout_ms=args.data_timeout, - finish_timeout_ms=args.finish_timeout, - ) + CON.info(f"Connected to {t.description}; flashing {args.binary} " + f"({len(image)} bytes)…") start = time.monotonic() - client.flash(image, image_size=size) + with ui.Progress(len(image), label="Flashing", quiet=args.quiet) as prog: + client = OtaClient( + t, + chunk_size=args.chunk_size, + progress=prog.update, + begin_timeout_ms=args.begin_timeout, + data_timeout_ms=args.data_timeout, + finish_timeout_ms=args.finish_timeout, + ) + client.flash(image, image_size=size) if not args.quiet: dt = time.monotonic() - start rate = len(image) / dt / 1024 if dt else 0 - print(f"OTA complete in {dt:.1f}s ({rate:.0f} KiB/s). The device " - f"activates the new image and reboots per its own policy.", file=sys.stderr) + CON.success(f"OTA complete in {dt:.1f}s ({rate:.0f} KiB/s). The device " + f"activates the new image and reboots per its own policy.") return 0 @@ -151,7 +87,7 @@ def _cmd_list(args) -> int: pid = None if args.pid is not None and args.pid < 0 else args.pid found = list_devices(vid=args.vid, pid=pid) if not found: - print("no matching USB devices found", file=sys.stderr) + CON.warn("no matching USB devices found") return 1 for vid, pid_, desc in found: print(f"0x{vid:04x}:0x{pid_:04x} {desc}") @@ -162,12 +98,12 @@ def _cmd_discover(args) -> int: with _make_transport(args) as t: frames = OtaClient(t).discover(timeout_ms=args.timeout) if not frames: - print("no discovery reply (device may not run a Dispatcher on the " - "vendor interface)", file=sys.stderr) + CON.warn("no discovery reply (device may not run a Dispatcher on the " + "vendor interface)") return 1 for fr in frames: - print(f"reply module=0x{fr.module:02x} type=0x{fr.type:02x} " - f"reply={fr.is_reply} payload={len(fr.payload)} bytes") + CON.info(f"reply module=0x{fr.module:02x} type=0x{fr.type:02x} " + f"reply={fr.is_reply} payload={len(fr.payload)} bytes") return 0 @@ -200,26 +136,18 @@ def build_parser() -> argparse.ArgumentParser: return p -def _error(msg: str) -> None: - """Print a prominent error. Red+bold on a color terminal; always a distinct - 'espp_ota ERROR:' prefix so it stands out even through idf.py's plain capture.""" - label = _paint("espp_ota ERROR:", "1;31") - sys.stderr.write(f"\n{label} {_paint(str(msg), '31')}\n") - sys.stderr.flush() - - def main(argv: Optional[list] = None) -> int: args = build_parser().parse_args(argv) try: return args.func(args) except (OtaError, TransportError) as exc: - _error(exc) + CON.error(exc) return 1 except FileNotFoundError as exc: - _error(exc) + CON.error(exc) return 2 except KeyboardInterrupt: - sys.stderr.write("\n" + _paint("interrupted", "33") + "\n") + CON.warn("interrupted") return 130 diff --git a/components/ota/python/espp_ota/ui.py b/components/ota/python/espp_ota/ui.py new file mode 100644 index 0000000000..dc8e68a2e3 --- /dev/null +++ b/components/ota/python/espp_ota/ui.py @@ -0,0 +1,160 @@ +"""Terminal UI: a nice progress bar + colorized messages, with graceful fallback. + +Two rendering paths so it looks good both standalone and under ``idf.py``: + +* **Standalone, real terminal** — if `rich` is available (it ships in the + ESP-IDF Python environment, and `pip install "espp[usb]"` pulls it in) we draw + a `rich` progress bar (spinner, bar, %, bytes, transfer speed, ETA) and print + colorized status/error lines. +* **Captured (e.g. under `idf.py ota-usb`)** — idf.py reads the target's output + line-by-line and re-renders any line ending in ``(NN %)`` *in place* (the same + mechanism that makes esptool's progress animate under `idf.py flash`). So there + we emit throttled ``… (NN %)`` lines, which idf.py turns into a live in-place + bar. `rich`'s own live display can't animate through that line capture, so it's + intentionally only used on a real TTY. + +Everything degrades to plain text; `rich` is optional. +""" + +from __future__ import annotations + +import os +import sys +import time +from typing import Optional + + +def _isatty() -> bool: + try: + return sys.stderr.isatty() + except Exception: + return False + + +def _have_rich() -> bool: + try: + import rich # noqa: F401 + return True + except Exception: + return False + + +def _ansi_enabled() -> bool: + if os.environ.get("NO_COLOR") is not None: + return False + if os.environ.get("CLICOLOR_FORCE") or os.environ.get("FORCE_COLOR"): + return True + return _isatty() + + +class Console: + """Styled status/error output (stderr). Uses rich when available.""" + + def __init__(self) -> None: + self._rich = None + if _have_rich(): + try: + from rich.console import Console as RichConsole + self._rich = RichConsole(file=sys.stderr, highlight=False) + except Exception: + self._rich = None + + def _emit(self, text: str, rich_style: Optional[str], ansi: Optional[str]) -> None: + if self._rich is not None: + self._rich.print(text, style=rich_style, soft_wrap=True) + return + if ansi and _ansi_enabled(): + text = f"\033[{ansi}m{text}\033[0m" + sys.stderr.write(text + "\n") + sys.stderr.flush() + + def info(self, text: str) -> None: + self._emit(text, None, None) + + def success(self, text: str) -> None: + self._emit(text, "bold green", "1;32") + + def warn(self, text: str) -> None: + self._emit(text, "yellow", "33") + + def error(self, text: str) -> None: + # Distinct prefix so it stands out even when color is stripped (idf.py). + self._emit(f"espp_ota ERROR: {text}", "bold red", "1;31") + + +class Progress: + """OTA progress reporter; use as a context manager, feed :meth:`update`. + + with Progress(total, "Flashing", quiet) as p: + client.flash(image) # progress=p.update + """ + + def __init__(self, total: int, label: str = "Flashing", quiet: bool = False) -> None: + self._total = total or 0 + self._label = label + self._quiet = quiet + self._rich = None + self._task = None + self._last_pct = -1000 + self._last_t = 0.0 + # rich's live bar can't animate through idf.py's line capture, so use it + # only on a real terminal; otherwise emit "(NN %)" lines idf.py renders. + self._use_rich = (not quiet) and _isatty() and _have_rich() + + def __enter__(self) -> "Progress": + if self._use_rich: + try: + from rich.console import Console as RichConsole + from rich.progress import (BarColumn, DownloadColumn, Progress as RichProgress, + SpinnerColumn, TaskProgressColumn, TextColumn, + TimeRemainingColumn, TransferSpeedColumn) + self._rich = RichProgress( + SpinnerColumn(), + TextColumn("[bold blue]{task.description}"), + BarColumn(), + TaskProgressColumn(), + DownloadColumn(), + TransferSpeedColumn(), + TimeRemainingColumn(), + console=RichConsole(file=sys.stderr), + ) + self._rich.start() + self._task = self._rich.add_task(self._label, total=self._total or None) + except Exception: + self._rich = None # fall back to text lines + return self + + def update(self, written: int, total: int) -> None: + if self._quiet: + return + if self._rich is not None: + self._rich.update(self._task, completed=written, total=total or None) + return + now = time.monotonic() + done = bool(total) and written >= total + if total: + pct = int(100 * written / total) + # every 1% (and always the final frame). The line ends in "(NN %)" so + # idf.py re-renders it in place; standalone it prints one line per %. + if done or pct >= self._last_pct + 1: + self._last_pct = 100 if done else pct + bar = self._text_bar(self._last_pct) + sys.stderr.write(f" {self._label} {bar} {written // 1024:>5}/" + f"{total // 1024} KB ({self._last_pct} %)\n") + sys.stderr.flush() + elif done or now - self._last_t >= 0.5: + self._last_t = now + sys.stderr.write(f" {self._label} {written // 1024} KB\n") + sys.stderr.flush() + + @staticmethod + def _text_bar(pct: int, width: int = 24) -> str: + filled = min(width, max(0, pct * width // 100)) + return "[" + "#" * filled + "-" * (width - filled) + "]" + + def __exit__(self, *exc) -> None: + if self._rich is not None: + try: + self._rich.stop() + except Exception: + pass diff --git a/pyproject.toml b/pyproject.toml index 577630b772..24395f81b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,10 @@ Issues = "https://github.com/esp-cpp/espp/issues" serial = ["pyserial"] # espp_ota's USB transport imports pyusb lazily; the frame codec + OTA protocol # are stdlib-only. `pip install espp[usb]` enables the OTA-over-USB host tool. -usb = ["pyusb"] +# `rich` is optional but gives the nice progress bar / colorized output (it also +# ships in the ESP-IDF Python env, so `idf.py ota-usb` has it already); the tool +# degrades to plain text without it. +usb = ["pyusb", "rich"] [project.scripts] # Standalone CLI (also runnable as `python -m espp_ota`). The `ota` component's From 98130ffa335d3cf50de205c0052da6906479a148 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Thu, 10 Sep 2026 09:15:09 -0500 Subject: [PATCH 05/11] fix(ota): draw the progress bar on the controlling terminal (idf.py) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous attempt relied on idf.py re-rendering lines ending in "(NN %)" in place, but idf.py only does that for its own build progression (needs force_progression + its stdout being a tty), so under `idf.py ota-usb` the lines just scrolled. idf.py captures the target's stdout/stderr as pipes, but the process still has a controlling terminal. So the progress bar now opens that terminal directly (/dev/tty, or CONOUT$ on Windows) and renders the rich bar there, bypassing the capture — a real animated bar under idf.py. Falls back to the tty stream with a manual \r bar when rich is absent, and to periodic plain lines when there is no terminal at all (CI / fully redirected). Status/error messages still go to stderr (which idf.py forwards, with color). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01DFjjnq3XCTRAXENSxsCxJU --- components/ota/python/README.md | 15 +++--- components/ota/python/espp_ota/ui.py | 78 ++++++++++++++++++++++------ 2 files changed, 71 insertions(+), 22 deletions(-) diff --git a/components/ota/python/README.md b/components/ota/python/README.md index 03291b6cca..89ffbb14ad 100644 --- a/components/ota/python/README.md +++ b/components/ota/python/README.md @@ -77,13 +77,14 @@ in `tests/test_ota_host.py` and run with plain `python3`. ## Output -On a real terminal the tool draws a [`rich`](https://pypi.org/project/rich/) -progress bar (spinner, bar, %, bytes, transfer speed, ETA) and colorizes status / -error lines. Run through `idf.py ota-usb`, whose output capture re-renders lines -ending in `(NN %)` in place (the same way esptool's progress shows under -`idf.py flash`), it emits a live in-place text bar. `rich` is optional — without -it the output degrades to plain text. It ships in the ESP-IDF Python environment -(so `idf.py ota-usb` already has it) and is pulled in by `pip install "espp[usb]"`. +The tool draws a [`rich`](https://pypi.org/project/rich/) progress bar (spinner, +bar, %, bytes, transfer speed, ETA) and colorizes status / error lines. Under +`idf.py ota-usb` the tool's stdout/stderr are captured pipes, so the bar is drawn +straight to the controlling terminal (`/dev/tty`, `CONOUT$` on Windows) and still +animates in place. Without a terminal (CI / redirected output) it prints periodic +plain-text lines instead. `rich` is optional — the output degrades to a plain +`\r` bar or text without it. It ships in the ESP-IDF Python environment (so +`idf.py ota-usb` already has it) and is pulled in by `pip install "espp[usb]"`. ## Requirements diff --git a/components/ota/python/espp_ota/ui.py b/components/ota/python/espp_ota/ui.py index dc8e68a2e3..059982b06c 100644 --- a/components/ota/python/espp_ota/ui.py +++ b/components/ota/python/espp_ota/ui.py @@ -31,6 +31,28 @@ def _isatty() -> bool: return False +def _open_progress_stream(): + """A writable stream connected to the real terminal, plus an 'owned' flag. + + A live progress bar needs a terminal to animate on. Under ``idf.py ota-usb`` + the tool's stdout/stderr are captured pipes (so it forwards our lines one at a + time, scrolling), but the process still has a *controlling terminal* — so we + open ``/dev/tty`` (``CONOUT$`` on Windows) and draw the bar straight to it, + bypassing the capture. Returns ``(stream, owned)`` or ``(None, False)`` when + there is no terminal at all (CI, fully redirected).""" + try: + if sys.stderr.isatty(): + return sys.stderr, False + except Exception: + pass + for name in ("/dev/tty", "CONOUT$"): + try: + return open(name, "w"), True + except Exception: + continue + return None, False + + def _have_rich() -> bool: try: import rich # noqa: F401 @@ -93,21 +115,27 @@ def __init__(self, total: int, label: str = "Flashing", quiet: bool = False) -> self._total = total or 0 self._label = label self._quiet = quiet - self._rich = None + self._rich = None # rich Progress, when available self._task = None + self._term = None # a real-terminal stream for an in-place bar + self._own_term = False + self._plain = False # draw a manual \r bar on self._term self._last_pct = -1000 self._last_t = 0.0 - # rich's live bar can't animate through idf.py's line capture, so use it - # only on a real terminal; otherwise emit "(NN %)" lines idf.py renders. - self._use_rich = (not quiet) and _isatty() and _have_rich() + self._newline_done = False def __enter__(self) -> "Progress": - if self._use_rich: + if self._quiet: + return self + self._term, self._own_term = _open_progress_stream() + if self._term is not None and _have_rich(): try: from rich.console import Console as RichConsole from rich.progress import (BarColumn, DownloadColumn, Progress as RichProgress, SpinnerColumn, TaskProgressColumn, TextColumn, TimeRemainingColumn, TransferSpeedColumn) + # force_terminal: the stream is a real tty (possibly /dev/tty) even + # though our stdout/stderr were captured by idf.py. self._rich = RichProgress( SpinnerColumn(), TextColumn("[bold blue]{task.description}"), @@ -116,12 +144,14 @@ def __enter__(self) -> "Progress": DownloadColumn(), TransferSpeedColumn(), TimeRemainingColumn(), - console=RichConsole(file=sys.stderr), + console=RichConsole(file=self._term, force_terminal=True), ) self._rich.start() self._task = self._rich.add_task(self._label, total=self._total or None) except Exception: - self._rich = None # fall back to text lines + self._rich = None + if self._rich is None and self._term is not None: + self._plain = True # manual in-place bar on the terminal return self def update(self, written: int, total: int) -> None: @@ -132,23 +162,36 @@ def update(self, written: int, total: int) -> None: return now = time.monotonic() done = bool(total) and written >= total + if self._plain: + # in-place carriage-return bar on the real terminal (throttled ~10 Hz) + if now - self._last_t < 0.1 and not done: + return + self._last_t = now + if total: + pct = min(100, int(100 * written / total)) + self._term.write(f"\r {self._label} {self._text_bar(pct)} " + f"{written // 1024}/{total // 1024} KB {pct:3d}%") + else: + self._term.write(f"\r {self._label} {written // 1024} KB") + if done: + self._term.write("\n") + self._term.flush() + return + # No terminal at all (CI / fully redirected): throttled newline lines. if total: pct = int(100 * written / total) - # every 1% (and always the final frame). The line ends in "(NN %)" so - # idf.py re-renders it in place; standalone it prints one line per %. - if done or pct >= self._last_pct + 1: + if done or pct >= self._last_pct + 5: self._last_pct = 100 if done else pct - bar = self._text_bar(self._last_pct) - sys.stderr.write(f" {self._label} {bar} {written // 1024:>5}/" - f"{total // 1024} KB ({self._last_pct} %)\n") + sys.stderr.write(f" {self._label} {written // 1024}/{total // 1024} KB " + f"({self._last_pct} %)\n") sys.stderr.flush() - elif done or now - self._last_t >= 0.5: + elif done or now - self._last_t >= 1.0: self._last_t = now sys.stderr.write(f" {self._label} {written // 1024} KB\n") sys.stderr.flush() @staticmethod - def _text_bar(pct: int, width: int = 24) -> str: + def _text_bar(pct: int, width: int = 28) -> str: filled = min(width, max(0, pct * width // 100)) return "[" + "#" * filled + "-" * (width - filled) + "]" @@ -158,3 +201,8 @@ def __exit__(self, *exc) -> None: self._rich.stop() except Exception: pass + try: + if self._own_term and self._term is not None: + self._term.close() + except Exception: + pass From ec9e2d9f1d1353871145a6f8922bea7c38dd5c14 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Thu, 10 Sep 2026 09:24:21 -0500 Subject: [PATCH 06/11] feat(ota): surface the connected device + add screenshots to docs/README - The "Connected to :" line is now a prominent bold-cyan status line (Console.note) on its own, with the flash target + human-readable size beneath it; the transport description spells out "(interface N)". - Add the on-device idf.py ota-usb screenshots (mid-flash progress bar and the completed flash) to the tool README and the ota docs page. Images are hosted GitHub user-content, not committed to the repo. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01DFjjnq3XCTRAXENSxsCxJU --- components/ota/python/README.md | 9 +++++++ components/ota/python/espp_ota/cli.py | 13 +++++++++-- components/ota/python/espp_ota/transport.py | 2 +- components/ota/python/espp_ota/ui.py | 4 ++++ doc/en/ota/ota.rst | 26 +++++++++++++++++++++ 5 files changed, 51 insertions(+), 3 deletions(-) diff --git a/components/ota/python/README.md b/components/ota/python/README.md index 89ffbb14ad..ab108292ec 100644 --- a/components/ota/python/README.md +++ b/components/ota/python/README.md @@ -86,6 +86,15 @@ plain-text lines instead. `rich` is optional — the output degrades to a plain `\r` bar or text without it. It ships in the ESP-IDF Python environment (so `idf.py ota-usb` already has it) and is pulled in by `pip install "espp[usb]"`. +`idf.py ota-usb` mid-flash — the rich bar (%, size, transfer speed, ETA) animates +in place even though idf.py captures the tool's output: + +![espp_ota flashing over USB](https://github.com/user-attachments/assets/a042481a-2964-4b06-9109-bb2dcb4e355b) + +…and on completion: + +![espp_ota OTA complete](https://github.com/user-attachments/assets/da8e1b71-c65f-4ecc-9223-e232f8591ceb) + ## Requirements - Python 3.8+ diff --git a/components/ota/python/espp_ota/cli.py b/components/ota/python/espp_ota/cli.py index 6171b902e8..7de669ceac 100644 --- a/components/ota/python/espp_ota/cli.py +++ b/components/ota/python/espp_ota/cli.py @@ -30,6 +30,15 @@ def _auto_int(text: str) -> int: return int(text, 0) # accepts 0x1209, 4617, etc. +def _human_size(n: int) -> str: + size = float(n) + for unit in ("B", "KiB", "MiB", "GiB"): + if size < 1024 or unit == "GiB": + return f"{int(size)} {unit}" if unit == "B" else f"{size:.1f} {unit}" + size /= 1024 + return f"{n} B" + + def _env_int(name: str, default: int) -> int: val = os.environ.get(name) return _auto_int(val) if val else default @@ -62,8 +71,8 @@ def _cmd_flash(args) -> int: size = 0 if args.unknown_size else len(image) with _make_transport(args) as t: if not args.quiet: - CON.info(f"Connected to {t.description}; flashing {args.binary} " - f"({len(image)} bytes)…") + CON.note(f"● Connected to {t.description}") + CON.info(f" Flashing {args.binary} ({_human_size(len(image))})") start = time.monotonic() with ui.Progress(len(image), label="Flashing", quiet=args.quiet) as prog: client = OtaClient( diff --git a/components/ota/python/espp_ota/transport.py b/components/ota/python/espp_ota/transport.py index 2b3cf761fa..cdf29e9a3b 100644 --- a/components/ota/python/espp_ota/transport.py +++ b/components/ota/python/espp_ota/transport.py @@ -189,4 +189,4 @@ def read(self, max_len: int, timeout_ms: int = 5000) -> bytes: def description(self) -> str: if self._dev is None: return "" - return f"0x{self._dev.idVendor:04x}:0x{self._dev.idProduct:04x} itf {self._itf_num}" + return f"0x{self._dev.idVendor:04x}:0x{self._dev.idProduct:04x} (interface {self._itf_num})" diff --git a/components/ota/python/espp_ota/ui.py b/components/ota/python/espp_ota/ui.py index 059982b06c..420a9a760b 100644 --- a/components/ota/python/espp_ota/ui.py +++ b/components/ota/python/espp_ota/ui.py @@ -93,6 +93,10 @@ def _emit(self, text: str, rich_style: Optional[str], ansi: Optional[str]) -> No def info(self, text: str) -> None: self._emit(text, None, None) + def note(self, text: str) -> None: + """A prominent status line (e.g. the connected device) — bold cyan.""" + self._emit(text, "bold cyan", "1;36") + def success(self, text: str) -> None: self._emit(text, "bold green", "1;32") diff --git a/doc/en/ota/ota.rst b/doc/en/ota/ota.rst index c00662bafa..ca52751a9d 100644 --- a/doc/en/ota/ota.rst +++ b/doc/en/ota/ota.rst @@ -34,6 +34,32 @@ message types on top); to run OTA alongside other protocols (crash-dump, CAN, :doc:`../dispatcher/index` — the ``ota`` example does exactly this (OTA is module id 0). +Command line: build → OTA +------------------------- + +The ``ota`` component ships a ``project_include.cmake`` and a pure-Python host +tool (``components/ota/python/espp_ota``), so any project using it can build and +OTA-flash over USB in one step — the OTA counterpart to ``idf.py flash``:: + + pip install pyusb # once (needs a libusb backend) + idf.py ota-usb # builds the app, then OTAs it over USB + +The tool draws a live progress bar (percent, size, transfer speed, ETA) and +colorizes its output. Because ``idf.py`` captures the target's output, the bar is +drawn straight to the controlling terminal so it still animates in place: + +.. image:: https://github.com/user-attachments/assets/a042481a-2964-4b06-9109-bb2dcb4e355b + :alt: espp_ota flashing an image over USB via idf.py ota-usb + :width: 100% + +.. image:: https://github.com/user-attachments/assets/da8e1b71-c65f-4ecc-9223-e232f8591ceb + :alt: espp_ota reporting a completed OTA over USB + :width: 100% + +For full control (a specific serial, chunk size, discovery probe) run it directly +with ``python -m espp_ota flash build/.bin`` — see +``components/ota/python/README.md``. + .. ------------------------------- Example ------------------------------------- .. toctree:: From 25a2868205f6503abe0e4dac8586e35e398e5f78 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Thu, 10 Sep 2026 10:05:45 -0500 Subject: [PATCH 07/11] fix(ota): address host-tool PR review - pyproject: `rich` is no longer forced by the `usb` extra. `usb = ["pyusb"]` again; a new `usb-ui = ["pyusb", "rich"]` adds the nicer UI (docs updated). - frame.py: StreamParser.reset() now also clears dropped_bytes so a reused parser fully resets. - client.py: _transact() uses its `timeout_ms` argument for the write side too, so BEGIN/FINISH writes get their own (larger) timeout instead of the DATA one. - transport.py: read() only treats a genuine timeout (USBTimeoutError, or errno 110/ETIMEDOUT) as empty; other USBError I/O failures propagate instead of being swallowed by the previous `errno is None` catch-all. - project_include.cmake: prepend the package dir to PYTHONPATH (host path separator) instead of overwriting an existing PYTHONPATH. - ui.py: comment the two best-effort empty-except blocks. Host tests pass with python3. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01DFjjnq3XCTRAXENSxsCxJU --- components/ota/project_include.cmake | 16 +++++++++++++++- components/ota/python/README.md | 4 ++-- components/ota/python/espp_ota/client.py | 2 +- components/ota/python/espp_ota/frame.py | 1 + components/ota/python/espp_ota/transport.py | 10 ++++++++-- components/ota/python/espp_ota/ui.py | 4 ++-- pyproject.toml | 9 +++++---- 7 files changed, 34 insertions(+), 12 deletions(-) diff --git a/components/ota/project_include.cmake b/components/ota/project_include.cmake index 66c9b4a581..b13e3dfbf9 100644 --- a/components/ota/project_include.cmake +++ b/components/ota/project_include.cmake @@ -25,8 +25,22 @@ if(NOT TARGET ota-usb) # idf_build_process includes this file); the app .bin lands in the build dir. set(__espp_ota_bin "${CMAKE_BINARY_DIR}/${CMAKE_PROJECT_NAME}.bin") + # Prepend our package dir to PYTHONPATH rather than replacing it, so a + # PYTHONPATH the environment already relies on is preserved. Use the host's + # path separator. ($ENV{PYTHONPATH} is the value at configure time, which is + # the same environment `idf.py ota-usb` runs in.) + if(WIN32) + set(__espp_ota_pathsep ";") + else() + set(__espp_ota_pathsep ":") + endif() + set(__espp_ota_pythonpath "${__espp_ota_pkg_dir}") + if(DEFINED ENV{PYTHONPATH} AND NOT "$ENV{PYTHONPATH}" STREQUAL "") + set(__espp_ota_pythonpath "${__espp_ota_pkg_dir}${__espp_ota_pathsep}$ENV{PYTHONPATH}") + endif() + add_custom_target(ota-usb - COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${__espp_ota_pkg_dir}" + COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${__espp_ota_pythonpath}" ${python} -m espp_ota flash "${__espp_ota_bin}" WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" VERBATIM diff --git a/components/ota/python/README.md b/components/ota/python/README.md index ab108292ec..444a85478b 100644 --- a/components/ota/python/README.md +++ b/components/ota/python/README.md @@ -41,7 +41,7 @@ python -m espp_ota discover # probe the device's dispatche ``` Installed with the espp wheel it's also available as the `espp-ota` command -(`pip install "espp[usb]"`). +(`pip install "espp[usb]"`, or `"espp[usb-ui]"` to also get the `rich` UI). ## Library use @@ -84,7 +84,7 @@ straight to the controlling terminal (`/dev/tty`, `CONOUT$` on Windows) and stil animates in place. Without a terminal (CI / redirected output) it prints periodic plain-text lines instead. `rich` is optional — the output degrades to a plain `\r` bar or text without it. It ships in the ESP-IDF Python environment (so -`idf.py ota-usb` already has it) and is pulled in by `pip install "espp[usb]"`. +`idf.py ota-usb` already has it) and is pulled in by `pip install "espp[usb-ui]"`. `idf.py ota-usb` mid-flash — the rich bar (%, size, transfer speed, ETA) animates in place even though idf.py captures the tool's output: diff --git a/components/ota/python/espp_ota/client.py b/components/ota/python/espp_ota/client.py index 67e8a61753..5b943a1fda 100644 --- a/components/ota/python/espp_ota/client.py +++ b/components/ota/python/espp_ota/client.py @@ -58,7 +58,7 @@ def _transact(self, request: bytes, timeout_ms: int) -> _f.Frame: PROGRESS frames are surfaced to the callback and skipped; an ERROR reply raises :class:`OtaError`; frames for other modules are ignored.""" - self._t.write(request, timeout_ms=self._data_to) + self._t.write(request, timeout_ms=timeout_ms) deadline = time.monotonic() + timeout_ms / 1000.0 while True: fr = self._next_frame(deadline) diff --git a/components/ota/python/espp_ota/frame.py b/components/ota/python/espp_ota/frame.py index 0cd7a3d770..6a407e32df 100644 --- a/components/ota/python/espp_ota/frame.py +++ b/components/ota/python/espp_ota/frame.py @@ -115,6 +115,7 @@ def __init__(self) -> None: def reset(self) -> None: self._buf.clear() + self.dropped_bytes = 0 def buffered(self) -> int: return len(self._buf) diff --git a/components/ota/python/espp_ota/transport.py b/components/ota/python/espp_ota/transport.py index cdf29e9a3b..9ed003f145 100644 --- a/components/ota/python/espp_ota/transport.py +++ b/components/ota/python/espp_ota/transport.py @@ -179,8 +179,14 @@ def read(self, max_len: int, timeout_ms: int = 5000) -> bytes: try: arr = self._ep_in.read(max_len, timeout_ms) except self._core.USBError as exc: - # errno 110 == ETIMEDOUT; pyusb>=1.1 raises the USBTimeoutError subclass. - if getattr(exc, "errno", None) in (110, None) or "timeout" in str(exc).lower(): + # A genuine timeout is expected (poll again), but a real I/O error must + # propagate. pyusb>=1.1 raises the USBTimeoutError subclass; older pyusb + # raises USBError with errno 110 (ETIMEDOUT). Do NOT treat an unknown + # errno as a timeout — that would silently swallow backend failures. + timeout_cls = getattr(self._core, "USBTimeoutError", None) + is_timeout = (timeout_cls is not None and isinstance(exc, timeout_cls)) or ( + getattr(exc, "errno", None) == 110) + if is_timeout: return b"" raise return bytes(arr) diff --git a/components/ota/python/espp_ota/ui.py b/components/ota/python/espp_ota/ui.py index 420a9a760b..b6f0ee19f2 100644 --- a/components/ota/python/espp_ota/ui.py +++ b/components/ota/python/espp_ota/ui.py @@ -204,9 +204,9 @@ def __exit__(self, *exc) -> None: try: self._rich.stop() except Exception: - pass + pass # tearing down the display must never raise try: if self._own_term and self._term is not None: self._term.close() except Exception: - pass + pass # closing the borrowed /dev/tty handle is best-effort diff --git a/pyproject.toml b/pyproject.toml index 24395f81b5..2ebb0710b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,10 +37,11 @@ Issues = "https://github.com/esp-cpp/espp/issues" serial = ["pyserial"] # espp_ota's USB transport imports pyusb lazily; the frame codec + OTA protocol # are stdlib-only. `pip install espp[usb]` enables the OTA-over-USB host tool. -# `rich` is optional but gives the nice progress bar / colorized output (it also -# ships in the ESP-IDF Python env, so `idf.py ota-usb` has it already); the tool -# degrades to plain text without it. -usb = ["pyusb", "rich"] +usb = ["pyusb"] +# `rich` is optional — it gives the nicer progress bar / colorized output (it also +# ships in the ESP-IDF Python env, so `idf.py ota-usb` already has it); the tool +# degrades to plain text without it. `pip install espp[usb-ui]` adds it. +usb-ui = ["pyusb", "rich"] [project.scripts] # Standalone CLI (also runnable as `python -m espp_ota`). The `ota` component's From ed9ce91f0cf00195dc324c5b686a3576f9a5d257 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Thu, 10 Sep 2026 12:40:50 -0500 Subject: [PATCH 08/11] style(ota): comment the remaining empty-except blocks Add inline explanations to the two `except Exception: pass` blocks the code-quality check still flagged (client.abort() best-effort cleanup, and the isatty() guard in ui._open_progress_stream()). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01DFjjnq3XCTRAXENSxsCxJU --- components/ota/python/espp_ota/client.py | 2 +- components/ota/python/espp_ota/ui.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/ota/python/espp_ota/client.py b/components/ota/python/espp_ota/client.py index 5b943a1fda..2c52b184de 100644 --- a/components/ota/python/espp_ota/client.py +++ b/components/ota/python/espp_ota/client.py @@ -121,7 +121,7 @@ def abort(self) -> None: try: self._transact(_p.make_abort(), self._data_to) except Exception: - pass + pass # best-effort cleanup; the link may already be gone def discover(self, timeout_ms: int = 2000) -> List[_f.Frame]: """Send a dispatcher ListModules request; return the matching reply. diff --git a/components/ota/python/espp_ota/ui.py b/components/ota/python/espp_ota/ui.py index b6f0ee19f2..36fa02ca9f 100644 --- a/components/ota/python/espp_ota/ui.py +++ b/components/ota/python/espp_ota/ui.py @@ -44,7 +44,7 @@ def _open_progress_stream(): if sys.stderr.isatty(): return sys.stderr, False except Exception: - pass + pass # stderr may not support isatty() (e.g. a wrapped stream); fall through for name in ("/dev/tty", "CONOUT$"): try: return open(name, "w"), True From 674177d84b790171c238de6b1cd19c049524c9fa Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Thu, 10 Sep 2026 20:13:48 -0500 Subject: [PATCH 09/11] fix(ota): handle stale sessions, USB errors, and correct the rich extra docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address the latest host-tool review: - client.flash(): if BEGIN fails (a prior interrupted flash left the device's OTA session open, so it rejects the new BEGIN as busy), send an ABORT to clear the stale session and retry BEGIN once before giving up. Added a mock-device test (BEGIN rejected as busy until ABORT) covering the recovery. - cli.main(): catch OSError so routine pyusb failures — pyusb's USBError derives from OSError/IOError (unplug mid-flash, permission denied, missing libusb backend) — are reported cleanly instead of dumping a traceback. - ui.py docstring: point to the correct extra (`espp[usb-ui]` pulls in rich, not `espp[usb]`) and drop the stale "(NN %)" idf.py mechanism description (the bar is now drawn on the controlling terminal, see _open_progress_stream). Host tests pass with python3. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01DFjjnq3XCTRAXENSxsCxJU --- components/ota/python/espp_ota/cli.py | 6 +++++ components/ota/python/espp_ota/client.py | 14 ++++++++---- components/ota/python/espp_ota/ui.py | 24 ++++++++------------ components/ota/python/tests/test_ota_host.py | 16 +++++++++++++ 4 files changed, 42 insertions(+), 18 deletions(-) diff --git a/components/ota/python/espp_ota/cli.py b/components/ota/python/espp_ota/cli.py index 7de669ceac..037c58267c 100644 --- a/components/ota/python/espp_ota/cli.py +++ b/components/ota/python/espp_ota/cli.py @@ -155,6 +155,12 @@ def main(argv: Optional[list] = None) -> int: except FileNotFoundError as exc: CON.error(exc) return 2 + except OSError as exc: + # pyusb's USBError derives from OSError/IOError, so routine USB failures + # (unplug mid-flash, permission denied, missing libusb backend) land here + # instead of raising an ugly traceback. Report them cleanly. + CON.error(exc) + return 1 except KeyboardInterrupt: CON.warn("interrupted") return 130 diff --git a/components/ota/python/espp_ota/client.py b/components/ota/python/espp_ota/client.py index 2c52b184de..0219dea2d3 100644 --- a/components/ota/python/espp_ota/client.py +++ b/components/ota/python/espp_ota/client.py @@ -92,13 +92,19 @@ def flash(self, image: bytes, image_size: Optional[int] = None) -> None: raise OtaError("empty image") size = len(image) if image_size is None else image_size - # The device keeps its OTA session across a host disconnect, so a failed - # or interrupted flash would leave it "busy" and reject a retry's BEGIN. - # On any failure (device ERROR, timeout, Ctrl-C, transport error) send a - # best-effort ABORT to release the session before propagating. + # The device keeps its OTA session across a host disconnect, so a + # previously interrupted flash can leave it "busy" and reject this run's + # BEGIN. If BEGIN fails, send an ABORT to release any stale session and + # retry BEGIN once before giving up. try: self._transact(_p.make_begin(size), self._begin_to) + except OtaError: + self.abort() # clear a stale session left by a prior interrupted run + self._transact(_p.make_begin(size), self._begin_to) + # From here on, on any failure (device ERROR, timeout, Ctrl-C, transport + # error) send a best-effort ABORT to release the session before propagating. + try: total = len(image) sent = 0 for off in range(0, total, self._chunk): diff --git a/components/ota/python/espp_ota/ui.py b/components/ota/python/espp_ota/ui.py index 36fa02ca9f..e6513bc116 100644 --- a/components/ota/python/espp_ota/ui.py +++ b/components/ota/python/espp_ota/ui.py @@ -1,19 +1,15 @@ """Terminal UI: a nice progress bar + colorized messages, with graceful fallback. -Two rendering paths so it looks good both standalone and under ``idf.py``: - -* **Standalone, real terminal** — if `rich` is available (it ships in the - ESP-IDF Python environment, and `pip install "espp[usb]"` pulls it in) we draw - a `rich` progress bar (spinner, bar, %, bytes, transfer speed, ETA) and print - colorized status/error lines. -* **Captured (e.g. under `idf.py ota-usb`)** — idf.py reads the target's output - line-by-line and re-renders any line ending in ``(NN %)`` *in place* (the same - mechanism that makes esptool's progress animate under `idf.py flash`). So there - we emit throttled ``… (NN %)`` lines, which idf.py turns into a live in-place - bar. `rich`'s own live display can't animate through that line capture, so it's - intentionally only used on a real TTY. - -Everything degrades to plain text; `rich` is optional. +The progress bar is drawn on the controlling terminal, so it animates in place +both standalone and under ``idf.py ota-usb`` (where the tool's stdout/stderr are +captured pipes — see ``_open_progress_stream``, which opens ``/dev/tty`` / +``CONOUT$`` to bypass the capture). If `rich` is available it draws a rich bar +(spinner, bar, %, bytes, transfer speed, ETA) and colorizes status/error lines; +otherwise it falls back to a manual ``\r`` bar, and to periodic plain-text lines +when there is no terminal at all (CI / redirected output). + +`rich` is optional. It ships in the ESP-IDF Python environment (so +``idf.py ota-usb`` already has it) and is pulled in by ``pip install "espp[usb-ui]"``. """ from __future__ import annotations diff --git a/components/ota/python/tests/test_ota_host.py b/components/ota/python/tests/test_ota_host.py index 3be2a99187..eb3565905e 100644 --- a/components/ota/python/tests/test_ota_host.py +++ b/components/ota/python/tests/test_ota_host.py @@ -50,6 +50,11 @@ def _handle(self, fr): return t = fr.type if t == MessageType.BEGIN: + # Emulate a stale session left by a prior interrupted flash: reject the + # first BEGIN as busy until an ABORT clears it. + if getattr(self, "_busy", False): + self._reply(P._build(MessageType.ERROR, struct.pack(" Date: Fri, 11 Sep 2026 08:07:21 -0500 Subject: [PATCH 10/11] feat(ota): host-driven mark-valid / rollback + review fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rollback confirmation is host-driven: a freshly flashed image boots pending-verify and must be confirmed, but the running app must not confirm itself (a broken build could mark itself valid before failing). Add a MARK_VALID path so the host (CLI + web console) confirms it after verifying the device. Protocol (ota_stream_protocol.hpp): new messages GET_STATUS (0x08) -> STATUS (0x0B, u8 flags: pending-verify / rollback-supported), MARK_VALID (0x09), MARK_INVALID (0x0A, roll back + reboot). Handled in ota_example.cpp (ota.mark_app_valid / mark_app_invalid_and_rollback / is_pending_verify). Host tool (espp_ota): `status`, `mark-valid`, `rollback` CLI commands and OtaClient.get_status()/mark_valid()/mark_invalid(); _transact() takes the expected reply type; mark_invalid tolerates the reboot (no reply). Mock-device test covers the round trip. Web console (ota_console.html): "Check status", "Mark valid", "Roll back" buttons with a note that the app must not confirm itself. Also address the latest review: - client.flash(): recover the stale-session case ONLY on a device_or_resource_busy (EBUSY) BEGIN reply, not on a timeout — retrying an uncorrelated timeout could pair a delayed reply with the wrong request and desync the stream. - ui._open_progress_stream(): try only the platform's terminal device (/dev/tty on POSIX, CONOUT$ on Windows) so a headless POSIX run no longer creates a stray "CONOUT$" file instead of falling back to stderr. Device example builds clean on ESP-IDF v6.1; host tests pass; web JS parses. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01DFjjnq3XCTRAXENSxsCxJU --- components/ota/README.md | 16 +++-- components/ota/example/main/ota_example.cpp | 27 ++++++++ .../include/detail/ota_stream_protocol.hpp | 57 ++++++++++++++-- components/ota/python/README.md | 12 ++++ components/ota/python/espp_ota/cli.py | 40 +++++++++++ components/ota/python/espp_ota/client.py | 56 ++++++++++++--- components/ota/python/espp_ota/protocol.py | 39 ++++++++++- components/ota/python/espp_ota/ui.py | 14 ++-- components/ota/python/tests/test_ota_host.py | 27 ++++++++ components/ota/web/ota_console.html | 68 +++++++++++++++++-- 10 files changed, 324 insertions(+), 32 deletions(-) diff --git a/components/ota/README.md b/components/ota/README.md index 85826ee03e..b1eee96057 100644 --- a/components/ota/README.md +++ b/components/ota/README.md @@ -94,11 +94,19 @@ is the routing id (OTA is **module 0**). OTA layers its message types on it. rejects and resynchronizes past oversized or corrupt frames, so buffering stays bounded - host → device (requests): `0x01 BEGIN(u32 image_size)`, `0x02 DATA(bytes)`, - `0x03 FINISH`, `0x04 ABORT`; device → host (replies, reply flag set): + `0x03 FINISH`, `0x04 ABORT`, `0x08 GET_STATUS`, `0x09 MARK_VALID`, + `0x0A MARK_INVALID`; device → host (replies, reply flag set): `0x05 OK(u32 bytes_received)`, `0x06 ERROR(u32 code + utf8 message)`, - `0x07 PROGRESS(u32 written, u32 total)` -- transactions are serialized: the host waits for `OK` / `ERROR` before the - next frame + `0x07 PROGRESS(u32 written, u32 total)`, `0x0B STATUS(u8 flags)` (bit0 = + pending-verify, bit1 = rollback-supported) +- transactions are serialized: the host waits for `OK` / `ERROR` (or `STATUS`) + before the next frame +- **rollback is host-driven** (see below): after an OTA the new image boots + *pending verify*, and the **host** confirms it with `MARK_VALID` once it has + checked the device is healthy — the running app must not confirm itself, or a + broken build could mark itself valid before failing. `MARK_INVALID` rolls back + to the previous image and reboots; `GET_STATUS` reports whether the running + image is still pending verify. The [espp OTA Console](https://esp-cpp.github.io/espp/apps/ota_console.html) (`web/ota_console.html`) implements this protocol over WebUSB in the browser. diff --git a/components/ota/example/main/ota_example.cpp b/components/ota/example/main/ota_example.cpp index fea3bae584..49f22f6bb5 100644 --- a/components/ota/example/main/ota_example.cpp +++ b/components/ota/example/main/ota_example.cpp @@ -348,6 +348,33 @@ extern "C" void app_main(void) { reply_error(ec, "abort failed"); break; } + case proto::MessageType::GetStatus: { + // Report rollback status so the host can decide whether to confirm the + // running image. Session-independent (does not require BEGIN). + uint8_t flags = 0; +#if defined(CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE) + flags |= proto::kStatusRollbackSupported; + if (ota.is_pending_verify()) + flags |= proto::kStatusPendingVerify; +#endif + usb.write_vendor(proto::make_status(flags)); + break; + } + case proto::MessageType::MarkValid: + // The HOST confirms the running image after its own health checks — the app + // must not confirm itself. Cancels the pending rollback. + if (ota.mark_app_valid(ec)) + usb.write_vendor(proto::make_ok(0)); + else + reply_error(ec, "mark valid failed"); + break; + case proto::MessageType::MarkInvalid: + // Reject the running image: roll back to the previous app and reboot. This + // does not return on success (the device reboots), so reply first. + usb.write_vendor(proto::make_ok(0)); + if (!ota.mark_app_invalid_and_rollback(ec)) + reply_error(ec, "rollback failed"); // only reached if rollback failed + break; default: reply_error(std::make_error_code(std::errc::not_supported), "unknown message type"); break; diff --git a/components/ota/include/detail/ota_stream_protocol.hpp b/components/ota/include/detail/ota_stream_protocol.hpp index 6cbfa505ab..0059d4a007 100644 --- a/components/ota/include/detail/ota_stream_protocol.hpp +++ b/components/ota/include/detail/ota_stream_protocol.hpp @@ -25,15 +25,24 @@ // message. // // Message types & payloads (host -> device, requests): -// 0x01 BEGIN — payload: u32 image_size (0 = unknown / streaming). -// 0x02 DATA — payload: raw image bytes (1..kMaxPayloadSize per frame). -// 0x03 FINISH — no payload. Validates + activates the received image. -// 0x04 ABORT — no payload. Discards the in-progress session. +// 0x01 BEGIN — payload: u32 image_size (0 = unknown / streaming). +// 0x02 DATA — payload: raw image bytes (1..kMaxPayloadSize per frame). +// 0x03 FINISH — no payload. Validates + activates the received image. +// 0x04 ABORT — no payload. Discards the in-progress session. +// 0x08 GET_STATUS — no payload. Asks for a STATUS reply (rollback state). +// 0x09 MARK_VALID — no payload. Confirms the running image (cancel rollback). +// 0x0A MARK_INVALID— no payload. Rolls back to the previous image + reboots. // // Message types & payloads (device -> host, replies; reply flag set): // 0x05 OK — payload: u32 bytes_received so far. // 0x06 ERROR — payload: u32 code followed by a UTF-8 message. // 0x07 PROGRESS — payload: u32 written, u32 total (0 if unknown). Optional. +// 0x0B STATUS — payload: u8 flags (bit0 pending_verify, bit1 rollback_supported). +// +// Rollback (bootloader rollback support): a freshly flashed image boots "pending +// verify" and rolls back on the next reset unless confirmed. The device app must +// NOT confirm itself; the HOST confirms it (MARK_VALID) after its own health +// checks — a broken build could otherwise mark itself valid before failing. // // Flow control: the host serializes transactions — it sends one frame and waits // for the matching OK / ERROR reply before sending the next — so the device @@ -86,11 +95,26 @@ enum class MessageType : uint8_t { Ok = 0x05, ///< device -> host: success reply (payload: u32 bytes_received so far) Error = 0x06, ///< device -> host: failure reply (payload: u32 code + utf8 message) Progress = 0x07, ///< device -> host: optional progress (payload: u32 written, u32 total) + // Rollback control (bootloader rollback support). After an OTA the new image + // boots "pending verify" and rolls back on the next reset unless confirmed. + // The device app must NOT confirm itself (a broken app could still do so before + // failing); the HOST confirms it once it has verified the device is healthy. + GetStatus = 0x08, ///< host -> device: query rollback status (no payload) -> Status reply + MarkValid = 0x09, ///< host -> device: confirm the running image (cancel rollback), no payload + MarkInvalid = 0x0A, ///< host -> device: reject the running image (roll back + reboot), no payload + Status = 0x0B, ///< device -> host reply: u8 flags (bit0 pending_verify, bit1 rollback_supported) +}; + +/// Status-reply flag bits (MessageType::Status payload byte 0). +enum StatusFlags : uint8_t { + kStatusPendingVerify = 0x01, ///< running image awaits confirmation (will roll back if not) + kStatusRollbackSupported = 0x02, ///< bootloader rollback support is compiled in }; /// Whether a message type is a device->host reply (sets the frame reply flag). inline bool is_reply(MessageType type) { - return type == MessageType::Ok || type == MessageType::Error || type == MessageType::Progress; + return type == MessageType::Ok || type == MessageType::Error || type == MessageType::Progress || + type == MessageType::Status; } /// @brief Build an encoded OTA frame (typed overload of stream_frame::build_frame). @@ -120,6 +144,21 @@ inline std::vector make_finish() { return build_frame(MessageType::Fini /// Build an ABORT frame (no payload). inline std::vector make_abort() { return build_frame(MessageType::Abort); } +/// Build a GET_STATUS frame (no payload). The device replies with STATUS. +inline std::vector make_get_status() { return build_frame(MessageType::GetStatus); } + +/// Build a MARK_VALID frame (no payload). Confirms the running image. +inline std::vector make_mark_valid() { return build_frame(MessageType::MarkValid); } + +/// Build a MARK_INVALID frame (no payload). Rolls back + reboots the device. +inline std::vector make_mark_invalid() { return build_frame(MessageType::MarkInvalid); } + +/// Build a STATUS reply (flags: OR of StatusFlags). +inline std::vector make_status(uint8_t flags) { + const uint8_t p[] = {flags}; + return build_frame(MessageType::Status, p); +} + /// Build an OK reply (bytes_received so far). inline std::vector make_ok(uint32_t bytes_received) { std::vector payload; @@ -187,6 +226,14 @@ inline std::optional parse_progress(const Frame &frame) { return info; } +/// Parse a STATUS frame payload; returns the flags byte, or std::nullopt if the +/// payload is empty. +inline std::optional parse_status(const Frame &frame) { + if (frame.payload.empty()) + return std::nullopt; + return frame.payload[0]; +} + } // namespace ota_stream } // namespace detail } // namespace espp diff --git a/components/ota/python/README.md b/components/ota/python/README.md index 444a85478b..8da4fae81a 100644 --- a/components/ota/python/README.md +++ b/components/ota/python/README.md @@ -38,11 +38,23 @@ python -m espp_ota flash build/my_app.bin # BEGIN -> stream -> FINISH python -m espp_ota flash build/my_app.bin --pid 0x1234 --chunk-size 2048 python -m espp_ota list # list matching USB devices python -m espp_ota discover # probe the device's dispatcher +python -m espp_ota status # is the running image pending verify? +python -m espp_ota mark-valid # confirm the running image (cancel rollback) +python -m espp_ota rollback # reject it: roll back + reboot ``` Installed with the espp wheel it's also available as the `espp-ota` command (`pip install "espp[usb]"`, or `"espp[usb-ui]"` to also get the `rich` UI). +### Rollback: confirm the image from the host + +With bootloader rollback enabled, a freshly flashed image boots **pending verify** +and rolls back on the next reset unless it is confirmed. Confirmation is +deliberately **host-driven** — the running app must not mark *itself* valid, since +a broken build could do so right before crashing. So the flow is: `flash` → the +device reboots into the new image → you verify it works → `mark-valid` (or +`rollback` to reject it). `status` reports whether confirmation is still pending. + ## Library use ```python diff --git a/components/ota/python/espp_ota/cli.py b/components/ota/python/espp_ota/cli.py index 037c58267c..f0f4495111 100644 --- a/components/ota/python/espp_ota/cli.py +++ b/components/ota/python/espp_ota/cli.py @@ -116,6 +116,33 @@ def _cmd_discover(args) -> int: return 0 +def _cmd_status(args) -> int: + with _make_transport(args) as t: + st = OtaClient(t).get_status() + if not st.rollback_supported: + CON.info("rollback: not supported (CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE off)") + elif st.pending_verify: + CON.warn("running image is PENDING VERIFY — confirm it with `mark-valid` " + "(or it rolls back on the next reset)") + else: + CON.success("running image is confirmed (not pending verify)") + return 0 + + +def _cmd_mark_valid(args) -> int: + with _make_transport(args) as t: + OtaClient(t).mark_valid() + CON.success("running image marked valid; rollback cancelled") + return 0 + + +def _cmd_rollback(args) -> int: + with _make_transport(args) as t: + OtaClient(t).mark_invalid() + CON.success("device rolling back to the previous image and rebooting") + return 0 + + def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(prog="espp_ota", description=__doc__.split("\n")[0]) p.add_argument("--version", action="version", version=f"espp_ota {__version__}") @@ -142,6 +169,19 @@ def build_parser() -> argparse.ArgumentParser: _add_device_args(d) d.add_argument("--timeout", type=int, default=2000, help="ms (default 2000)") d.set_defaults(func=_cmd_discover) + + st = sub.add_parser("status", help="query rollback status (is the image pending verify?)") + _add_device_args(st) + st.set_defaults(func=_cmd_status) + + mv = sub.add_parser("mark-valid", + help="confirm the running image (cancel rollback) after verifying it") + _add_device_args(mv) + mv.set_defaults(func=_cmd_mark_valid) + + rb = sub.add_parser("rollback", help="reject the running image: roll back + reboot") + _add_device_args(rb) + rb.set_defaults(func=_cmd_rollback) return p diff --git a/components/ota/python/espp_ota/client.py b/components/ota/python/espp_ota/client.py index 0219dea2d3..b6ee534cde 100644 --- a/components/ota/python/espp_ota/client.py +++ b/components/ota/python/espp_ota/client.py @@ -9,6 +9,7 @@ from __future__ import annotations +import errno import time from collections import deque from typing import Callable, Deque, List, Optional @@ -53,19 +54,22 @@ def _next_frame(self, deadline: float) -> _f.Frame: if data: self._pending.extend(self._parser.feed(data)) - def _transact(self, request: bytes, timeout_ms: int) -> _f.Frame: - """Send one request and return the matching OK reply (module 0). + def _transact(self, request: bytes, timeout_ms: int, + want: MessageType = MessageType.OK) -> _f.Frame: + """Send one request and return the matching reply (module 0). - PROGRESS frames are surfaced to the callback and skipped; an ERROR reply - raises :class:`OtaError`; frames for other modules are ignored.""" + ``want`` is the success reply type expected (OK by default; STATUS for a + status query). PROGRESS frames are surfaced to the callback and skipped; + an ERROR reply raises :class:`OtaError`; frames for other modules are + ignored.""" self._t.write(request, timeout_ms=timeout_ms) deadline = time.monotonic() + timeout_ms / 1000.0 while True: fr = self._next_frame(deadline) # Only device->host replies on module 0 are ours. Skip other modules' # traffic and any device-originated *request* on module 0 (reply flag - # clear) so a request whose type collides with OK/ERROR/PROGRESS can - # never be mistaken for a reply (as the browser probe also enforces). + # clear) so a request whose type collides with a reply type can never + # be mistaken for a reply (as the browser probe also enforces). if fr.module != _p.MODULE or not fr.is_reply: continue if fr.type == MessageType.PROGRESS: @@ -78,7 +82,7 @@ def _transact(self, request: bytes, timeout_ms: int) -> _f.Frame: if info: raise OtaError(f"device error: {info.message}", info.code) raise OtaError("device error (unparseable ERROR reply)") - if fr.type == MessageType.OK: + if fr.type == want: return fr raise OtaError(f"unexpected reply type 0x{fr.type:02x}") @@ -94,11 +98,16 @@ def flash(self, image: bytes, image_size: Optional[int] = None) -> None: # The device keeps its OTA session across a host disconnect, so a # previously interrupted flash can leave it "busy" and reject this run's - # BEGIN. If BEGIN fails, send an ABORT to release any stale session and - # retry BEGIN once before giving up. + # BEGIN. Recover from THAT case only: if BEGIN is rejected specifically + # with device_or_resource_busy (EBUSY), send an ABORT to release the stale + # session and retry BEGIN once. Any other failure (a timeout, a transport + # error) is NOT retried — retrying on the same uncorrelated stream could + # pair a delayed reply with the wrong request and desync the protocol. try: self._transact(_p.make_begin(size), self._begin_to) - except OtaError: + except OtaError as exc: + if exc.code != errno.EBUSY: + raise self.abort() # clear a stale session left by a prior interrupted run self._transact(_p.make_begin(size), self._begin_to) @@ -129,6 +138,33 @@ def abort(self) -> None: except Exception: pass # best-effort cleanup; the link may already be gone + # -- rollback control ----------------------------------------------------- + def get_status(self) -> "_p.StatusInfo": + """Query the device's rollback status (a STATUS reply).""" + fr = self._transact(_p.make_get_status(), self._data_to, want=MessageType.STATUS) + info = _p.parse_status(fr) + if info is None: + raise OtaError("unparseable STATUS reply") + return info + + def mark_valid(self) -> None: + """Confirm the running image (cancel the pending rollback). The host does + this after verifying the device is healthy — the app must not confirm + itself.""" + self._transact(_p.make_mark_valid(), self._data_to) + + def mark_invalid(self) -> None: + """Reject the running image: the device rolls back to the previous app and + reboots. The device may reboot before/without replying, so a missing reply + is treated as success.""" + try: + self._transact(_p.make_mark_invalid(), self._data_to) + except OtaError as exc: + # A timeout (no code) is expected — the device rebooted. Re-raise a + # real device ERROR (rollback refused, e.g. no previous app). + if exc.code is not None: + raise + def discover(self, timeout_ms: int = 2000) -> List[_f.Frame]: """Send a dispatcher ListModules request; return the matching reply. diff --git a/components/ota/python/espp_ota/protocol.py b/components/ota/python/espp_ota/protocol.py index 25236cce2d..f5eb8fc91c 100644 --- a/components/ota/python/espp_ota/protocol.py +++ b/components/ota/python/espp_ota/protocol.py @@ -34,9 +34,18 @@ class MessageType(IntEnum): OK = 0x05 # device->host: u32 bytes_received so far ERROR = 0x06 # device->host: u32 code + utf-8 message PROGRESS = 0x07 # device->host: u32 written, u32 total (0 if unknown) + GET_STATUS = 0x08 # host->device: query rollback status (no payload) -> STATUS + MARK_VALID = 0x09 # host->device: confirm the running image (cancel rollback) + MARK_INVALID = 0x0A # host->device: reject the running image (roll back + reboot) + STATUS = 0x0B # device->host: u8 flags (see StatusFlags) -_REPLY_TYPES = {MessageType.OK, MessageType.ERROR, MessageType.PROGRESS} +class StatusFlags(IntEnum): + PENDING_VERIFY = 0x01 # running image awaits confirmation (rolls back if not) + ROLLBACK_SUPPORTED = 0x02 # bootloader rollback support is compiled in + + +_REPLY_TYPES = {MessageType.OK, MessageType.ERROR, MessageType.PROGRESS, MessageType.STATUS} def _build(type_: MessageType, payload: bytes = b"") -> bytes: @@ -60,6 +69,18 @@ def make_abort() -> bytes: return _build(MessageType.ABORT) +def make_get_status() -> bytes: + return _build(MessageType.GET_STATUS) + + +def make_mark_valid() -> bytes: + return _build(MessageType.MARK_VALID) + + +def make_mark_invalid() -> bytes: + return _build(MessageType.MARK_INVALID) + + def make_discovery_request() -> bytes: """A dispatcher discovery (ListModules) request on module 0xFF.""" return _f.build_frame(DISCOVERY_MODULE, DISCOVERY_LIST_MODULES, b"", reply=False) @@ -100,6 +121,22 @@ def parse_progress(fr: _f.Frame) -> Optional[ProgressInfo]: return ProgressInfo(written, total) +@dataclass +class StatusInfo: + pending_verify: bool # running image awaits confirmation (rolls back if not) + rollback_supported: bool # bootloader rollback support is compiled in + + +def parse_status(fr: _f.Frame) -> Optional[StatusInfo]: + if not fr.payload: + return None + flags = fr.payload[0] + return StatusInfo( + pending_verify=bool(flags & StatusFlags.PENDING_VERIFY), + rollback_supported=bool(flags & StatusFlags.ROLLBACK_SUPPORTED), + ) + + class OtaError(RuntimeError): """An ERROR reply, a protocol violation, or a transport failure.""" diff --git a/components/ota/python/espp_ota/ui.py b/components/ota/python/espp_ota/ui.py index e6513bc116..d8d5681019 100644 --- a/components/ota/python/espp_ota/ui.py +++ b/components/ota/python/espp_ota/ui.py @@ -41,12 +41,14 @@ def _open_progress_stream(): return sys.stderr, False except Exception: pass # stderr may not support isatty() (e.g. a wrapped stream); fall through - for name in ("/dev/tty", "CONOUT$"): - try: - return open(name, "w"), True - except Exception: - continue - return None, False + # Try only the terminal device for THIS platform. Using the wrong name (e.g. + # "CONOUT$" on POSIX) would create a stray regular file in the cwd and write + # progress there instead of falling back to stderr. + term_name = "CONOUT$" if os.name == "nt" else "/dev/tty" + try: + return open(term_name, "w"), True + except Exception: + return None, False # no controlling terminal (CI / redirected) -> stderr fallback def _have_rich() -> bool: diff --git a/components/ota/python/tests/test_ota_host.py b/components/ota/python/tests/test_ota_host.py index eb3565905e..2174705f91 100644 --- a/components/ota/python/tests/test_ota_host.py +++ b/components/ota/python/tests/test_ota_host.py @@ -74,6 +74,18 @@ def _handle(self, fr): elif t == MessageType.ABORT: self._busy = False # ABORT clears a stale session self._reply(P._build(MessageType.OK, struct.pack(" device requests (flags reply bit 0): 0x01 BEGIN(u32 image_size), - 0x02 DATA(bytes), 0x03 FINISH, 0x04 ABORT; device -> host replies - (flags reply bit 1): 0x05 OK(u32 bytes_received), - 0x06 ERROR(u32 code + utf8 message), 0x07 PROGRESS(u32 written, u32 total). + 0x02 DATA(bytes), 0x03 FINISH, 0x04 ABORT, 0x08 GET_STATUS, 0x09 MARK_VALID, + 0x0A MARK_INVALID; device -> host replies (flags reply bit 1): + 0x05 OK(u32 bytes_received), 0x06 ERROR(u32 code + utf8 message), + 0x07 PROGRESS(u32 written, u32 total), 0x0B STATUS(u8 flags: bit0 pending + verify, bit1 rollback supported). - transactions are serialized: exactly one command frame is in flight and the host waits for its OK / ERROR reply (PROGRESS frames are informational and may arrive before the reply). @@ -218,6 +220,14 @@

Firmware update

+
+ + + +
+

After an OTA the new image boots + pending verify: confirm it here (the app must not confirm itself) + or it rolls back on the next reset.

0 / 0 bytes @@ -268,8 +278,12 @@

Log

const FLAGS_REQUEST = (FLAGS_VERSION << 4) | 0; // host->device request: 0x10 (reply bit 0) const FLAGS_REPLY_BIT = 0x01; // bit0 set on device->host replies const FLAG_CORRELATION = 0x02; // bit1: optional u16 correlation id present after `type` - const TYPE = { BEGIN: 0x01, DATA: 0x02, FINISH: 0x03, ABORT: 0x04, OK: 0x05, ERROR: 0x06, PROGRESS: 0x07 }; - const TYPE_NAME = { 0x01: "BEGIN", 0x02: "DATA", 0x03: "FINISH", 0x04: "ABORT", 0x05: "OK", 0x06: "ERROR", 0x07: "PROGRESS" }; + const TYPE = { BEGIN: 0x01, DATA: 0x02, FINISH: 0x03, ABORT: 0x04, OK: 0x05, ERROR: 0x06, PROGRESS: 0x07, + GET_STATUS: 0x08, MARK_VALID: 0x09, MARK_INVALID: 0x0A, STATUS: 0x0B }; + const TYPE_NAME = { 0x01: "BEGIN", 0x02: "DATA", 0x03: "FINISH", 0x04: "ABORT", 0x05: "OK", 0x06: "ERROR", 0x07: "PROGRESS", + 0x08: "GET_STATUS", 0x09: "MARK_VALID", 0x0A: "MARK_INVALID", 0x0B: "STATUS" }; + // STATUS reply flag bits. + const STATUS_PENDING_VERIFY = 0x01, STATUS_ROLLBACK_SUPPORTED = 0x02; // esp_ota_begin erases flash (seconds for a large / unknown-size image) and // esp_ota_end re-reads + hashes the whole image, so give BEGIN / FINISH a // much longer deadline than the per-4KiB DATA writes. @@ -282,7 +296,8 @@

Log

// =================================================================== const els = {}; for (const id of ["status", "connectBtn", "anyDevice", "devInfo", "fileInput", "fileInfo", - "uploadBtn", "abortBtn", "progressFill", "statBytes", "statPercent", + "uploadBtn", "abortBtn", "statusBtn", "markValidBtn", "rollbackBtn", + "progressFill", "statBytes", "statPercent", "statRate", "statNote", "logFrames", "clearLogBtn", "log", "unsupported"]) { els[id] = document.getElementById(id); } @@ -571,6 +586,11 @@

Log

const file = els.fileInput.files && els.fileInput.files[0]; els.uploadBtn.disabled = !(device && file && !uploading); els.abortBtn.disabled = !uploading; + // Rollback controls need a connection and no upload in flight. + const rbReady = !!device && !uploading; + els.statusBtn.disabled = !rbReady; + els.markValidBtn.disabled = !rbReady; + els.rollbackBtn.disabled = !rbReady; if (!device) els.devInfo.textContent = "Not connected. Default filter: any espp device (VID 0x1209); the vendor (0xFF) interface is discovered from the descriptors at runtime."; } @@ -674,6 +694,8 @@

Log

if (progress) updateProgress(progress.written, progress.total || null); continue; // informational; keep waiting for OK / ERROR } + // STATUS is the reply to GET_STATUS; return its raw payload (flags byte). + if (reply.type === TYPE.STATUS) return reply.payload; if (reply.type === TYPE.OK) return parseU32(reply.payload); if (reply.type === TYPE.ERROR) { const info = parseError(reply.payload); @@ -864,6 +886,40 @@

Log

logLine("sys", "Abort requested; stopping after the in-flight chunk..."); }); + // ---- rollback controls (host confirms the image; the app must not) -------- + els.statusBtn.addEventListener("click", async () => { + try { + const payload = await transact(TYPE.GET_STATUS, null, DATA_TIMEOUT_MS); + const flags = (payload && payload.length) ? payload[0] : 0; + if (!(flags & STATUS_ROLLBACK_SUPPORTED)) { + logLine("sys", "Rollback not supported on the device (CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE off)."); + } else if (flags & STATUS_PENDING_VERIFY) { + logLine("err", "Running image is PENDING VERIFY — click “Mark valid” to confirm it, or it rolls back on the next reset."); + } else { + logLine("ok", "Running image is confirmed (not pending verify)."); + } + } catch (e) { logLine("err", "Status failed: " + e.message); } + }); + + els.markValidBtn.addEventListener("click", async () => { + try { + await transact(TYPE.MARK_VALID, null, DATA_TIMEOUT_MS); + logLine("ok", "Running image marked valid; rollback cancelled."); + } catch (e) { logLine("err", "Mark valid failed: " + e.message); } + }); + + els.rollbackBtn.addEventListener("click", async () => { + if (!confirm("Roll back to the previous image and reboot the device?")) return; + // The device reboots on success and may not reply, so a deadline/link loss + // here is the expected outcome, not an error. + try { + await transact(TYPE.MARK_INVALID, null, DATA_TIMEOUT_MS); + logLine("ok", "Rollback acknowledged; device rebooting into the previous image."); + } catch (e) { + logLine("sys", "Rollback sent; device is rebooting (no reply expected). [" + e.message + "]"); + } + }); + logLine("sys", "espp OTA Console ready. Connect a device running the espp ota example (vendor/WebUSB interface)."); From c636ffd531c0eb73c699f3afa16629f2e9f6826c Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Fri, 11 Sep 2026 08:46:58 -0500 Subject: [PATCH 11/11] feat(ota): auto-verify after flash + report running firmware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The host now confirms a freshly flashed image only once it has proven the image booted and can respond — the running app must never mark itself valid. - STATUS reply now also carries the running app's version + project name (each a u8-length-prefixed string), from ota.running_app_description(). Protocol + ota_example + C++ StatusInfo parser + Python StatusInfo + web parseStatus all decode it; older flags-only replies still parse. - Python `flash` auto-verifies by default: records the firmware it started with, flashes, waits for the reboot, reconnects (retries until the device re-enumerates, --verify-timeout), reads status, and — if the new image is responding AND still pending-verify — marks it valid. Prints the before -> after firmware. `--no-verify` skips it. `status`/`mark-valid` now print the firmware. - Web console arms a post-reboot check: after FINISH it records the previous firmware and, because the reboot drops WebUSB, prompts the user to reconnect; on reconnect it queries status and, if pending-verify, shows a confirm() dialog with the previous -> now-running firmware asking to mark it valid. Device example builds clean on ESP-IDF v6.1; Python + C++ host tests pass; web JS parses. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01DFjjnq3XCTRAXENSxsCxJU --- components/ota/README.md | 5 +- components/ota/example/main/ota_example.cpp | 7 +- .../include/detail/ota_stream_protocol.hpp | 55 +++++++++-- components/ota/python/README.md | 22 ++++- components/ota/python/espp_ota/cli.py | 84 ++++++++++++++++- components/ota/python/espp_ota/protocol.py | 28 +++++- components/ota/python/tests/test_ota_host.py | 6 +- components/ota/web/ota_console.html | 94 ++++++++++++++++--- 8 files changed, 266 insertions(+), 35 deletions(-) diff --git a/components/ota/README.md b/components/ota/README.md index b1eee96057..52af2e6553 100644 --- a/components/ota/README.md +++ b/components/ota/README.md @@ -97,8 +97,9 @@ is the routing id (OTA is **module 0**). OTA layers its message types on it. `0x03 FINISH`, `0x04 ABORT`, `0x08 GET_STATUS`, `0x09 MARK_VALID`, `0x0A MARK_INVALID`; device → host (replies, reply flag set): `0x05 OK(u32 bytes_received)`, `0x06 ERROR(u32 code + utf8 message)`, - `0x07 PROGRESS(u32 written, u32 total)`, `0x0B STATUS(u8 flags)` (bit0 = - pending-verify, bit1 = rollback-supported) + `0x07 PROGRESS(u32 written, u32 total)`, `0x0B STATUS(u8 flags + running app + version + project name)` (flags bit0 = pending-verify, bit1 = rollback-supported; + each string is u8-length-prefixed) - transactions are serialized: the host waits for `OK` / `ERROR` (or `STATUS`) before the next frame - **rollback is host-driven** (see below): after an OTA the new image boots diff --git a/components/ota/example/main/ota_example.cpp b/components/ota/example/main/ota_example.cpp index 49f22f6bb5..922f56fc02 100644 --- a/components/ota/example/main/ota_example.cpp +++ b/components/ota/example/main/ota_example.cpp @@ -349,15 +349,16 @@ extern "C" void app_main(void) { break; } case proto::MessageType::GetStatus: { - // Report rollback status so the host can decide whether to confirm the - // running image. Session-independent (does not require BEGIN). + // Report rollback status + the running firmware (so the host can show what + // is now running before confirming it). Session-independent (no BEGIN). uint8_t flags = 0; #if defined(CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE) flags |= proto::kStatusRollbackSupported; if (ota.is_pending_verify()) flags |= proto::kStatusPendingVerify; #endif - usb.write_vendor(proto::make_status(flags)); + const auto desc = ota.running_app_description(); + usb.write_vendor(proto::make_status(flags, desc.version, desc.project_name)); break; } case proto::MessageType::MarkValid: diff --git a/components/ota/include/detail/ota_stream_protocol.hpp b/components/ota/include/detail/ota_stream_protocol.hpp index 0059d4a007..116b0d5353 100644 --- a/components/ota/include/detail/ota_stream_protocol.hpp +++ b/components/ota/include/detail/ota_stream_protocol.hpp @@ -37,7 +37,9 @@ // 0x05 OK — payload: u32 bytes_received so far. // 0x06 ERROR — payload: u32 code followed by a UTF-8 message. // 0x07 PROGRESS — payload: u32 written, u32 total (0 if unknown). Optional. -// 0x0B STATUS — payload: u8 flags (bit0 pending_verify, bit1 rollback_supported). +// 0x0B STATUS — payload: u8 flags (bit0 pending_verify, bit1 rollback_supported), +// then the running app's version and project name (each a +// u8-length-prefixed UTF-8 string). // // Rollback (bootloader rollback support): a freshly flashed image boots "pending // verify" and rolls back on the next reset unless confirmed. The device app must @@ -153,9 +155,22 @@ inline std::vector make_mark_valid() { return build_frame(MessageType:: /// Build a MARK_INVALID frame (no payload). Rolls back + reboots the device. inline std::vector make_mark_invalid() { return build_frame(MessageType::MarkInvalid); } -/// Build a STATUS reply (flags: OR of StatusFlags). -inline std::vector make_status(uint8_t flags) { - const uint8_t p[] = {flags}; +/// Append a length-prefixed (u8 length) UTF-8 string, truncated to 255 bytes. +inline void put_str(std::vector &out, std::string_view s) { + const uint8_t len = static_cast(std::min(s.size(), 255)); + out.push_back(len); + out.insert(out.end(), s.begin(), s.begin() + len); +} + +/// Build a STATUS reply: flags (OR of StatusFlags) plus the running app's version +/// and project name (each a u8-length-prefixed string), so the host can report +/// what firmware is now running before confirming it. +inline std::vector make_status(uint8_t flags, std::string_view version = {}, + std::string_view project = {}) { + std::vector p; + p.push_back(flags); + put_str(p, version); + put_str(p, project); return build_frame(MessageType::Status, p); } @@ -226,12 +241,36 @@ inline std::optional parse_progress(const Frame &frame) { return info; } -/// Parse a STATUS frame payload; returns the flags byte, or std::nullopt if the -/// payload is empty. -inline std::optional parse_status(const Frame &frame) { +/// Decoded STATUS reply payload. +struct StatusInfo { + uint8_t flags{}; ///< OR of StatusFlags (pending_verify / rollback_supported) + std::string version; ///< Running app version (may be empty) + std::string project_name; ///< Running app project name (may be empty) + + bool pending_verify() const { return (flags & kStatusPendingVerify) != 0; } + bool rollback_supported() const { return (flags & kStatusRollbackSupported) != 0; } +}; + +/// Parse a STATUS frame payload: [flags u8][version u8-len+bytes][project +/// u8-len+bytes]. Trailing strings are optional (older devices sent flags only); +/// returns std::nullopt only if the payload is empty. +inline std::optional parse_status(const Frame &frame) { if (frame.payload.empty()) return std::nullopt; - return frame.payload[0]; + StatusInfo info{}; + info.flags = frame.payload[0]; + size_t i = 1; + auto read_str = [&](std::string &out) { + if (i >= frame.payload.size()) + return; + const size_t len = frame.payload[i++]; + const size_t n = std::min(len, frame.payload.size() - i); + out.assign(frame.payload.begin() + i, frame.payload.begin() + i + n); + i += n; + }; + read_str(info.version); + read_str(info.project_name); + return info; } } // namespace ota_stream diff --git a/components/ota/python/README.md b/components/ota/python/README.md index 8da4fae81a..d2c8c04b9f 100644 --- a/components/ota/python/README.md +++ b/components/ota/python/README.md @@ -51,9 +51,25 @@ Installed with the espp wheel it's also available as the `espp-ota` command With bootloader rollback enabled, a freshly flashed image boots **pending verify** and rolls back on the next reset unless it is confirmed. Confirmation is deliberately **host-driven** — the running app must not mark *itself* valid, since -a broken build could do so right before crashing. So the flow is: `flash` → the -device reboots into the new image → you verify it works → `mark-valid` (or -`rollback` to reject it). `status` reports whether confirmation is still pending. +a broken build could do so right before crashing. + +`flash` **auto-verifies by default**: after streaming the image it waits for the +device to reboot, reconnects (the device re-enumerates with the same VID/PID), +and reads the running firmware + rollback status. If the new image is *responding* +to OTA commands and still *pending verify*, it has booted — so the host marks it +valid. It prints the before → after firmware, e.g.: + +``` +● Currently running: ota_example 1.0.0 + ... flashing ... +● ota_example 1.0.0 → ota_example 1.1.0 +The new image booted and responded, so it has been marked valid (rollback cancelled). +``` + +Pass `--no-verify` to skip that step (then confirm later with `mark-valid`), or +`--verify-timeout ` to change how long it waits for the device to +reappear. The `status`, `mark-valid`, and `rollback` commands remain available to +do it manually; `rollback` rejects the running image (roll back + reboot). ## Library use diff --git a/components/ota/python/espp_ota/cli.py b/components/ota/python/espp_ota/cli.py index f0f4495111..1caf8bcd0f 100644 --- a/components/ota/python/espp_ota/cli.py +++ b/components/ota/python/espp_ota/cli.py @@ -62,6 +62,18 @@ def _make_transport(args) -> UsbVendorTransport: interface=args.interface) +def _reconnect(args, timeout_s: float = 20.0): + """Reopen the device after it reboots (same VID/PID, re-enumerates). Retries + until it appears or the timeout elapses; returns the opened transport or None.""" + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + try: + return _make_transport(args) + except TransportError: + time.sleep(0.5) + return None + + def _cmd_flash(args) -> int: with open(args.binary, "rb") as fh: image = fh.read() @@ -69,9 +81,19 @@ def _cmd_flash(args) -> int: CON.error("image is empty") return 2 size = 0 if args.unknown_size else len(image) + + before = None # firmware running before the flash with _make_transport(args) as t: if not args.quiet: CON.note(f"● Connected to {t.description}") + client = OtaClient(t) + try: + before = client.get_status() + if not args.quiet: + CON.info(f" Currently running: {before.firmware_str()}") + except OtaError: + pass # older device without GET_STATUS; carry on + if not args.quiet: CON.info(f" Flashing {args.binary} ({_human_size(len(image))})") start = time.monotonic() with ui.Progress(len(image), label="Flashing", quiet=args.quiet) as prog: @@ -87,8 +109,51 @@ def _cmd_flash(args) -> int: if not args.quiet: dt = time.monotonic() - start rate = len(image) / dt / 1024 if dt else 0 - CON.success(f"OTA complete in {dt:.1f}s ({rate:.0f} KiB/s). The device " - f"activates the new image and reboots per its own policy.") + CON.success(f"OTA complete in {dt:.1f}s ({rate:.0f} KiB/s). " + f"The device is activating the new image and rebooting.") + + if args.no_verify: + CON.info("Auto-verify disabled. Once you have checked the device, confirm the " + "image with `espp-ota mark-valid` (or it rolls back on the next reset).") + return 0 + return _auto_verify(args, before) + + +def _auto_verify(args, before) -> int: + """Reconnect after the reboot and confirm the running image can respond before + marking it valid. This is the safety check: a pending-verify image that can + answer OTA commands has booted, so the host confirms it.""" + CON.info("Waiting for the device to reboot and reconnect…") + time.sleep(2.0) # give it a moment to drop off the bus before we scan + t = _reconnect(args, timeout_s=args.verify_timeout) + if t is None: + CON.warn("device did not reappear after the reboot. If it booted correctly, " + "confirm the image with `espp-ota mark-valid`; otherwise it will roll " + "back on the next reset.") + return 1 + with t: + client = OtaClient(t) + try: + st = client.get_status() + except OtaError as exc: + CON.error(f"reconnected but the device did not report status: {exc}. " + "Not confirming — it will roll back on the next reset.") + return 1 + # Report the transition. + if before is not None: + CON.note(f"● {before.firmware_str()} → {st.firmware_str()}") + else: + CON.note(f"● Now running: {st.firmware_str()}") + if not st.rollback_supported: + CON.success("Update complete (device has no rollback; nothing to confirm).") + return 0 + if not st.pending_verify: + CON.success("Update complete; the running image is already confirmed.") + return 0 + # It responded to OTA commands AND is pending verify -> it booted; confirm. + client.mark_valid() + CON.success("The new image booted and responded, so it has been marked valid " + "(rollback cancelled).") return 0 @@ -119,6 +184,7 @@ def _cmd_discover(args) -> int: def _cmd_status(args) -> int: with _make_transport(args) as t: st = OtaClient(t).get_status() + CON.note(f"● Running: {st.firmware_str()}") if not st.rollback_supported: CON.info("rollback: not supported (CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE off)") elif st.pending_verify: @@ -131,8 +197,14 @@ def _cmd_status(args) -> int: def _cmd_mark_valid(args) -> int: with _make_transport(args) as t: - OtaClient(t).mark_valid() - CON.success("running image marked valid; rollback cancelled") + client = OtaClient(t) + try: + fw = client.get_status().firmware_str() + except OtaError: + fw = None + client.mark_valid() + CON.success("running image marked valid; rollback cancelled" + + (f" ({fw})" if fw else "")) return 0 @@ -159,6 +231,10 @@ def build_parser() -> argparse.ArgumentParser: f.add_argument("--data-timeout", type=int, default=5000, help="ms (default 5000)") f.add_argument("--finish-timeout", type=int, default=60000, help="ms (default 60000)") f.add_argument("-q", "--quiet", action="store_true", help="suppress progress output") + f.add_argument("--no-verify", action="store_true", + help="don't auto-verify: skip the reconnect + mark-valid after reboot") + f.add_argument("--verify-timeout", type=float, default=20.0, + help="seconds to wait for the device to reappear after reboot (default 20)") f.set_defaults(func=_cmd_flash) lst = sub.add_parser("list", help="list matching USB devices") diff --git a/components/ota/python/espp_ota/protocol.py b/components/ota/python/espp_ota/protocol.py index f5eb8fc91c..17d6d9a763 100644 --- a/components/ota/python/espp_ota/protocol.py +++ b/components/ota/python/espp_ota/protocol.py @@ -123,17 +123,43 @@ def parse_progress(fr: _f.Frame) -> Optional[ProgressInfo]: @dataclass class StatusInfo: - pending_verify: bool # running image awaits confirmation (rolls back if not) + pending_verify: bool # running image awaits confirmation (rolls back if not) rollback_supported: bool # bootloader rollback support is compiled in + version: str = "" # running app version (may be empty) + project_name: str = "" # running app project name (may be empty) + + def firmware_str(self) -> str: + """A short 'project vX.Y' label for the running firmware.""" + if self.project_name and self.version: + return f"{self.project_name} {self.version}" + return self.project_name or self.version or "(unknown)" def parse_status(fr: _f.Frame) -> Optional[StatusInfo]: + # payload: [flags u8][version u8-len+bytes][project u8-len+bytes]; the strings + # are optional (older devices sent flags only). if not fr.payload: return None flags = fr.payload[0] + i = 1 + + def read_str() -> str: + nonlocal i + if i >= len(fr.payload): + return "" + n = fr.payload[i] + i += 1 + s = fr.payload[i:i + n].decode("utf-8", errors="replace") + i += n + return s + + version = read_str() + project = read_str() return StatusInfo( pending_verify=bool(flags & StatusFlags.PENDING_VERIFY), rollback_supported=bool(flags & StatusFlags.ROLLBACK_SUPPORTED), + version=version, + project_name=project, ) diff --git a/components/ota/python/tests/test_ota_host.py b/components/ota/python/tests/test_ota_host.py index 2174705f91..751aaa76df 100644 --- a/components/ota/python/tests/test_ota_host.py +++ b/components/ota/python/tests/test_ota_host.py @@ -78,7 +78,10 @@ def _handle(self, fr): flags = P.StatusFlags.ROLLBACK_SUPPORTED if getattr(self, "_pending", False): flags |= P.StatusFlags.PENDING_VERIFY - self._reply(P._build(MessageType.STATUS, bytes([flags]))) + ver = getattr(self, "_version", "1.0.0").encode() + proj = b"ota_example" + payload = bytes([flags, len(ver)]) + ver + bytes([len(proj)]) + proj + self._reply(P._build(MessageType.STATUS, payload)) elif t == MessageType.MARK_VALID: self.marked_valid = True self._pending = False @@ -165,6 +168,7 @@ def test_rollback_control(): dev._pending = True st = OtaClient(dev).get_status() _ok("status pending+supported", st.pending_verify and st.rollback_supported) + _ok("status reports firmware", st.version == "1.0.0" and st.project_name == "ota_example") OtaClient(dev).mark_valid() _ok("mark_valid confirmed", getattr(dev, "marked_valid", False) and not dev._pending) st2 = OtaClient(dev).get_status() diff --git a/components/ota/web/ota_console.html b/components/ota/web/ota_console.html index 161a32f376..e50ce2aec6 100644 --- a/components/ota/web/ota_console.html +++ b/components/ota/web/ota_console.html @@ -441,6 +441,25 @@

Log

return { written: view.getUint32(0, true), total: view.getUint32(4, true) }; } + // STATUS payload: [flags u8][version u8-len+bytes][project u8-len+bytes]. + function parseStatus(payload) { + if (!payload || payload.length < 1) return null; + const dec = new TextDecoder(); + let i = 1; + const readStr = () => { + if (i >= payload.length) return ""; + const n = payload[i++]; const s = dec.decode(payload.subarray(i, i + n)); i += n; return s; + }; + const version = readStr(), project = readStr(); + return { + flags: payload[0], + pendingVerify: !!(payload[0] & STATUS_PENDING_VERIFY), + rollbackSupported: !!(payload[0] & STATUS_ROLLBACK_SUPPORTED), + version, project, + firmware: (project && version) ? (project + " " + version) : (project || version || "(unknown)"), + }; + } + // =================================================================== // WebUSB connect / disconnect (vendor 0xFF interface discovery) // =================================================================== @@ -449,6 +468,10 @@

Log

let manualDisconnect = false; let uploading = false; let expectRestart = false; + // After a successful upload the device reboots (dropping WebUSB). When the + // user reconnects we auto-check the new firmware and offer to confirm it. + // Holds { before: "" } or null. + let awaitingVerify = null; const parser = new StreamParser(); if (!navigator.usb) { @@ -502,6 +525,32 @@

Log

logLine("sys", "Connected to " + name + ". Vendor iface " + ifaceNumber + ", bulk IN 0x" + epIn.toString(16) + " / OUT 0x" + epOut.toString(16) + "."); checkModulePresent().catch(() => {}); // best-effort: warn if this device lacks the OTA module + maybePromptVerify().catch(() => {}); // if we just flashed, verify + confirm the new image + } + + // After a reboot-triggering upload the user reconnects here; auto-check the + // running image and, if it is pending verify (so it booted and can answer), + // ask the user to confirm it — the firmware must not confirm itself. + async function maybePromptVerify() { + if (!awaitingVerify) return; + const before = awaitingVerify.before; + awaitingVerify = null; + const st = await queryStatus(); + if (!st) return; + if (!st.rollbackSupported) { logLine("ok", "Reconnected; running " + st.firmware + " (no rollback to confirm)."); return; } + if (!st.pendingVerify) { logLine("ok", "Reconnected; running " + st.firmware + " (already confirmed)."); return; } + const msg = "The device rebooted into new firmware and is responding.\n\n" + + (before ? "Previous: " + before + "\n" : "") + + "Now running: " + st.firmware + "\n\n" + + "Mark this image VALID? It rolls back on the next reset if you don't."; + if (confirm(msg)) { + try { + await transact(TYPE.MARK_VALID, null, DATA_TIMEOUT_MS); + logLine("ok", "New image (" + st.firmware + ") marked valid; rollback cancelled."); + } catch (e) { logLine("err", "Mark valid failed: " + e.message); } + } else { + logLine("err", "Left unconfirmed (" + st.firmware + "); it rolls back on the next reset. Use “Mark valid” to confirm later."); + } } async function openAndClaim() { @@ -830,6 +879,14 @@

Log

const startedAt = Date.now(); let sent = 0; + // Record the firmware we're replacing so the post-reboot prompt can show + // the before -> after transition (best-effort; older devices lack STATUS). + let beforeFirmware = ""; + try { + const st0 = parseStatus(await transact(TYPE.GET_STATUS, null, DATA_TIMEOUT_MS)); + if (st0) beforeFirmware = st0.firmware; + } catch (_) {} + try { logLine("sys", "BEGIN: image size " + image.length.toLocaleString() + " bytes (device erases the update partition — this can take a while)..."); await transact(TYPE.BEGIN, u32Payload(image.length), BEGIN_TIMEOUT_MS); @@ -851,9 +908,14 @@

Log

els.progressFill.className = "progress-fill done"; els.progressFill.style.width = "100%"; setStatus("connected", "Update complete"); - setNote("Update complete — device is restarting into the new firmware. " + - "With rollback enabled it must mark itself valid on first boot."); + // Arm the post-reboot verification: when the user reconnects, we check the + // new image can respond and prompt to confirm it (the app must not confirm + // itself). The reboot drops WebUSB, so this needs a manual reconnect. + awaitingVerify = { before: beforeFirmware }; + setNote("Update complete — device is restarting into the new firmware. It " + + "boots pending-verify; reconnect to confirm it (it must not confirm itself)."); logLine("sys", "Update complete (" + sent.toLocaleString() + " bytes). Device restarting — expect a USB disconnect."); + logLine("sys", "Reconnect with the Connect button to verify + confirm the new image."); } catch (e) { els.progressFill.className = "progress-fill failed"; if (e && e.userAbort) { @@ -887,18 +949,24 @@

Log

}); // ---- rollback controls (host confirms the image; the app must not) -------- - els.statusBtn.addEventListener("click", async () => { + // Query STATUS and return the parsed info (logs + returns null on failure). + async function queryStatus() { try { - const payload = await transact(TYPE.GET_STATUS, null, DATA_TIMEOUT_MS); - const flags = (payload && payload.length) ? payload[0] : 0; - if (!(flags & STATUS_ROLLBACK_SUPPORTED)) { - logLine("sys", "Rollback not supported on the device (CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE off)."); - } else if (flags & STATUS_PENDING_VERIFY) { - logLine("err", "Running image is PENDING VERIFY — click “Mark valid” to confirm it, or it rolls back on the next reset."); - } else { - logLine("ok", "Running image is confirmed (not pending verify)."); - } - } catch (e) { logLine("err", "Status failed: " + e.message); } + const st = parseStatus(await transact(TYPE.GET_STATUS, null, DATA_TIMEOUT_MS)); + return st; + } catch (e) { logLine("err", "Status failed: " + e.message); return null; } + } + els.statusBtn.addEventListener("click", async () => { + const st = await queryStatus(); + if (!st) return; + logLine("sys", "Running firmware: " + st.firmware); + if (!st.rollbackSupported) { + logLine("sys", "Rollback not supported on the device (CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE off)."); + } else if (st.pendingVerify) { + logLine("err", "Running image is PENDING VERIFY — click “Mark valid” to confirm it, or it rolls back on the next reset."); + } else { + logLine("ok", "Running image is confirmed (not pending verify)."); + } }); els.markValidBtn.addEventListener("click", async () => {