diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 13fe62190..c876e77d2 100755 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -329,6 +329,9 @@ jobs: target: esp32 - path: 'components/task/example' target: esp32 + - path: 'components/telemetry/example' + target: esp32s3 + command: 'IDF_COMPONENT_MANAGER=0 idf.py build' - path: 'components/thermistor/example' target: esp32 - path: 'components/thread_pool/example' diff --git a/.github/workflows/upload_components.yml b/.github/workflows/upload_components.yml index f55b5491d..99397c2c3 100755 --- a/.github/workflows/upload_components.yml +++ b/.github/workflows/upload_components.yml @@ -157,7 +157,6 @@ jobs: components/runqueue components/rx8130ce components/seeed-studio-round-display - components/serial_plotter components/serialization components/smartpanlee-sc01-plus components/socket @@ -171,6 +170,7 @@ jobs: components/t-dongle-s3 components/tabulate components/task + components/telemetry components/thermistor components/thread_pool components/timer diff --git a/components/serial_plotter/CMakeLists.txt b/components/serial_plotter/CMakeLists.txt deleted file mode 100644 index 1afe3ab49..000000000 --- a/components/serial_plotter/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# serial_plotter currently ships only a self-contained browser webapp -# (web/serial_plotter.html), auto-hosted by the docs "web apps" pipeline and -# usable offline over file://. There is no firmware source yet: a telemetry -# service (headers + example) built on stream_frame / dispatcher is a planned -# follow-up. This registration keeps the directory a valid ESP-IDF component so -# repo tooling and the component registry stay consistent; it gains INCLUDE_DIRS -# and REQUIRES when the telemetry service lands. -idf_component_register() diff --git a/components/serial_plotter/idf_component.yml b/components/serial_plotter/idf_component.yml deleted file mode 100644 index 22e11aef3..000000000 --- a/components/serial_plotter/idf_component.yml +++ /dev/null @@ -1,17 +0,0 @@ -## IDF Component Manager Manifest File -license: "MIT" -description: "Browser Web Serial data plotter (uPlot): auto-parses columnar serial output and plots a high number of points, with CSV save/load and optional 2D/3D modes. A firmware-side telemetry service is a planned follow-up." -url: "https://github.com/esp-cpp/espp/tree/main/components/serial_plotter" -repository: "git://github.com/esp-cpp/espp.git" -maintainers: - - William Emfinger -documentation: "https://esp-cpp.github.io/espp/apps/serial_plotter.html" -tags: - - webapp - - plotting - - serial - - web-serial - - telemetry -# No dependencies: this component currently ships only the browser webapp and -# has no firmware source, so it imposes no ESP-IDF version constraint. The -# planned telemetry service will add the deps (and an idf floor) it needs. diff --git a/components/telemetry/CMakeLists.txt b/components/telemetry/CMakeLists.txt new file mode 100644 index 000000000..eb47e604f --- /dev/null +++ b/components/telemetry/CMakeLists.txt @@ -0,0 +1,11 @@ +# The telemetry component ships: +# - web/telemetry.html: the browser data plotter (Web Serial + WebUSB), +# auto-hosted by the docs "web apps" pipeline and usable offline over file://. +# - include/telemetry.hpp: espp::Telemetry, a binary telemetry emitter +# (dispatcher module 3 over stream_frame) that the web app plots over WebUSB. +# The header only depends on base_component + stream_frame; an app pairs it with +# a Dispatcher and a transport (e.g. usb_device) — see the example. +idf_component_register( + INCLUDE_DIRS "include" + REQUIRES base_component stream_frame +) diff --git a/components/serial_plotter/README.md b/components/telemetry/README.md similarity index 60% rename from components/serial_plotter/README.md rename to components/telemetry/README.md index 592ebd52f..cb456981b 100644 --- a/components/serial_plotter/README.md +++ b/components/telemetry/README.md @@ -1,17 +1,20 @@ -# Serial Plotter +# Telemetry -A self-contained browser tool for reading columnar serial data and plotting it -efficiently — modeled on [esp-cpp/uart_serial_plotter](https://github.com/esp-cpp/uart_serial_plotter), -but running entirely in a Chromium-based browser over the Web Serial API. No -install, no CDN, no network access. +The `telemetry` component pairs a firmware-side `espp::Telemetry` emitter with a +self-contained browser **Serial Plotter** web app that reads data and plots it +efficiently — modeled on +[esp-cpp/uart_serial_plotter](https://github.com/esp-cpp/uart_serial_plotter), +but running entirely in a Chromium-based browser. No install, no CDN, no network +access. Two transports feed the same plot: -- **Hosted:** -- **Offline:** open `web/serial_plotter.html` directly via a `file://` URL. +- **Web Serial (text / CSV)** — auto-parses columnar output (a header line plus + numeric rows) from any device that prints it. +- **WebUSB (binary telemetry)** — an espp device streams typed float channels + via `espp::Telemetry` (see `include/telemetry.hpp` and the + [example](example/)) for higher rate and device-accurate timestamps. -> This component currently ships the webapp only. A firmware-side binary -> **telemetry** transport (a `stream_frame` / `dispatcher` module for -> higher-bandwidth, typed channels) is a planned follow-up; the same webapp will -> gain a WebUSB transport that feeds the same plot. +- **Hosted:** +- **Offline:** open `web/telemetry.html` directly via a `file://` URL. ## Screenshots @@ -47,6 +50,11 @@ rest, with the per-series filter bar: (`Float32Array`) and are drawn with [uPlot](https://github.com/leeoniya/uPlot), which does the pixel decimation. Redraws are coalesced to one per animation frame. The retained-points cap is configurable (default 200k per series). +- **Live zoom (Follow).** While streaming, the view auto-scrolls to the latest + data — optionally to a rolling **Window** of the last _N_ seconds. Drag to zoom + and it drops out of **Follow** so your zoomed view stays put (all retained + samples remain there to pan/zoom through); double-click, or click **Follow**, + to snap back to live. - **Series filter.** A filter bar shows a colored chip per column: click to toggle a series on/off, or type in the name box to plot only the columns / tags that match (composes with the manual toggles), plus **All** / **None**. @@ -60,16 +68,23 @@ rest, with the per-series filter bar: (or any matching CSV) to view it offline with no device connected. - **Serial controls.** Baud selector, pause / resume, clear, and a DTR/RTS device reset. +- **Binary telemetry over WebUSB.** Connect with **USB** to an espp device + running `espp::Telemetry`: the app reads the channel schema and plots the + device-timestamped sample stream (decoded from the `stream_frame` framing, + dispatcher module 3) into the same plot. Requests the schema on connect and + can pause/resume the device stream. ## Requirements -Web Serial is available only in Chromium-based browsers (Chrome, Edge, Opera) and -needs a secure context — it works from `https`, `http://localhost`, or `file://`. -In an unsupported browser the app still loads and can **Load CSV** for viewing. +Web Serial and WebUSB are available only in Chromium-based browsers (Chrome, +Edge, Opera) and need a secure context — they work from `https`, +`http://localhost`, or `file://`. In an unsupported browser the app still loads +and can **Load CSV** for viewing. Native USB telemetry needs an ESP32-S3 (also +S2 / P4) device; see [`example/`](example/). ## Third-party Plotting uses **uPlot** (`web/uPlot.iife.min.js`), MIT-licensed, pinned to v1.6.31 — . It is vendored as a sibling `.js` file (the docs workflow ships `web/*.html` and `web/*.js`); uPlot's small CSS is -inlined into `serial_plotter.html`. Everything else is dependency-free. +inlined into `telemetry.html`. Everything else is dependency-free. diff --git a/components/telemetry/example/CMakeLists.txt b/components/telemetry/example/CMakeLists.txt new file mode 100644 index 000000000..772795025 --- /dev/null +++ b/components/telemetry/example/CMakeLists.txt @@ -0,0 +1,51 @@ +# The following lines of boilerplate have to be in your project's CMakeLists +# in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.20) + +# Build the example (and the espp components it pulls in) as C++20 (the repo +# standard; the telemetry service needs no C++23 feature). Must be set before +# the project.cmake include / project() call so IDF picks it up. +set(CMAKE_CXX_STANDARD 20) + +# NOTE: the IDF component manager is intentionally left ENABLED here (unlike most +# espp examples) so that it can fetch the managed `espressif/esp_tinyusb` +# dependency declared by the usb_device component's idf_component.yml. To avoid +# the component manager scanning every espp component manifest (some board +# components declare target-specific constraints that would fail on esp32s3), +# EXTRA_COMPONENT_DIRS is narrowed to just the components this example uses; the +# in-repo espp components there satisfy the `espp/*` dependencies locally. +include($ENV{IDF_PATH}/tools/cmake/project.cmake) + +# add only the component directories that we want to use +set(EXTRA_COMPONENT_DIRS + "../../../components/base_component" + "../../../components/dispatcher" + "../../../components/format" + "../../../components/logger" + "../../../components/telemetry" + "../../../components/stream_frame" + "../../../components/task" + "../../../components/timer" + "../../../components/usb_device" +) + +# With the component manager disabled (IDF_COMPONENT_MANAGER=0, e.g. in CI so the +# build does not need the as-yet unpublished espp/* components in the registry), +# esp_tinyusb/tinyusb are not fetched; add the vendored submodule copies under +# external/ to the search path. esp_tinyusb's CMakeLists adds `tinyusb` to its +# REQUIRES when the manager is off, so both directories must be discoverable. +if(DEFINED ENV{IDF_COMPONENT_MANAGER} AND "$ENV{IDF_COMPONENT_MANAGER}" STREQUAL "0") + list(APPEND EXTRA_COMPONENT_DIRS + "../../../external/esp-usb/device/esp_tinyusb" + "../../../external/tinyusb" + ) +endif() + +set( + COMPONENTS + "main esptool_py base_component dispatcher format logger telemetry stream_frame task timer usb_device esp_tinyusb" + CACHE STRING + "List of components to include" + ) + +project(telemetry_example) diff --git a/components/telemetry/example/README.md b/components/telemetry/example/README.md new file mode 100644 index 000000000..20727033f --- /dev/null +++ b/components/telemetry/example/README.md @@ -0,0 +1,46 @@ +# Telemetry — USB example (→ Serial Plotter web app) + +Streams synthetic float channels from an ESP32-S3 to the browser **Serial +Plotter** web app over USB, using `espp::Telemetry` (a binary telemetry emitter +carried on the `stream_frame` framing, dispatcher module 3). + +The hosted app — — +connects on the **vendor (WebUSB)** interface, reads the channel **schema**, and +plots the live **sample** stream. It is the binary, higher-rate, +device-timestamped counterpart to the app's text/CSV Web Serial transport. + +## What it does + +- Declares four channels — `sine`, `cosine`, `noise`, `ramp` — as the schema. +- A producer task emits one sample (a `float` per channel) every ~10 ms (100 Hz), + timestamped with the device clock. +- Exposes the stream over the USB **vendor (WebUSB)** interface; a `Dispatcher` + routes module-3 frames to the emitter and serves capability discovery so the + browser **Device Hub** lists this device and links to `telemetry.html`. +- The web app can pause/resume the stream and request a rate (`SET_STREAM`), and + requests the schema on connect (`GET_SCHEMA`). + +`espp::Telemetry` itself is transport-agnostic (the `stream_frame` framing works +over CDC / UART / a socket too); this example streams over WebUSB because that is +what the web app's binary path consumes. + +Swap the synthetic generator for your real signals: build a `std::array` +in channel order and call `telemetry.emit(...)`. + +## Build & run + +```sh +idf.py -p /dev/ttyACM0 flash monitor # target esp32s3 (set in sdkconfig.defaults) +``` + +Then open the Serial Plotter web app, click **Connect (USB)**, and pick the +"espp Serial Plotter" device. The system console/logs go to the separate +built-in USB-Serial-JTAG. + +## Notes + +- Native USB (vendor / WebUSB) needs an ESP32-S3 (also S2 / P4) — not the + classic ESP32. `sdkconfig.defaults` pins `esp32s3` and enables the TinyUSB + vendor class. +- WebUSB / Web Serial are Chromium-only and need a secure context (`https`, + `http://localhost`, or `file://`). diff --git a/components/telemetry/example/main/CMakeLists.txt b/components/telemetry/example/main/CMakeLists.txt new file mode 100644 index 000000000..1dff4c6b9 --- /dev/null +++ b/components/telemetry/example/main/CMakeLists.txt @@ -0,0 +1,5 @@ +idf_component_register( + SRC_DIRS "." + INCLUDE_DIRS "." + REQUIRES telemetry dispatcher stream_frame task timer usb_device esp_tinyusb +) diff --git a/components/telemetry/example/main/telemetry_example.cpp b/components/telemetry/example/main/telemetry_example.cpp new file mode 100644 index 000000000..079cd536e --- /dev/null +++ b/components/telemetry/example/main/telemetry_example.cpp @@ -0,0 +1,228 @@ +// USB telemetry -> Serial Plotter web app example. +// +// Streams a few synthetic float channels from an ESP32-S3 to the browser over +// USB using the espp::Telemetry emitter (a small binary protocol carried on the +// stream_frame framing, dispatcher module 3). The hosted `telemetry.html` +// web app connects on the vendor (WebUSB) interface, reads the SCHEMA (channel +// names), and plots the SAMPLE stream live — the binary, higher-rate, +// device-timestamped counterpart to the app's text/CSV Web Serial transport. +// +// Replace the synthetic generator below with your real signals: build a +// std::array in channel (schema) order and call telemetry.emit(...). +// The system console/logs go to the separate built-in USB-Serial-JTAG. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dispatcher.hpp" +#include "logger.hpp" +#include "stream_frame.hpp" +#include "task.hpp" +#include "telemetry.hpp" +#include "timer.hpp" +#include "usb_device.hpp" + +using namespace std::chrono_literals; +namespace sf = espp::stream_frame; + +extern "C" void app_main(void) { + espp::Logger logger({.tag = "Telemetry", .level = espp::Logger::Verbosity::INFO}); + logger.info("Starting USB telemetry (Serial Plotter) example"); + + // --- the telemetry emitter: one float per named channel, in schema order --- + espp::Telemetry telemetry({ + .channels = {"sine", "cosine", "noise", "ramp"}, + // Start paused: only stream once a host connects and sends SET_STREAM, so + // the vendor TX FIFO doesn't fill with un-drained telemetry before anyone + // is reading (which a reconnecting host would then have to parse past). + .stream_on_start = false, + .period_ms = 10, // default 100 Hz (the web app may request another rate) + .log_level = espp::Logger::Verbosity::WARN, + }); + + // --- USB: the vendor (WebUSB) interface carries the telemetry protocol ------ + // The browser Serial Plotter plots binary telemetry over WebUSB. (The + // Telemetry service itself is transport-agnostic - CDC / UART / a socket work + // too - but the web app's Web Serial path parses text/CSV, so this example + // streams the binary protocol over WebUSB only.) The console/logs go to the + // separate built-in USB-Serial-JTAG. + espp::UsbDevice::Config usb_cfg; + usb_cfg.manufacturer = "espp"; + usb_cfg.product = "espp Serial Plotter"; + usb_cfg.log_level = espp::Logger::Verbosity::WARN; + espp::UsbDevice::VendorFunction vendor; + vendor.interface_name = "espp Telemetry (WebUSB)"; + vendor.webusb = true; + vendor.landing_page_url = "esp-cpp.github.io/espp/apps/telemetry.html"; + usb_cfg.vendor = vendor; + espp::UsbDevice usb(usb_cfg); + + // Every device->host write (samples, request replies, discovery replies) goes + // through this one tx_mutex-guarded helper so the sample-producer task and the + // RX worker never write the TinyUSB vendor FIFO at the same time. + std::mutex tx_mutex; + auto send = [&](std::span bytes) { + std::lock_guard lock(tx_mutex); + if (usb.write_vendor(bytes)) + return; + // Backpressure: the host stopped draining (e.g. a WebUSB tab closed without + // unmounting the device). write_vendor is all-or-nothing, so nothing partial + // was sent. Drop the stale queued backlog so it isn't delivered to (and + // mis-parsed by) the next host ahead of its first schema reply... + usb.vendor_write_clear(); + // ...then retry THIS frame into the now-empty FIFO. This matters for the + // one-shot SCHEMA (Telemetry has already committed the new channel width, so + // dropping it would make the host decode later SAMPLEs at the wrong width); + // for a SAMPLE the retry just delivers the latest values. Only warn if the + // frame is lost even after making room (host truly gone). + if (!usb.write_vendor(bytes)) + logger.warn_rate_limited("vendor TX backpressure; dropped a {}-byte frame", bytes.size()); + }; + telemetry.set_send(send); + + // --- dispatcher: route module-3 frames to the telemetry service + discovery - + espp::Dispatcher dispatcher; + const espp::Dispatcher::ModuleInfo info{.name = "Serial Plotter", + .app = "telemetry.html", + .description = "Live device telemetry plotting"}; + dispatcher.register_module( + espp::Telemetry::kModule, [&](const sf::Frame &frame) { telemetry.handle(frame); }, info); + dispatcher.set_device_info(usb_cfg.product); + dispatcher.serve_discovery(send); + + // --- USB RX plumbing: queue in the TinyUSB callback, dispatch from a worker - + std::mutex rx_mutex; + std::condition_variable rx_cv; + std::deque> rx_queue; + size_t rx_queued_bytes = 0; + bool rx_overflow = false; + bool rx_reset = false; // a (re)enumeration asked for a parser reset; done in rx_task + static constexpr size_t kMaxQueuedRxBytes = 8 * sf::kMaxFrameSize; + usb.set_vendor_receive_callback([&](std::span data) { + { + std::lock_guard lock(rx_mutex); + if (rx_queued_bytes + data.size() > kMaxQueuedRxBytes) { + rx_queue.clear(); + rx_queued_bytes = 0; + rx_overflow = true; + } else { + rx_queue.emplace_back(data.begin(), data.end()); + rx_queued_bytes += data.size(); + } + } + rx_cv.notify_one(); + }); + + // A physical unplug/replug re-enumerates the device: stop streaming, drop any + // stale vendor TX backlog, and reset the frame parser so the next host starts + // clean. (A WebUSB tab close does NOT unmount, so that case is handled by the + // backpressure clear in `send` above.) + // These run on the TinyUSB task, concurrently with rx_task's dispatcher.feed(). + // The dispatcher's frame parser is not thread-safe, so don't reset it here - + // drop any queued pre-reset chunks and flag a reset for rx_task to perform, + // so all parser access stays on the one thread. (set_streaming is atomic and + // vendor_write_clear touches only the TX FIFO, so both are fine to call here.) + auto request_parser_reset = [&] { + { + std::lock_guard lock(rx_mutex); + rx_queue.clear(); + rx_queued_bytes = 0; + rx_reset = true; + } + rx_cv.notify_one(); + }; + usb.set_unmount_callback([&] { + telemetry.set_streaming(false); + usb.vendor_write_clear(); + request_parser_reset(); + }); + usb.set_mount_callback([&] { + usb.vendor_write_clear(); + request_parser_reset(); + }); + + std::error_code usb_ec; + if (!usb.initialize(usb_ec)) { + logger.error("Failed to initialize USB device: {} - no host transport available; aborting", + usb_ec.message()); + return; + } + + espp::Task rx_task({.callback = [&](std::mutex &, std::condition_variable &) -> bool { + std::deque> chunks; + bool overflowed = false, do_reset = false; + { + std::unique_lock lock(rx_mutex); + rx_cv.wait_for(lock, 100ms, [&] { + return !rx_queue.empty() || rx_overflow || rx_reset; + }); + std::swap(chunks, rx_queue); + rx_queued_bytes = 0; + overflowed = rx_overflow; + rx_overflow = false; + do_reset = rx_reset; + rx_reset = false; + } + // All parser mutation happens here on the one rx thread. + // Reset first (enumeration change or overflow), then feed + // any chunks that arrived after the reset was requested. + if (do_reset || overflowed) + dispatcher.reset(); + if (overflowed) + return false; // chunks were dropped on overflow + for (const auto &chunk : chunks) + dispatcher.feed(chunk); + return false; + }, + .task_config = {.name = "telemetry_rx", .stack_size_bytes = 8192}}); + rx_task.start(); + + // --- synthetic signal producer: emit one SAMPLE per period ------------------ + // Replace this with your real signals. Values are in schema (channel) order. + const auto t0 = std::chrono::steady_clock::now(); + espp::Timer gen_timer( + {.period = std::chrono::milliseconds(telemetry.period_ms()), + .callback = [&]() -> bool { + const float t = + std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + const float two_pi = 6.2831853f; + const float noise = static_cast(std::rand()) / RAND_MAX * 0.4f - 0.2f; + const std::array values = { + std::sin(two_pi * 1.0f * t), // sine @ 1 Hz + std::cos(two_pi * 0.5f * t), // cosine @ 0.5 Hz + noise, // uniform noise + std::fmod(t, 2.0f) - 1.0f, // -1..1 sawtooth ramp + }; + telemetry.emit(values); // no-op while streaming is paused + return false; // keep running + }, + .task_config = {.name = "telemetry_gen", .stack_size_bytes = 4096}}); + gen_timer.start(); + + logger.info("Telemetry ready. Open the Serial Plotter web app and connect over WebUSB " + "(channels: sine, cosine, noise, ramp)."); + + // The web app can change the sample rate via SET_STREAM, which updates the + // Telemetry period atomic but not the running producer timer. Poll it and + // retune the timer when it changes so the advertised rate is actually honored + // (set_period() is called here, off the timer's own callback thread). + uint16_t applied_period_ms = telemetry.period_ms(); + while (true) { + std::this_thread::sleep_for(200ms); + const uint16_t want = telemetry.period_ms(); + if (want != 0 && want != applied_period_ms) { + gen_timer.set_period(std::chrono::milliseconds(want)); + applied_period_ms = want; + logger.debug("retuned producer timer to {} ms", want); + } + } +} diff --git a/components/telemetry/example/sdkconfig.defaults b/components/telemetry/example/sdkconfig.defaults new file mode 100644 index 000000000..314a2e515 --- /dev/null +++ b/components/telemetry/example/sdkconfig.defaults @@ -0,0 +1,35 @@ +# This example uses the native USB-OTG peripheral (vendor / WebUSB), which is +# only available on the ESP32-S3 (also S2 / P4) -- NOT the classic ESP32. Pin the +# target here so a bare `idf.py build` does not fall back to esp32. +CONFIG_IDF_TARGET="esp32s3" + +# Common ESP-related +CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192 + +# Enable the TinyUSB vendor-specific class (THE key enablement for the vendor / +# WebUSB interface). esp_tinyusb gates CFG_TUD_VENDOR behind +# CONFIG_TINYUSB_VENDOR_COUNT; setting it > 0 compiles in the vendor class +# driver so espp::UsbDevice's vendor function (bInterfaceClass 0xFF + WebUSB) +# works. The telemetry web app plots over this WebUSB interface. +CONFIG_TINYUSB_VENDOR_COUNT=1 + +# CDC + HID are enabled only to keep the usb_device component compiling (it +# includes the TinyUSB CDC/HID headers unconditionally). This example creates +# NO CDC or HID interface -- it exposes the vendor (WebUSB) interface only, so +# the device enumerates with just that one telemetry interface. +CONFIG_TINYUSB_CDC_ENABLED=y +CONFIG_TINYUSB_CDC_COUNT=1 +CONFIG_TINYUSB_HID_COUNT=1 + +# The vendor TX FIFO must hold the LARGEST single frame the emitter can produce +# so UsbDevice::write_vendor() stays all-or-nothing (the send callback relies on +# that: a rejected frame leaves nothing partial in the FIFO). The biggest frame +# is a full SCHEMA: clamp_channels() bounds its payload to stream_frame's +# kMaxPayloadSize (4096), which encodes to ~4109 bytes with the 9-byte header and +# 4-byte CRC. Size the FIFO comfortably above that (8192) — this also leaves +# headroom for a burst of batched SAMPLE frames. The vendor RX FIFO only carries +# short requests (GET_SCHEMA / SET_STREAM), so 512 bytes is plenty there. +CONFIG_TINYUSB_VENDOR_RX_BUFSIZE=512 +CONFIG_TINYUSB_VENDOR_TX_BUFSIZE=8192 + +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y diff --git a/components/telemetry/idf_component.yml b/components/telemetry/idf_component.yml new file mode 100644 index 000000000..cb3ce809e --- /dev/null +++ b/components/telemetry/idf_component.yml @@ -0,0 +1,22 @@ +## IDF Component Manager Manifest File +license: "MIT" +description: "Browser data plotter (Web Serial + WebUSB) that auto-parses columnar serial output and plots it, plus espp::Telemetry: a binary telemetry emitter (dispatcher module over stream_frame) the web app plots over WebUSB." +url: "https://github.com/esp-cpp/espp/tree/main/components/telemetry" +repository: "git://github.com/esp-cpp/espp.git" +maintainers: + - William Emfinger +documentation: "https://esp-cpp.github.io/espp/apps/telemetry.html" +examples: + - path: example +tags: + - webapp + - plotting + - serial + - web-serial + - webusb + - telemetry +dependencies: + idf: + version: '>=5.0' + espp/base_component: '>=1.0' + espp/stream_frame: '>=1.0' diff --git a/components/telemetry/include/telemetry.hpp b/components/telemetry/include/telemetry.hpp new file mode 100644 index 000000000..4b8b7234b --- /dev/null +++ b/components/telemetry/include/telemetry.hpp @@ -0,0 +1,405 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "base_component.hpp" +#include "stream_frame.hpp" + +namespace espp { +/// @brief Binary telemetry emitter for the Serial Plotter web app. +/// +/// A tiny device->host protocol carried over the espp `stream_frame` framing +/// (so it can share one USB vendor / CDC stream with other modules via +/// `espp::Dispatcher`). Firmware declares a fixed set of named float channels +/// (the SCHEMA) and pushes SAMPLE frames — a device timestamp plus one float +/// per channel — which the hosted `telemetry.html` web app decodes and +/// plots, exactly like the columnar Web-Serial path but binary, higher rate, +/// and with device-accurate timestamps. +/// +/// This is the purpose-built counterpart to the app's text (CSV-style) +/// Web-Serial transport: instead of parsing printed columns, the device sends +/// typed samples directly. +/// +/// ## Wire protocol (dispatcher module id 3) +/// +/// Every message is a `stream_frame` frame with `module == kModule`. The frame +/// `type` byte's high bit distinguishes direction: host->device requests are +/// `0x0X`, device->host frames are `0x8X` (which sets the frame reply flag). +/// +/// - **GET_SCHEMA** (host->device, no payload): request the current SCHEMA. +/// - **SET_STREAM** (host->device, `[enabled u8][period_ms u16]`): enable or +/// disable streaming and request a sample period (informational — the +/// firmware's emit cadence is authoritative; the requested period is exposed +/// via period_ms() so an app can honor it). Answered with OK. +/// - **SCHEMA** (device->host): `[version u8][flags u8][nchannels u8]` then, per +/// channel, `[type u8][name_len u8][name bytes]`. `version == kSchemaVersion`, +/// `flags == 0` (reserved), channel `type == 0` (f32). Sent on GET_SCHEMA, on +/// set_channels(), and on demand via send_schema(). +/// - **SAMPLE** (device->host): one or more packed records, each +/// `[timestamp u32 microseconds][f32 × nchannels]` little-endian. A frame may +/// batch several records (payload size is an exact multiple of the record +/// size) for higher throughput. +/// - **OK** (device->host, `[request_type u8]`): acknowledges a request. +/// - **ERROR** (device->host, `[request_type u8][code u32][utf8 message]`). +/// +/// ## Threading +/// +/// emit() is typically called from a producer task while requests are handled +/// on a transport RX task; both are safe to call concurrently. Every outbound +/// frame is serialized on an internal send mutex that is held across the user +/// `send` callback, so the callback (a) never runs concurrently with itself — a +/// `send` that is not itself thread-safe cannot interleave the bytes of two +/// frames — and (b) always delivers a SAMPLE together with the channel set it +/// was built from, so the host never decodes a sample against a stale SCHEMA. +/// +/// Because that send mutex is held while `send` runs, the callback MUST NOT +/// re-enter this object — do not call emit(), send_schema(), set_channels(), +/// handle() or feed() from inside `send`, or it will deadlock on the same +/// (non-recursive) mutex. Keep `send` to writing the bytes to the transport. +class Telemetry : public espp::BaseComponent { +public: + /// Dispatcher module id owned by the telemetry protocol (the frame `module` + /// byte). Device->host Type values keep the high bit set, which the framing + /// maps to the reply flag. + static constexpr uint8_t kModule = 3; + + /// Version byte at the head of a SCHEMA payload, so the wire format can evolve. + static constexpr uint8_t kSchemaVersion = 1; + + /// Maximum channel count (the SCHEMA encodes it as a u8). + static constexpr size_t kMaxChannels = 255; + + /// Frame `type` values within the telemetry module. + enum class Type : uint8_t { + // host -> device + GetSchema = 0x01, ///< request the current SCHEMA + SetStream = 0x02, ///< [enabled u8][period_ms u16]: enable/disable + rate + // device -> host (high bit set) + Schema = 0x81, ///< channel schema (see class docs) + Sample = 0x82, ///< one or more [timestamp u32 us][f32 x nchannels] records + Ok = 0x83, ///< [request_type u8]: request acknowledged + Error = 0x84, ///< [request_type u8][code u32][utf8 message] + }; + + /// Channel value type (only 32-bit float today; reserved for future widening). + enum class ChannelType : uint8_t { F32 = 0 }; + + /// @brief Function used to transmit one encoded frame to the host. + /// @param frame The complete encoded frame bytes (header + payload + CRC). + using send_fn = std::function frame)>; + + /// Configuration for the Telemetry emitter. + struct Config { + std::vector channels; ///< Channel names, in sample order (>= 1). + send_fn send{nullptr}; ///< Transmits an encoded frame (may be set later). + bool stream_on_start{true}; ///< Start with streaming enabled. + uint16_t period_ms{20}; ///< Default requested sample period (informational). + espp::Logger::Verbosity log_level{espp::Logger::Verbosity::WARN}; ///< Logger verbosity. + }; + + /// @brief Construct the emitter. + /// @param config Channel names, the (optional) send function, and defaults. + explicit Telemetry(const Config &config) + : BaseComponent("Telemetry", config.log_level) + , channels_(config.channels) + , send_(config.send) + , streaming_(config.stream_on_start) + , period_ms_(config.period_ms) { + clamp_channels(); + } + + /// @brief Set (or replace) the transmit function, e.g. after USB init. + void set_send(send_fn fn) { + std::lock_guard lock(mutex_); + send_ = std::move(fn); + } + + /// @brief Push one sample with an explicit device timestamp. + /// @param values One value per channel, in schema order (size must match the + /// channel count, else the sample is dropped with a rate-limited warning). + /// @param timestamp_us Device timestamp in microseconds (u32; wraps ~71 min). + /// @note No-op while streaming is disabled or no send function is configured. + void emit(std::span values, uint32_t timestamp_us) { + if (!streaming_.load()) + return; // fast path; the authoritative check is under send_mutex_ below + // Hold send_mutex_ across BOTH building this sample (which reads the current + // channel count) and sending it. That makes the sample's width and its + // delivery atomic with respect to set_channels()/send_schema() (which take + // the same outer lock), so a new SCHEMA can never be interleaved between a + // sample built from one channel set and its transmission - which would make + // the host decode the sample at the wrong record width. Lock order is always + // send_mutex_ (outer) -> mutex_ (inner); no sender ever takes them the other + // way, so this cannot deadlock. + std::lock_guard send_lock(send_mutex_); + // Re-check streaming under send_mutex_: a SET_STREAM(false) handler stores + // false and then sends its OK through this same mutex. Re-reading the flag + // here guarantees any SAMPLE we send precedes that OK - a sample can never be + // delivered after the stream-disable acknowledgment. + if (!streaming_.load()) + return; + std::vector frame; + send_fn s; + { + std::lock_guard lock(mutex_); + if (values.size() != channels_.size()) { + logger_.warn_rate_limited("emit(): {} values for {} channels; dropping", values.size(), + channels_.size()); + return; + } + if (!send_) + return; + s = send_; + std::vector p; + p.reserve(4 + 4 * values.size()); + espp::stream_frame::put_u32(p, timestamp_us); + for (float v : values) + put_f32(p, v); + frame = build(Type::Sample, p); + } + s(std::span(frame)); // send while still holding send_mutex_ + } + + /// @brief Push one sample timestamped with the current device time. + void emit(std::span values) { emit(values, now_us()); } + + /// @brief Redefine the channel set at runtime and send a fresh SCHEMA. + /// @note The channel-set update and the outbound SCHEMA are performed together + /// under send_mutex_, so a concurrent emit() cannot deliver a sample + /// built from the new channels before (or the old channels after) the + /// matching SCHEMA reaches the host. + void set_channels(std::vector channels) { + std::lock_guard send_lock(send_mutex_); + std::vector frame; + send_fn s; + { + std::lock_guard lock(mutex_); + channels_ = std::move(channels); + clamp_channels(); + if (!send_) + return; // no transport yet; the next GetSchema will carry the new set + s = send_; + frame = build(Type::Schema, build_schema_payload_locked()); + } + s(std::span(frame)); // send while still holding send_mutex_ + } + + /// @brief The current channel names (schema order). + std::vector channels() const { + std::lock_guard lock(mutex_); + return channels_; + } + + /// @brief Whether streaming is currently enabled. + bool streaming() const { return streaming_.load(); } + + /// @brief Enable or disable streaming (SAMPLE emission). + void set_streaming(bool on) { streaming_.store(on); } + + /// @brief The host-requested sample period in milliseconds (informational). + uint16_t period_ms() const { return period_ms_.load(); } + + /// @brief Send the current SCHEMA frame now (device->host). + /// @note Uses the same send_mutex_-outer / mutex_-inner order as emit() so the + /// SCHEMA and any concurrent SAMPLE stay consistently ordered. + void send_schema() const { + std::lock_guard send_lock(send_mutex_); + std::vector frame; + send_fn s; + { + std::lock_guard lock(mutex_); + if (!send_) + return; + s = send_; + frame = build(Type::Schema, build_schema_payload_locked()); + } + s(std::span(frame)); // send while still holding send_mutex_ + } + + /// @brief Dispatcher handler: process one frame addressed to this module. + /// + /// Register with `dispatcher.register_module(Telemetry::kModule, ...)`. Ignores + /// reply-flagged frames (device->host pushes are never host requests). + void handle(const espp::stream_frame::Frame &frame) { + if (frame.is_reply()) + return; + handle_request(frame.type, frame.payload); + } + + /// @brief Feed raw transport bytes through an internal frame parser (for use + /// without a Dispatcher). Processes every complete frame for this module. + void feed(std::span data) { + std::vector frames; + { + std::lock_guard lock(mutex_); + frames = parser_.feed(data); + } + for (const auto &frame : frames) { + if (frame.module != kModule || frame.is_reply()) + continue; + handle_request(frame.type, frame.payload); + } + } + + /// @brief Discard any partially-buffered bytes in the internal parser (e.g. + /// on transport reconnect). Only relevant when using feed(). + void reset_parser() { + std::lock_guard lock(mutex_); + parser_.reset(); + } + +protected: + /// Handle one request `type` + `payload`; builds any reply under the lock and + /// transmits it with the lock released. + void handle_request(uint8_t type, std::span payload) { + switch (static_cast(type)) { + case Type::GetSchema: + send_schema(); + break; + case Type::SetStream: { + if (payload.size() < 3) { + send_error(type, "SET_STREAM payload too short"); + return; + } + const bool enabled = payload[0] != 0; + const uint16_t period = + static_cast(payload[1] | (static_cast(payload[2]) << 8)); + if (period != 0) + period_ms_.store(period); + streaming_.store(enabled); + logger_.debug("SET_STREAM enabled={} period_ms={}", enabled, period_ms_.load()); + send_ok(type); + break; + } + default: + // Unknown telemetry type: report it (the module id already matched). + send_error(type, "unknown telemetry message"); + break; + } + } + + /// Serialize the SCHEMA payload. Caller must hold `mutex_`. The channel count + /// is a u8; channels_ is capped at kMaxChannels (see clamp_channels), but the + /// count and the serialized channels are derived from the same bound so the + /// payload is always self-consistent. + std::vector build_schema_payload_locked() const { + const size_t n = std::min(channels_.size(), kMaxChannels); + std::vector p; + p.push_back(kSchemaVersion); + p.push_back(0); // flags (reserved) + p.push_back(static_cast(n)); + for (size_t i = 0; i < n; i++) { + const auto &name = channels_[i]; + p.push_back(static_cast(ChannelType::F32)); + const uint8_t len = static_cast(std::min(name.size(), 255)); + p.push_back(len); + p.insert(p.end(), name.begin(), name.begin() + len); + } + return p; + } + + /// Clamp channels_ so a SCHEMA frame is always well-formed (caller holds + /// mutex_, or is the constructor). Enforces the u8 channel-count limit AND the + /// stream_frame payload-size limit, and warns for an empty configuration. + void clamp_channels() { + if (channels_.empty()) { + // The config contract asks for >= 1 channel; a zero-channel SCHEMA is + // ignored by the web client. Warn rather than emit a silently-useless one. + logger_.warn("Telemetry configured with no channels; SCHEMA will be empty"); + return; + } + // Cap the channel count to the u8 nchannels field. + if (channels_.size() > kMaxChannels) { + logger_.warn("{} channels exceeds the {}-channel SCHEMA limit; truncating", channels_.size(), + kMaxChannels); + channels_.resize(kMaxChannels); + } + // Cap so the encoded SCHEMA fits kMaxPayloadSize: a 3-byte header (version, + // flags, nchannels) plus, per channel, [type u8][len u8][<=255 name bytes]. + // Drop overflowing channels so build_frame() never rejects an oversized + // payload (which would call `send` with an empty frame). At least one + // channel always fits (3 + 2 + 255 < kMaxPayloadSize). + size_t bytes = 3, kept = 0; + for (; kept < channels_.size(); kept++) { + const size_t next = bytes + 2 + std::min(channels_[kept].size(), 255); + if (next > espp::stream_frame::kMaxPayloadSize) + break; + bytes = next; + } + if (kept < channels_.size()) { + logger_.warn("encoded SCHEMA would exceed {} bytes; keeping {} of {} channels", + espp::stream_frame::kMaxPayloadSize, kept, channels_.size()); + channels_.resize(kept); + } + } + + void send_ok(uint8_t request_type) const { + const uint8_t p[] = {request_type}; + send_frame(build(Type::Ok, p)); + } + + void send_error(uint8_t request_type, std::string_view message) const { + logger_.warn("{} (type 0x{:02x})", message, request_type); + std::vector p; + p.push_back(request_type); + espp::stream_frame::put_u32(p, 0); // reserved code + p.insert(p.end(), message.begin(), message.end()); + send_frame(build(Type::Error, p)); + } + + /// Transmit an already-built frame via the configured send function. Takes + /// send_mutex_ (outer) then mutex_ (inner) - the same order as emit() and + /// send_schema() - so frames' bytes never interleave and no lock-order + /// inversion is possible. Never called while mutex_ is already held. + void send_frame(const std::vector &frame) const { + std::lock_guard send_lock(send_mutex_); + send_fn s; + { + std::lock_guard lock(mutex_); + s = send_; + } + if (s) + s(std::span(frame)); // send while still holding send_mutex_ + } + + /// Build an encoded frame for a telemetry message type. Device->host types + /// (high bit set) map to the frame reply flag. + static std::vector build(Type type, std::span payload = {}) { + const bool reply = (static_cast(type) & 0x80) != 0; + return espp::stream_frame::build_frame(reply, kModule, static_cast(type), payload); + } + + /// Append a little-endian IEEE-754 float32 to a byte buffer. + static void put_f32(std::vector &out, float value) { + uint32_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + espp::stream_frame::put_u32(out, bits); + } + + /// Current device time in microseconds (truncated to u32; wraps ~71 min). + static uint32_t now_us() { + using namespace std::chrono; + return static_cast( + duration_cast(steady_clock::now().time_since_epoch()).count()); + } + + mutable std::mutex mutex_; ///< guards channels_, send_, and the parser + mutable std::mutex send_mutex_; ///< outer lock: serializes send callbacks and + ///< keeps each frame's build+send atomic (order: + ///< send_mutex_ then mutex_) + std::vector channels_; + send_fn send_; + std::atomic streaming_; + std::atomic period_ms_; + espp::stream_frame::StreamParser parser_; +}; +} // namespace espp diff --git a/components/serial_plotter/web/serial_plotter.html b/components/telemetry/web/telemetry.html similarity index 56% rename from components/serial_plotter/web/serial_plotter.html rename to components/telemetry/web/telemetry.html index 312a526bb..f76a8c6c1 100644 --- a/components/serial_plotter/web/serial_plotter.html +++ b/components/telemetry/web/telemetry.html @@ -4,7 +4,7 @@ espp Serial Plotter - +