From fcc789acaab8f0803b8842ced29381606ede98a6 Mon Sep 17 00:00:00 2001 From: Mark Bacon Date: Sat, 17 Oct 2026 08:24:30 +0100 Subject: [PATCH] Primers: merge the Tech 101s, the technical-documentation primers and the archive topic notes One set of 62 short primers in 8 areas (architecture, distributed systems, APIs and data, cloud, IoT and embedded, security, methods and delivery, programming and tools), each area with its own page. The old primers are rewritten rather than copied, duplicates merged (edge computing, RCA), out-of-date material fixed (Cloud IoT Core, the Paho 2 API, OWASP 2025), and 24 new primers drawn from the archive's topic notes. documents.primers lists the area pages with a page break and their primers one heading level down, so Word and PDF nest the primers under their area. Co-Authored-By: Claude Opus 5.5 --- content/primers/agile-development.md | 58 +++++++++++ content/primers/anti-patterns.md | 51 ++++++++++ content/primers/apis-and-data.md | 14 +++ .../primers/architecture-decision-records.md | 62 ++++++++++++ content/primers/architecture.md | 20 ++++ content/primers/bootloaders.md | 60 ++++++++++++ content/primers/c4-model.md | 62 ++++++++++++ content/primers/cap-and-pacelc.md | 58 +++++++++++ content/primers/cbor.md | 68 +++++++++++++ content/primers/cellular-lpwan.md | 60 ++++++++++++ content/primers/cicd-pipelines.md | 68 +++++++++++++ content/primers/cloud.md | 30 ++++++ content/primers/coap.md | 63 ++++++++++++ content/primers/code-review.md | 64 ++++++++++++ content/primers/consistency-models.md | 55 +++++++++++ content/primers/cryptography-basics.md | 58 +++++++++++ content/primers/data-formats.md | 77 +++++++++++++++ content/primers/design-patterns.md | 97 +++++++++++++++++++ content/primers/design-principles.md | 46 +++++++++ .../digital-twins-and-device-shadows.md | 76 +++++++++++++++ content/primers/distributed-systems.md | 14 +++ content/primers/docs-as-code.md | 62 ++++++++++++ content/primers/domain-driven-design.md | 54 +++++++++++ content/primers/edge-computing-for-iot.md | 68 +++++++++++++ content/primers/embedded-systems-design.md | 65 +++++++++++++ content/primers/etsi-en-303-645.md | 47 +++++++++ content/primers/event-driven-architecture.md | 60 ++++++++++++ content/primers/event-storming.md | 65 +++++++++++++ content/primers/finite-state-machines.md | 80 +++++++++++++++ content/primers/firmware-over-the-air.md | 63 ++++++++++++ content/primers/git-best-practices.md | 51 ++++++++++ .../primers/hardware-in-the-loop-testing.md | 63 ++++++++++++ content/primers/hexagonal-architecture.md | 60 ++++++++++++ content/primers/idempotency.md | 59 +++++++++++ content/primers/index.md | 61 +++++++++++- content/primers/infrastructure-as-code.md | 62 ++++++++++++ content/primers/interrupts-and-dma.md | 77 +++++++++++++++ content/primers/iot-and-embedded.md | 41 ++++++++ content/primers/iot-on-google-cloud.md | 60 ++++++++++++ content/primers/iso-30141.md | 59 +++++++++++ content/primers/kafka-event-streaming.md | 65 +++++++++++++ .../lightweight-architecture-evaluations.md | 67 +++++++++++++ content/primers/lwm2m.md | 70 +++++++++++++ content/primers/methods-and-delivery.md | 28 ++++++ .../modular-monolith-vs-microservices.md | 60 ++++++++++++ content/primers/mqtt.md | 85 ++++++++++++++++ content/primers/oauth-and-oidc.md | 75 ++++++++++++++ content/primers/openapi-and-asyncapi.md | 95 ++++++++++++++++++ content/primers/owasp-top-10.md | 46 +++++++++ content/primers/pki-and-certificates.md | 66 +++++++++++++ content/primers/plantuml-and-mermaid.md | 75 ++++++++++++++ content/primers/post-quantum-cryptography.md | 46 +++++++++ content/primers/programming-and-tools.md | 16 +++ content/primers/python-for-automation.md | 82 ++++++++++++++++ content/primers/rest-apis.md | 64 ++++++++++++ content/primers/root-cause-analysis.md | 60 ++++++++++++ content/primers/rtos-and-freertos.md | 75 ++++++++++++++ content/primers/rust-for-systems.md | 56 +++++++++++ content/primers/saga-pattern.md | 59 +++++++++++ content/primers/scrum-vs-kanban.md | 61 ++++++++++++ content/primers/sdlc-models.md | 63 ++++++++++++ content/primers/secure-coding-practices.md | 55 +++++++++++ content/primers/security.md | 28 ++++++ content/primers/serverless-computing.md | 68 +++++++++++++ content/primers/static-code-analysis.md | 48 +++++++++ content/primers/technical-debt.md | 48 +++++++++ content/primers/terminal-modernisation.md | 67 +++++++++++++ content/primers/testing-strategies.md | 60 ++++++++++++ content/primers/tls-and-dtls.md | 59 +++++++++++ content/primers/watchdog-timers.md | 48 +++++++++ content/primers/zero-trust-architecture.md | 60 ++++++++++++ py-doc.yml | 73 +++++++++++++- 72 files changed, 4243 insertions(+), 3 deletions(-) create mode 100644 content/primers/agile-development.md create mode 100644 content/primers/anti-patterns.md create mode 100644 content/primers/apis-and-data.md create mode 100644 content/primers/architecture-decision-records.md create mode 100644 content/primers/architecture.md create mode 100644 content/primers/bootloaders.md create mode 100644 content/primers/c4-model.md create mode 100644 content/primers/cap-and-pacelc.md create mode 100644 content/primers/cbor.md create mode 100644 content/primers/cellular-lpwan.md create mode 100644 content/primers/cicd-pipelines.md create mode 100644 content/primers/cloud.md create mode 100644 content/primers/coap.md create mode 100644 content/primers/code-review.md create mode 100644 content/primers/consistency-models.md create mode 100644 content/primers/cryptography-basics.md create mode 100644 content/primers/data-formats.md create mode 100644 content/primers/design-patterns.md create mode 100644 content/primers/design-principles.md create mode 100644 content/primers/digital-twins-and-device-shadows.md create mode 100644 content/primers/distributed-systems.md create mode 100644 content/primers/docs-as-code.md create mode 100644 content/primers/domain-driven-design.md create mode 100644 content/primers/edge-computing-for-iot.md create mode 100644 content/primers/embedded-systems-design.md create mode 100644 content/primers/etsi-en-303-645.md create mode 100644 content/primers/event-driven-architecture.md create mode 100644 content/primers/event-storming.md create mode 100644 content/primers/finite-state-machines.md create mode 100644 content/primers/firmware-over-the-air.md create mode 100644 content/primers/git-best-practices.md create mode 100644 content/primers/hardware-in-the-loop-testing.md create mode 100644 content/primers/hexagonal-architecture.md create mode 100644 content/primers/idempotency.md create mode 100644 content/primers/infrastructure-as-code.md create mode 100644 content/primers/interrupts-and-dma.md create mode 100644 content/primers/iot-and-embedded.md create mode 100644 content/primers/iot-on-google-cloud.md create mode 100644 content/primers/iso-30141.md create mode 100644 content/primers/kafka-event-streaming.md create mode 100644 content/primers/lightweight-architecture-evaluations.md create mode 100644 content/primers/lwm2m.md create mode 100644 content/primers/methods-and-delivery.md create mode 100644 content/primers/modular-monolith-vs-microservices.md create mode 100644 content/primers/mqtt.md create mode 100644 content/primers/oauth-and-oidc.md create mode 100644 content/primers/openapi-and-asyncapi.md create mode 100644 content/primers/owasp-top-10.md create mode 100644 content/primers/pki-and-certificates.md create mode 100644 content/primers/plantuml-and-mermaid.md create mode 100644 content/primers/post-quantum-cryptography.md create mode 100644 content/primers/programming-and-tools.md create mode 100644 content/primers/python-for-automation.md create mode 100644 content/primers/rest-apis.md create mode 100644 content/primers/root-cause-analysis.md create mode 100644 content/primers/rtos-and-freertos.md create mode 100644 content/primers/rust-for-systems.md create mode 100644 content/primers/saga-pattern.md create mode 100644 content/primers/scrum-vs-kanban.md create mode 100644 content/primers/sdlc-models.md create mode 100644 content/primers/secure-coding-practices.md create mode 100644 content/primers/security.md create mode 100644 content/primers/serverless-computing.md create mode 100644 content/primers/static-code-analysis.md create mode 100644 content/primers/technical-debt.md create mode 100644 content/primers/terminal-modernisation.md create mode 100644 content/primers/testing-strategies.md create mode 100644 content/primers/tls-and-dtls.md create mode 100644 content/primers/watchdog-timers.md create mode 100644 content/primers/zero-trust-architecture.md diff --git a/content/primers/agile-development.md b/content/primers/agile-development.md new file mode 100644 index 0000000..442c6e3 --- /dev/null +++ b/content/primers/agile-development.md @@ -0,0 +1,58 @@ +--- +title: Agile development +--- + +# Agile development + +Agile development is a family of ways of working built on short cycles, working software and frequent feedback. Rather than planning everything up front, an agile team delivers a small, usable piece, learns from it, and adjusts. The term comes from the *Manifesto for Agile Software Development*, written in 2001 by seventeen practitioners who had each been developing lighter alternatives to heavyweight processes. + +## The values + +The manifesto states four preferences. Its authors value the items on the left more, while still valuing those on the right: + +- Individuals and interactions over processes and tools. +- Working software over comprehensive documentation. +- Customer collaboration over contract negotiation. +- Responding to change over following a plan. + +Twelve principles follow, including delivering working software frequently, welcoming changing requirements, keeping a sustainable pace, and reflecting regularly on how to become more effective. The full text is short and worth reading at [agilemanifesto.org](https://agilemanifesto.org/). + +## The cycle + +```mermaid +flowchart LR + P[Plan a small increment] --> B[Build and test] + B --> R[Review with users and stakeholders] + R --> L[Reflect and improve] + L --> P +``` + +[[CAPTION:Figure]] The feedback loop at the heart of agile work. + +## Common practices + +- **A prioritised backlog**: one ordered list of work, with the most valuable items at the top, refined continuously. +- **User stories**: small descriptions of value from a user's point of view ("As a field engineer, I want to see a device's last contact time, so that I can tell if it is offline"), each with acceptance criteria. +- **Short iterations** of one to four weeks, or a continuous flow of small items. +- **Definitions of ready and done**: shared checklists for when work can start and when it is truly finished. +- **Daily coordination**: a short meeting on progress towards the goal and anything blocking it. +- **Reviews and retrospectives**: show the work to stakeholders; then discuss how the team itself can improve. +- **Technical practices**: [continuous integration](cicd-pipelines.md), automated [testing](testing-strategies.md), [code review](code-review.md) and refactoring. Without them, short cycles slowly fill with [technical debt](technical-debt.md). + +## Frameworks + +- **[Scrum and Kanban](scrum-vs-kanban.md)** are the two most common team-level approaches. +- **Extreme Programming (XP)** focuses on engineering practices: pair programming, test-driven development, continuous integration and simple design. +- **Lean software development** applies ideas from Toyota's production system: remove waste, limit work in progress, optimise the whole flow. +- **Scaling frameworks** such as SAFe, LeSS and Nexus coordinate many teams. They add structure and overhead, so use the lightest that works. + +## Pitfalls + +- **Ceremony without the point**: stand-ups, sprints and story points with no real feedback from users is "agile" in name only. See cargo cult programming in [anti-patterns](anti-patterns.md). +- **No architecture**: agile does not mean no design. Make the decisions that are hard to change early and record them as [ADRs](architecture-decision-records.md). +- **Hardware reality**: physical products have lead times and certification. Iterate on firmware, software and prototypes, and plan hardware milestones explicitly. + +## Where next + +- [Scrum and Kanban](scrum-vs-kanban.md) compares the two common frameworks. +- The [Templates](../templates/index.md) section has definition of ready and definition of done checklists. diff --git a/content/primers/anti-patterns.md b/content/primers/anti-patterns.md new file mode 100644 index 0000000..e4833b4 --- /dev/null +++ b/content/primers/anti-patterns.md @@ -0,0 +1,51 @@ +--- +title: Anti-patterns +--- + +# Anti-patterns + +An anti-pattern is a common response to a problem that looks reasonable at the time and turns out badly. Like a design pattern, it has a name, which makes it easier to spot and talk about in a review. Most anti-patterns are not caused by bad engineers; they are what happens to good code under deadline pressure when nobody steps back. + +## Structural anti-patterns + +| Anti-pattern | What it looks like | The way out | +| --- | --- | --- | +| Big ball of mud | No visible structure; everything calls everything | Find seams, draw module boundaries, enforce them with tooling | +| God object | One class or module that knows and does far too much | Split by responsibility; move behaviour to the data it uses | +| Spaghetti code | Control flow that jumps around with no clear structure | Extract functions, add tests, then restructure | +| Lava flow | Dead or half-finished code that nobody dares remove | Find out what it does, test the behaviour you need, delete the rest | +| Lava layer | Several generations of technology for the same job (three ways to access the database) | Pick one, migrate the others deliberately, finish the migration | +| Shotgun surgery | One logical change needs edits in many places | Bring the scattered knowledge into one module | + +[[CAPTION:Table]] Anti-patterns in the structure of code. + +## Behavioural anti-patterns + +These are about how teams make decisions rather than the shape of the code. + +- **Golden hammer**: using a familiar tool for everything, whether or not it fits. The cure is to state the problem before naming the tool. +- **Cargo cult programming**: copying code or practices without understanding why they work. Daily stand-ups with no shared goal, or configuration pasted from a forum, are typical. Ask what each part is for. +- **Premature optimisation**: making code faster, more complex or harder to read before measuring whether speed matters there. Measure first; optimise the hot path; keep the rest simple. +- **Not invented here**: rebuilding something that a well-maintained library already does. + +## Spotting them early + +Anti-patterns grow slowly, so look for the trends: + +- Files or classes that appear in most changes, or that keep growing. +- Changes that need edits in many unrelated places. +- Areas that people avoid or that have no tests. +- Code that nobody can explain in a review. + +Static analysis can measure some of this (size, complexity, coupling, duplication). See [static code analysis](static-code-analysis.md). + +## Fixing them + +Rewrites rarely work. Improve the code in small, tested steps, with each step leaving it working. Michael Feathers' *Working Effectively with Legacy Code* is the standard guide. The strangler fig approach, where new code slowly replaces old code behind a stable interface, works well for larger areas. + +Record the decision to tolerate or remove an anti-pattern as [technical debt](technical-debt.md), so it is visible and scheduled rather than forgotten. + +## Where next + +- [Design principles](design-principles.md) and [design patterns](design-patterns.md) describe what good looks like. +- The [Guides](../guides/index.md) section covers code review and refactoring in more depth. diff --git a/content/primers/apis-and-data.md b/content/primers/apis-and-data.md new file mode 100644 index 0000000..5c6ea6a --- /dev/null +++ b/content/primers/apis-and-data.md @@ -0,0 +1,14 @@ +--- +title: APIs and data primers +--- + +# APIs and data primers + +Systems talk to each other through interfaces, and the interface is a contract: once other teams or devices depend on it, it is expensive to change. These primers cover the common styles and formats. + +- [REST APIs](rest-apis.md): resources, HTTP methods and status codes. +- [OpenAPI and AsyncAPI](openapi-and-asyncapi.md): describing request-response and event-driven interfaces. +- [Data formats](data-formats.md): JSON, YAML and Protocol Buffers. +- [CBOR and compact binary formats](cbor.md): encoding data for constrained devices and networks. + +The [Guidelines](../guidelines/index.md) and [Standards](../standards/index.md) sections set the rules for API design here; these primers explain the ideas behind them. diff --git a/content/primers/architecture-decision-records.md b/content/primers/architecture-decision-records.md new file mode 100644 index 0000000..3e3757a --- /dev/null +++ b/content/primers/architecture-decision-records.md @@ -0,0 +1,62 @@ +--- +title: Architecture decision records +--- + +# Architecture decision records + +An architecture decision record (ADR) is a short document that captures one significant decision, why it was made and what follows from it. A set of ADRs, kept in order next to the code, becomes the history of how a system got to be the shape it is. + +## Why bother + +Code shows what was built, but not why. Six months later nobody remembers why the team picked PostgreSQL over a document store, or why the gateway buffers messages rather than dropping them. Without that context, people either repeat old arguments or undo a decision without knowing what it protected. An ADR takes ten minutes to write and saves that. + +ADRs also help new joiners: reading the decision log is one of the quickest ways to understand a system. + +## What goes in one + +Michael Nygard's original format is still the most common, and it is deliberately small: + +| Section | What it says | +| --- | --- | +| Title | A short noun phrase, numbered: "ADR 7: Use MQTT for device telemetry" | +| Status | Proposed, accepted, deprecated or superseded (by which ADR) | +| Context | The forces at play: requirements, constraints, what has been tried | +| Decision | What we will do, in active voice: "We will..." | +| Consequences | What becomes easier, what becomes harder, what we now have to do | + +[[CAPTION:Table]] The sections of a Nygard-style ADR. + +Longer templates such as MADR add the options considered and the pros and cons of each. That is worth doing for a contested decision, and overkill for most. + +## Lifecycle + +An ADR is never edited once it is accepted, apart from its status. When a decision changes, write a new ADR that supersedes the old one and link the two. The log is append-only, so the reasoning at each point stays visible. + +```mermaid +stateDiagram-v2 + [*] --> Proposed + Proposed --> Accepted: agreed + Proposed --> Rejected: not agreed + Accepted --> Deprecated: no longer relevant + Accepted --> Superseded: replaced by a newer ADR +``` + +[[CAPTION:Figure]] The states an ADR moves through. + +## What deserves an ADR + +Write one when a decision is hard to reverse, affects more than one team or component, or was argued about. Typical examples are the choice of a database, a messaging protocol, an authentication scheme, a deployment platform, or a rule such as "all services publish events in CloudEvents format". Do not write one for every library upgrade. + +## Practical tips + +- Keep ADRs in the repository they describe, in a folder such as `docs/adr/`, numbered in order. Review them in pull requests like code. +- Write them when the decision is made, not afterwards. A proposed ADR is a good way to start the discussion. +- Keep each one to a page. If the context needs more, link to a design document or RFC. +- Tools such as `adr-tools` or `log4brains` help, but a text editor and a template are enough. + +## Where next + +- The [Templates](../templates/index.md) section has an ADR template ready to copy. +- The [Handbook](../handbook/index.md) covers how ADRs fit into architecture governance and review. +- Michael Nygard's 2011 post, [Documenting architecture decisions](https://cognitect.com/blog/2011/11/15/documenting-architecture-decisions), started the practice. +- [adr.github.io](https://adr.github.io/) collects templates and tools. diff --git a/content/primers/architecture.md b/content/primers/architecture.md new file mode 100644 index 0000000..1563a5a --- /dev/null +++ b/content/primers/architecture.md @@ -0,0 +1,20 @@ +--- +title: Architecture primers +--- + +# Architecture primers + +Software architecture is the set of decisions about a system that are expensive to change later. They cover how it is split into parts, how those parts talk, where data lives, and which qualities (performance, security, availability, cost) win when they conflict. These primers introduce the vocabulary and the common styles. + +- [Architecture decision records](architecture-decision-records.md): writing down why, not only what. +- [The C4 model](c4-model.md): diagrams at four levels of zoom. +- [Design principles](design-principles.md): SOLID, DRY, KISS and YAGNI. +- [Design patterns](design-patterns.md): the Gang of Four catalogue and how to use it. +- [Anti-patterns](anti-patterns.md): the shapes that code drifts into when nobody is looking. +- [Hexagonal architecture](hexagonal-architecture.md): ports, adapters and keeping the domain clean. +- [Domain-driven design](domain-driven-design.md): modelling the business and drawing boundaries. +- [Event-driven architecture](event-driven-architecture.md): components that react to things that happened. +- [Modular monolith or microservices](modular-monolith-vs-microservices.md): choosing how to deploy the parts. +- [Digital twins and device shadows](digital-twins-and-device-shadows.md): keeping a cloud copy of a device's state. + +The [Handbook](../handbook/index.md) goes deeper into each of these, with the trade-offs and the review process. The [Glossary](../glossary/index.md) defines the terms. diff --git a/content/primers/bootloaders.md b/content/primers/bootloaders.md new file mode 100644 index 0000000..f400e69 --- /dev/null +++ b/content/primers/bootloaders.md @@ -0,0 +1,60 @@ +--- +title: Bootloaders +--- + +# Bootloaders + +A bootloader is the first code that runs when a microcontroller starts. Its job is to check that the application firmware is present and trustworthy, start it, and, when asked, install a new version. Because the bootloader itself is rarely or never updated, it must be small, simple and correct the first time. + +## What happens at power-on + +```mermaid +flowchart TD + R[Reset] --> ROM[ROM boot code in the chip] + ROM --> BL[Bootloader in flash] + BL --> U{Update waiting?} + U -- yes --> I[Verify and install new image] + U -- no --> V{Current image valid and signed?} + I --> V + V -- yes --> APP[Jump to the application] + V -- no --> F[Fall back to previous image or recovery mode] +``` + +[[CAPTION:Figure]] A typical boot sequence. + +Many microcontrollers have a ROM bootloader from the vendor that can load firmware over UART or USB in the factory. The bootloader discussed here is your own, in flash, which runs on every start. + +## Memory layouts for updates + +| Layout | How it works | Trade-off | +| --- | --- | --- | +| Single slot | The bootloader receives the new image and overwrites the old one | Least flash; a failed write leaves the device without an application, so it needs a recovery path | +| Dual slot (A/B) with swap | The new image is written to a second slot; the bootloader swaps the slots and can swap back if the new image fails | Needs twice the space, or external flash for the second slot; safest | +| Dual slot, execute in place | Both slots can run; the bootloader picks the newer valid one | No copying; images must be built for their slot address | + +[[CAPTION:Table]] Common flash layouts. + +## Secure boot + +Secure boot means the device runs only firmware signed by the manufacturer: + +- The image is signed with a private key held securely by the build system (an HSM or a signing service). +- The bootloader holds the matching public key, or a hash of it, in memory that cannot be changed (one-time-programmable fuses or protected flash). +- At every start, the bootloader checks the signature (ECDSA P-256 and Ed25519 are common) before running the image. +- The bootloader itself is protected by the chip's own root of trust where the hardware supports it, giving a chain of trust from silicon to application. + +Add **anti-rollback** protection with a security counter in the image, so an attacker cannot install an older version with a known vulnerability. + +## Good habits + +- Keep the bootloader small and its features few. Every feature is code you cannot easily fix. +- Mark a new image as "pending" and confirm it only after the application has started and connected successfully. If it never confirms, revert. This test-then-confirm step turns a bad update into a short outage instead of a dead device. +- Protect against power loss at every step of an update; assume the battery can fail mid-write. +- Enable the [watchdog](watchdog-timers.md) in the bootloader too. +- Use an established open-source bootloader rather than writing your own: MCUboot (used by Zephyr, Mbed and others) supports signing, encryption, A/B slots and rollback protection. + +## Where next + +- [Firmware over the air](firmware-over-the-air.md) covers getting the image to the device. +- [PKI and certificates](pki-and-certificates.md) and [cryptography basics](cryptography-basics.md) explain the signatures. +- The [MCUboot documentation](https://docs.mcuboot.com/) describes a well-tested design. diff --git a/content/primers/c4-model.md b/content/primers/c4-model.md new file mode 100644 index 0000000..4f75bf7 --- /dev/null +++ b/content/primers/c4-model.md @@ -0,0 +1,62 @@ +--- +title: The C4 model +--- + +# The C4 model + +The C4 model, created by Simon Brown, is a way of drawing software architecture as a set of maps at different levels of zoom. Each level answers a different question for a different reader, and each one is drawn with plain boxes and labelled arrows rather than a special notation. + +## The four levels + +| Level | Shows | Typical reader | +| --- | --- | --- | +| 1. System context | The system as one box, the people who use it and the other systems it talks to | Everyone, including non-technical stakeholders | +| 2. Container | The separately deployable or runnable parts: web app, API, database, message broker, device firmware | Architects, developers, operations | +| 3. Component | The main building blocks inside one container and how they interact | Developers working on that container | +| 4. Code | Classes, functions or tables inside one component | Rarely drawn by hand; generate it if you need it | + +[[CAPTION:Table]] The four levels of the C4 model. + +"Container" here means anything that runs or holds data on its own. It has nothing to do with Docker, which confuses people at first. + +Most teams find the first two levels give most of the value. Component diagrams are useful for a complex container. Code diagrams go out of date as soon as they are drawn. + +## A context diagram + +```plantuml +@startuml +!include +Person(user, "Field engineer", "Installs and checks devices") +System(platform, "Device platform", "Collects telemetry and manages devices") +System_Ext(idp, "Identity provider", "Signs users in") +System_Ext(devices, "Devices", "Sensors in the field") +Rel(user, platform, "Uses", "HTTPS") +Rel(devices, platform, "Sends telemetry", "MQTT over TLS") +Rel(platform, idp, "Authenticates users with", "OpenID Connect") +@enduml +``` + +[[CAPTION:Figure]] A level 1 system context diagram drawn with C4-PlantUML. + +## Rules that make the diagrams work + +- Every box has a name, a type and a one-line description of what it does. +- Every arrow is one-way and labelled with what flows and, at container level, how (protocol or technology). +- Every diagram has a title and a key if it uses colour or shape to mean something. +- One diagram per question. If a diagram needs a scroll bar, split it. + +## Supporting diagrams + +C4 also allows a few extra views when they help. A system landscape shows all the systems in an organisation; a dynamic diagram shows the sequence for one use case; a deployment diagram shows which containers run on which infrastructure. + +## Tools + +- **C4-PlantUML** adds C4 shapes to PlantUML. PlantUML's standard library includes it, so `!include ` works without downloading anything. +- **Structurizr** keeps one model in a text file (the Structurizr DSL) and draws every view from it, so the levels never disagree. +- **Mermaid** has experimental C4 support, good enough for simple context diagrams. + +## Where next + +- [PlantUML and Mermaid](plantuml-and-mermaid.md) introduces diagrams as code. +- The [Handbook](../handbook/index.md) shows how C4 views fit into an architecture description, and the [Templates](../templates/index.md) section has starter diagrams. +- [c4model.com](https://c4model.com/) is the official guide. diff --git a/content/primers/cap-and-pacelc.md b/content/primers/cap-and-pacelc.md new file mode 100644 index 0000000..ae8aa6e --- /dev/null +++ b/content/primers/cap-and-pacelc.md @@ -0,0 +1,58 @@ +--- +title: CAP and PACELC +--- + +# CAP and PACELC + +The CAP theorem and its extension, PACELC, describe the trade-off every replicated data store makes. They are often quoted loosely, so it is worth knowing what they actually say. + +## CAP + +Eric Brewer stated it as a conjecture in 2000, and Gilbert and Lynch proved it in 2002. A system that keeps copies of data on several nodes can offer at most two of these three at once: + +- **Consistency (C)**: every read sees the most recent write, as if there were one copy. (This is linearisability, which is stricter than the C in ACID.) +- **Availability (A)**: every request to a working node gets a response. +- **Partition tolerance (P)**: the system keeps working when the network splits the nodes into groups that cannot reach each other. + +Networks do partition, so P is not optional. The real choice is what to do during a partition: + +- **CP**: refuse some requests (or time out) so that nobody reads stale data. Examples: etcd, ZooKeeper, a relational database with synchronous replication. +- **AP**: keep answering on both sides of the split and reconcile afterwards. Examples: Cassandra and DynamoDB in their default settings, DNS. + +```mermaid +flowchart LR + W[Write x=2 on node A] -.-> X{{Network partition}} + X -.-> R[Read x on node B] + R --> CP[CP: refuse or wait] + R --> AP[AP: answer x=1, fix later] +``` + +[[CAPTION:Figure]] During a partition, a read on the far side must either wait or risk being stale. + +## PACELC + +CAP only talks about the partition case, which is rare. Daniel Abadi's PACELC (2012) adds the normal case: + +> If there is a **P**artition, choose **A**vailability or **C**onsistency; **E**lse, choose **L**atency or **C**onsistency. + +Even with a healthy network, keeping replicas in step means waiting for other nodes before replying. Systems that answer from the nearest copy are faster but may return stale data. + +| System (default settings) | During a partition | Normally | Class | +| --- | --- | --- | --- | +| Cassandra, DynamoDB | Availability | Latency | PA/EL | +| MongoDB | Consistency | Latency | PC/EL | +| Spanner, etcd | Consistency | Consistency | PC/EC | + +[[CAPTION:Table]] Where some well-known stores sit. Most can be tuned per query. + +## Using this in practice + +- Decide per piece of data, not per system. An account balance needs consistency; a dashboard counter can be a few seconds stale. +- Read the store's documentation for its actual guarantees and tuning options (quorum reads and writes, read-your-writes sessions). +- If you choose availability, plan how conflicts are resolved: last writer wins, merge rules, or [CRDTs](consistency-models.md). +- IoT devices behave like the AP side by nature: they work offline and sync later. [Device shadows](digital-twins-and-device-shadows.md) are a way of reconciling. + +## Where next + +- [Consistency models](consistency-models.md) covers the options between "strong" and "eventual". +- Martin Kleppmann's *Designing Data-Intensive Applications* is the best single book on the subject, and his article [Please stop calling databases CP or AP](https://martin.kleppmann.com/2015/05/11/please-stop-calling-databases-cp-or-ap.html) explains the limits of CAP. diff --git a/content/primers/cbor.md b/content/primers/cbor.md new file mode 100644 index 0000000..710292b --- /dev/null +++ b/content/primers/cbor.md @@ -0,0 +1,68 @@ +--- +title: CBOR and compact binary formats +--- + +# CBOR and compact binary formats + +On a battery-powered device sending data over a narrow radio link, every byte costs energy and airtime. JSON spends many of those bytes on field names, quotes and digits written as text. Compact binary formats carry the same data in far fewer bytes, and are cheaper to parse on a small microcontroller. + +## CBOR + +CBOR (Concise Binary Object Representation, RFC 8949) is the IETF's binary format for constrained environments. It has the same data model as JSON (maps, arrays, strings, numbers, booleans, null). It adds byte strings, tags for extra types such as dates, and separate integer and floating-point numbers. It is self-describing: a reader needs no schema to decode a message. + +Each item starts with one byte holding a 3-bit major type and a 5-bit value or length. Small integers fit in that byte. + +| Major type | Meaning | +| --- | --- | +| 0 | Unsigned integer | +| 1 | Negative integer | +| 2 | Byte string | +| 3 | Text string (UTF-8) | +| 4 | Array | +| 5 | Map | +| 6 | Tag (adds meaning, such as a timestamp) | +| 7 | Floats and simple values (`true`, `false`, `null`) | + +[[CAPTION:Table]] The eight CBOR major types. + +A worked example: `{"t": 21, "b": 71}` is 15 bytes as compact JSON and 8 bytes as CBOR: `A2` (a map of two pairs), `61 74` ("t"), `15` (21), `61 62` ("b"), `18 47` (71). Using integer keys instead of text keys shrinks it further, which is what most constrained protocols do. + +CBOR is used by CoAP, [LwM2M](lwm2m.md) (as the SenML CBOR format), COSE (CBOR Object Signing and Encryption), WebAuthn and the FIDO standards. + +## CDDL + +CBOR does not need a schema, but a protocol still needs a specification. CDDL (Concise Data Definition Language, RFC 8610) describes the expected structure in a compact grammar, and tools can validate messages against it: + +```cddl +reading = { + 1 => uint, ; device-local timestamp + 2 => float, ; temperature in °C + ? 3 => uint, ; battery percentage, optional +} +``` + +[[CAPTION:Figure]] A CDDL definition with integer keys. + +## Other compact formats + +| Format | Schema | Self-describing | Notes | +| --- | --- | --- | --- | +| CBOR | Optional (CDDL) | Yes | IETF standard; small decoders such as TinyCBOR, QCBOR, zcbor | +| Protocol Buffers | Required | No | Smallest messages; needs generated code (`nanopb` on microcontrollers) | +| MessagePack | No | Yes | Very similar to CBOR; popular in scripting languages, not an IETF standard | +| ASN.1 (PER, UPER) | Required | No | Very compact; used in telecoms and some security standards; heavy tooling | + +[[CAPTION:Table]] Compact binary formats compared. + +## Choosing + +- If you control both ends and can generate code, Protocol Buffers gives the smallest messages and the best schema evolution. +- If you need an open standard, interoperability with CoAP or LwM2M, or messages a gateway can decode without a schema, choose CBOR. +- Whatever the format, version your payloads and keep the schema or CDDL in version control. +- Use the same data on both sides of a gateway: decoding CBOR into JSON for the cloud is straightforward because the data models match. + +## Where next + +- [Data formats](data-formats.md) covers JSON, YAML and Protocol Buffers. +- [CoAP](coap.md) and [cellular LPWAN](cellular-lpwan.md) explain why size matters so much. +- [cbor.io](https://cbor.io/) lists implementations for most languages. diff --git a/content/primers/cellular-lpwan.md b/content/primers/cellular-lpwan.md new file mode 100644 index 0000000..affab2a --- /dev/null +++ b/content/primers/cellular-lpwan.md @@ -0,0 +1,60 @@ +--- +title: Cellular LPWAN +--- + +# Cellular LPWAN: NB-IoT and LTE-M + +A low-power wide-area network (LPWAN) carries small amounts of data over long distances from devices that must run for years on a battery. NB-IoT and LTE-M are the two LPWAN technologies built into the mobile network standards by 3GPP (the body behind 4G and 5G). Because they run on operators' licensed spectrum and existing masts, they need no private network infrastructure, and both are part of 5G's support for massive numbers of devices. + +The unlicensed alternatives, LoRaWAN and Sigfox, need their own gateways or a provider that runs them, and offer lower data rates. + +## NB-IoT and LTE-M compared + +| | NB-IoT (Cat-NB1, Cat-NB2) | LTE-M (Cat-M1, Cat-M2) | +| --- | --- | --- | +| 3GPP release | 13 (NB1), 14 (NB2) | 13 (M1), 14 (M2) | +| Bandwidth | 180 kHz | 1.4 MHz (M1), 5 MHz (M2) | +| Peak data rate | Tens of kbit/s (NB1) to about 130 kbit/s (NB2) | About 300 kbit/s to 1 Mbit/s (M1), up to about 4 Mbit/s (M2) | +| Latency | Seconds | 10 to 15 ms in good conditions | +| Mobility | Cell reselection when idle; no handover | Full handover while connected | +| Voice | No | VoLTE possible | +| Coverage | Best: designed for basements and cellars | Good, slightly less than NB-IoT | +| Roaming | Patchy, depends on operator agreements | Widely available | +| Good for | Static devices sending small, infrequent messages | Moving devices, larger downloads such as firmware, anything that needs a quick reply | + +[[CAPTION:Table]] The two cellular LPWAN technologies. Real-world rates are well below the peaks. + +Many modules support both, and choose according to what the network offers. Where an operator supports both, LTE-M is usually the safer choice unless deep indoor coverage is critical. + +## Saving power + +Two features let a device stay registered on the network while its radio is almost entirely off. + +- **PSM (Power Saving Mode)**: after sending, the device stays reachable for a short active time (timer T3324), then goes into a deep sleep in which it cannot be reached at all. It wakes when it has something to send, or when the periodic update timer (T3412) expires. Sleep current can be a few microamps. Best for devices that only send. +- **eDRX (extended Discontinuous Reception)**: the device sleeps and wakes briefly at a fixed interval (from seconds up to hours) to check for incoming data. It can receive downlink messages with a known delay. Best for devices that must sometimes be commanded. + +Devices request these timers with standard AT commands (`AT+CPSMS`, `AT+CEDRXS`), but the network decides what it actually grants. Always read back the values the network accepted. + +## Coverage and signal + +- **RSRP** (Reference Signal Received Power): signal strength. Around −100 dBm is good; below −120 dBm is weak. +- **RSRQ** and **SINR**: signal quality and interference. +- **Coverage enhancement (CE) levels**: in poor coverage, the network repeats transmissions many times so they still get through. Data still arrives, but each message takes far longer and uses far more energy. A device that is always at the deepest CE level will have a much shorter battery life than the datasheet suggests. + +## SIMs + +- **Plastic and MFF2 SIMs**: a removable card or a SIM chip soldered to the board. +- **eSIM (eUICC)**: the SIM holds several operator profiles that can be changed remotely, so devices can switch operator without a site visit. GSMA's SGP.32 specification defines remote provisioning designed for IoT devices. +- **iSIM**: the SIM function built into the modem chip itself, saving space and cost. + +## Designing for cellular LPWAN + +- Send as few and as small messages as possible. Use compact payloads ([CBOR](cbor.md)) and UDP-based protocols ([CoAP](coap.md), [LwM2M](lwm2m.md)) where you can; a TLS-over-TCP handshake on NB-IoT can cost more energy than the data itself. +- Use DTLS Connection ID or session resumption so a wake-up does not need a full handshake. +- Measure current with a real SIM on a real network, in the worst coverage you expect. +- Plan for network changes: 2G and 3G are being switched off in many countries, including the UK, so LTE-M and NB-IoT are the long-term options for new designs. + +## Where next + +- [Firmware over the air](firmware-over-the-air.md) covers updates over slow links. +- The [Guides](../guides/index.md) section covers modem integration in practice. diff --git a/content/primers/cicd-pipelines.md b/content/primers/cicd-pipelines.md new file mode 100644 index 0000000..484021d --- /dev/null +++ b/content/primers/cicd-pipelines.md @@ -0,0 +1,68 @@ +--- +title: CI/CD pipelines +--- + +# CI/CD pipelines + +CI/CD stands for continuous integration and continuous delivery (or deployment). Together they mean that every change is built, tested and made ready to release automatically, so releasing becomes routine rather than an event. + +## The three practices + +- **Continuous integration (CI)**: everyone merges small changes into the main branch at least daily, and every merge triggers an automated build and test. Problems show up within minutes, while the change is still fresh in the author's mind. +- **Continuous delivery**: every change that passes the pipeline produces a release candidate that *could* go to production at the press of a button. +- **Continuous deployment**: every change that passes goes to production automatically, with no manual step. + +Many teams do continuous delivery for products that need a release decision (firmware, regulated software) and continuous deployment for web services. + +## A typical pipeline + +```mermaid +flowchart LR + A[Commit or pull request] --> B[Build] + B --> C[Unit tests and static analysis] + C --> D[Package artefact] + D --> E[Integration tests] + E --> F[Deploy to staging] + F --> G{Release decision} + G --> H[Deploy to production] +``` + +[[CAPTION:Figure]] The stages of a typical pipeline. + +- **Build once**: produce one artefact (a container image, a firmware image, a package) and promote that same artefact through every environment. Rebuilding for production means testing something different from what you ship. +- **Fail fast**: run the quickest checks first so most failures report in a few minutes. +- **Keep the main branch releasable**: a red build is the team's top priority. + +## What belongs in the pipeline + +- Compilation and unit tests. +- [Static analysis](static-code-analysis.md), formatting and linting. +- Security checks: dependency vulnerability scanning, secret scanning, container image scanning, a software bill of materials (SBOM). +- Integration, contract and end-to-end tests, as far as they stay reliable. Flaky tests destroy trust in the pipeline, so fix or remove them. +- For embedded products, [hardware-in-the-loop tests](hardware-in-the-loop-testing.md) on real boards, and image signing. +- Deployment, with automatic rollback or a quick manual one. + +## Deployment strategies + +| Strategy | How it works | Good for | +| --- | --- | --- | +| Rolling | Replace instances a few at a time | Stateless services | +| Blue-green | Run the new version next to the old and switch traffic at once | Fast rollback | +| Canary | Send a small share of traffic (or devices) to the new version first, then widen | Catching problems with real use; firmware rollouts | +| Feature flags | Deploy code switched off; turn it on separately | Separating deployment from release | + +[[CAPTION:Table]] Common ways to release a new version. + +## Tools + +GitHub Actions, GitLab CI, Jenkins and Azure Pipelines are general CI servers; Argo CD and Flux deploy to Kubernetes from Git (GitOps). Pipeline definitions are code: keep them in the repository and review them like any other change. + +## Measuring it + +The DORA metrics are a good check on whether delivery is improving: deployment frequency, lead time for changes, change failure rate and time to restore service. + +## Where next + +- [Infrastructure as code](infrastructure-as-code.md) applies the same discipline to environments. +- [Git practices](git-best-practices.md) covers the branching that CI depends on. +- The [Guides](../guides/index.md) section covers CI setup in more detail. diff --git a/content/primers/cloud.md b/content/primers/cloud.md new file mode 100644 index 0000000..66cccdc --- /dev/null +++ b/content/primers/cloud.md @@ -0,0 +1,30 @@ +--- +title: Cloud primers +--- + +# Cloud primers + +Cloud computing means renting computing, storage and managed services from a provider, paying for what you use, and creating or removing resources through an API rather than buying hardware. The trade is less control and some lock-in in exchange for speed and not having to run the infrastructure yourself. + +## Service models + +| Model | You manage | The provider manages | Examples | +| --- | --- | --- | --- | +| IaaS (Infrastructure as a Service) | Operating system, runtime, application, data | Hardware, network, virtualisation | Virtual machines, block storage | +| PaaS (Platform as a Service) | Application and data | Everything underneath | Managed databases, app platforms, Kubernetes services | +| FaaS (Functions as a Service) | Functions and data | Everything else, including scaling | AWS Lambda, Cloud Run functions, Azure Functions | +| SaaS (Software as a Service) | Configuration and your data | The whole application | Email, CRM, source hosting | + +[[CAPTION:Table]] Cloud service models, from most to least control. + +The further down the table, the less you run yourself and the less you can change. Most systems mix several models. + +## The primers + +- [CI/CD pipelines](cicd-pipelines.md): building, testing and deploying automatically. +- [Infrastructure as code](infrastructure-as-code.md): describing infrastructure in version-controlled files. +- [Serverless computing](serverless-computing.md): functions and managed services that scale to zero. +- [Event streaming with Kafka](kafka-event-streaming.md): durable, replayable event logs. +- [IoT on Google Cloud](iot-on-google-cloud.md): building a device platform from general cloud services. + +The [Handbook](../handbook/index.md) covers cloud architecture decisions, and the [Tech Radar](../radar/index.md) shows which cloud tools are recommended. diff --git a/content/primers/coap.md b/content/primers/coap.md new file mode 100644 index 0000000..a211d2b --- /dev/null +++ b/content/primers/coap.md @@ -0,0 +1,63 @@ +--- +title: CoAP +--- + +# CoAP + +CoAP (the Constrained Application Protocol, RFC 7252) brings the web's request-response model to devices too small or too power-limited for HTTP. It has the same ideas as [REST](rest-apis.md) (resources with URIs, `GET`, `POST`, `PUT`, `DELETE`, response codes and content types) but runs over UDP with a 4-byte fixed header. A complete request often fits in a single small packet. + +## Why UDP + +TCP needs a handshake before any data flows and keeps state for every connection, which costs time, energy and memory. On a lossy radio link or a cellular modem that sleeps most of the time, that overhead dominates. CoAP over UDP can send one packet, get one reply, and go back to sleep. It adds its own lightweight reliability where needed. + +## Messages + +| Type | Meaning | +| --- | --- | +| CON (confirmable) | Must be acknowledged; resent with exponential backoff until it is | +| NON (non-confirmable) | Fire and forget, for frequent readings where loss is acceptable | +| ACK | Acknowledges a CON message, and can carry the response with it ("piggybacked") | +| RST (reset) | The receiver could not process the message | + +[[CAPTION:Table]] The four CoAP message types. + +```mermaid +sequenceDiagram + participant C as Device (client) + participant S as Server + C->>S: CON GET /config (message ID 0x7d34, token 0x71) + S-->>C: ACK 2.05 Content, payload {"interval": 300} +``` + +[[CAPTION:Figure]] A confirmable request with a piggybacked response. + +Response codes follow HTTP's classes in a compact form: 2.05 Content, 2.04 Changed, 4.04 Not Found, 5.03 Service Unavailable. + +## Features + +- **Observe** (RFC 7641): a client registers interest in a resource and the server sends a notification whenever it changes. This gives publish-subscribe behaviour without a broker. +- **Block-wise transfer** (RFC 7959): splits large payloads, such as firmware images, into blocks that fit in small packets. +- **Discovery**: `GET /.well-known/core` lists the resources a device offers. +- **Proxies**: CoAP maps cleanly to HTTP, so a gateway can translate between the two. +- **Content formats**: numeric codes for media types, with [CBOR](cbor.md) and SenML commonly used for payloads. + +## Security + +CoAP is secured with DTLS (TLS for datagrams), using pre-shared keys, raw public keys or certificates; see [TLS and DTLS](tls-and-dtls.md). DTLS 1.2 Connection ID (RFC 9146) lets a session survive the address changes that happen when a cellular device wakes from sleep, which avoids a full handshake every time. OSCORE (RFC 8613) protects the CoAP message itself, end to end, even through proxies. + +## CoAP or MQTT + +| | CoAP | [MQTT](mqtt.md) | +| --- | --- | --- | +| Model | Request-response, plus observe | Publish-subscribe through a broker | +| Transport | UDP (TCP and WebSockets also defined) | TCP | +| Best for | Sleepy, battery-powered devices on lossy or cellular links | Devices with steady connections and many-to-many messaging | +| Device management | Through [LwM2M](lwm2m.md) | Your own conventions | + +[[CAPTION:Table]] CoAP and MQTT compared. + +## Where next + +- [LwM2M](lwm2m.md) builds device management on top of CoAP. +- [Cellular LPWAN](cellular-lpwan.md) explains the networks CoAP suits best. +- Libraries include libcoap (C), Californium (Java), aiocoap (Python) and the CoAP support in Zephyr. diff --git a/content/primers/code-review.md b/content/primers/code-review.md new file mode 100644 index 0000000..5677303 --- /dev/null +++ b/content/primers/code-review.md @@ -0,0 +1,64 @@ +--- +title: Code review +--- + +# Code review + +Code review is the practice of having someone other than the author read a change before it is merged. It catches defects, but that is not its only or even its main value. It also spreads knowledge of the code across the team, keeps the codebase consistent, and helps people learn from each other. + +## What to look for + +In rough order of importance: + +1. **Does it do the right thing?** Does the change meet its purpose? Are edge cases and errors handled? +2. **Design**: does it fit the architecture? Is it in the right place? Is it simpler than it could be? +3. **Security**: is untrusted input validated? Are secrets kept out? See [secure coding practices](secure-coding-practices.md). +4. **Tests**: are there tests for the new behaviour, and do they test the right things? +5. **Readability**: will the next person understand it? Are names clear? Are comments explaining why, not what? +6. **Consistency** with the conventions of the codebase. + +Leave formatting, style and simple bug patterns to tools ([linters, formatters and static analysis](static-code-analysis.md)) in the [CI pipeline](cicd-pipelines.md), so that people review what only people can. + +## Making reviews work + +**For authors:** + +- Keep changes small. Review quality drops sharply once a change passes a few hundred lines. +- Explain what and why in the pull request description, and how you tested it. +- Review your own diff first. +- Separate refactoring from behaviour changes. + +**For reviewers:** + +- Respond quickly; waiting for review is one of the biggest delays in delivery. +- Be specific and kind. Comment on the code, not the person. Ask questions rather than make demands where you are not sure. +- Label the weight of each comment: a blocking problem, a suggestion, or a nit that the author can ignore. +- Approve when the change makes the code better, even if it is not how you would have written it. + +```mermaid +flowchart LR + A[Author opens pull request] --> CI[Automated checks] + CI --> R[Reviewer reads and comments] + R --> U[Author updates] + U --> R + R --> M[Approve and merge] +``` + +[[CAPTION:Figure]] The usual pull-request review loop. + +## Variations + +- **Pull-request review**: the usual asynchronous form on GitHub, GitLab or similar. +- **Pair programming**: continuous review while writing the code. +- **Walkthroughs and inspections**: the author presents to a group. Formal inspections (Fagan inspections) with checklists and roles are still used for safety-critical code. +- **Design review**: reviewing an approach before the code exists saves the most rework. Use an [ADR](architecture-decision-records.md) or RFC. + +## Measuring it + +Watch time to first review and time to merge. Do not count comments or defects found per reviewer: that turns review into a contest. + +## Where next + +- The [Templates](../templates/index.md) section has a code review checklist and a pull-request template. +- The [Guidelines](../guidelines/index.md) section sets the review rules used here. +- Google's [engineering practices guide to code review](https://google.github.io/eng-practices/review/) is a clear, practical reference. diff --git a/content/primers/consistency-models.md b/content/primers/consistency-models.md new file mode 100644 index 0000000..1af627f --- /dev/null +++ b/content/primers/consistency-models.md @@ -0,0 +1,55 @@ +--- +title: Consistency models +--- + +# Consistency models + +A consistency model is the contract a data store makes about what a reader can see after a write. Stronger models are easier to write code against and cost more in latency and availability (see [CAP and PACELC](cap-and-pacelc.md)). Weaker models are faster and keep working during failures, but push work onto the application. + +## From strongest to weakest + +| Model | Guarantee | Typical cost | +| --- | --- | --- | +| Linearisable (strong) | Every read sees the latest write, as if there were one copy | Coordination on every write; unavailable during partitions | +| Sequential | Everyone sees operations in the same order, though not necessarily in real time | Still needs coordination | +| Causal | If one write caused another, everyone sees them in that order; unrelated writes may appear in any order | Track dependencies | +| Read-your-writes | A client always sees its own earlier writes | Session affinity or version tracking | +| Monotonic reads | A client never sees data go backwards in time | Session affinity | +| Eventual | If writes stop, all replicas eventually agree | Cheapest; readers may see stale or conflicting data | + +[[CAPTION:Table]] Common consistency models. + +The middle models are often the useful ones. "Eventual consistency plus read-your-writes" is enough for most user-facing features: users see their own changes at once, and other people's changes shortly after. + +## Living with eventual consistency + +- Show that data may be stale ("last updated 30 seconds ago") rather than pretending it is live. +- Make writes [idempotent](idempotency.md) so retries and duplicates are harmless. +- Decide how to resolve conflicts when two replicas accept different writes. Options include last writer wins (simple, loses data), keeping both and asking a person, application-specific merge rules, or CRDTs. + +## CRDTs + +A conflict-free replicated data type (CRDT) is a data structure whose replicas can be changed independently. They always merge to the same result, whatever order the updates arrive in. No coordination is needed. + +Simple examples: + +- **G-Counter** (grow-only counter): each replica keeps its own count; the value is the sum; merging takes the maximum per replica. +- **PN-Counter**: two G-Counters, one for increments and one for decrements. +- **OR-Set** (observed-remove set): elements can be added and removed, with adds winning over concurrent removes. +- **LWW-Register**: a single value where the latest timestamp wins. + +```mermaid +flowchart LR + A["Replica A: {A:3, B:0}"] --> M["Merge: max per replica {A:3, B:2} = 5"] + B["Replica B: {A:1, B:2}"] --> M +``` + +[[CAPTION:Figure]] Merging a grow-only counter. + +CRDTs are used in collaborative editors, shopping carts, offline-first mobile apps and some databases (Riak, Redis Enterprise, Automerge and Yjs libraries). They suit devices that work offline and sync later. Their cost is extra metadata and the need to design the data model around them. + +## Where next + +- [The saga pattern](saga-pattern.md) handles consistency across services. +- The [Handbook](../handbook/index.md) covers data ownership and replication choices. +- [crdt.tech](https://crdt.tech/) collects papers and implementations. diff --git a/content/primers/cryptography-basics.md b/content/primers/cryptography-basics.md new file mode 100644 index 0000000..0c2fb85 --- /dev/null +++ b/content/primers/cryptography-basics.md @@ -0,0 +1,58 @@ +--- +title: Cryptography basics +--- + +# Cryptography basics + +Cryptography provides a small set of building blocks: keeping data secret (confidentiality), detecting changes (integrity), and proving who sent it (authenticity). Almost every security protocol is a combination of the primitives below. The first rule is never to invent your own: use well-reviewed algorithms through a well-maintained library. + +## Hash functions + +A cryptographic hash turns any input into a fixed-size digest. It is one-way (you cannot recover the input), and it is infeasible to find two inputs with the same digest. + +- **Use**: SHA-256 and SHA-384 (the SHA-2 family), SHA-3, BLAKE2 or BLAKE3. +- **Avoid**: MD5 and SHA-1, both broken for collision resistance. +- **Uses**: file and firmware integrity checks, digital signatures (which sign a hash), content addressing as in Git. +- **Not for passwords on their own**: a plain hash is far too fast to resist guessing. See key derivation below. + +## Symmetric encryption + +One shared key both encrypts and decrypts. It is fast and suits bulk data. + +- **AES** (Advanced Encryption Standard) with 128- or 256-bit keys is the standard, and most microcontrollers and processors have hardware acceleration for it. **ChaCha20** is a fast alternative in software. +- Always use an authenticated mode, which encrypts and protects integrity together: **AES-GCM**, **AES-CCM** (common in constrained devices) or **ChaCha20-Poly1305**. Never use ECB mode, and never reuse a nonce with the same key in GCM: doing so breaks both secrecy and integrity. +- The hard part is sharing and protecting the key. + +## Message authentication codes + +A MAC (message authentication code) is a short tag computed from a message and a secret key. Anyone with the key can check that the message has not changed and came from another key holder. **HMAC-SHA256** is the common choice; authenticated encryption modes include a MAC already. Compare tags in constant time to avoid leaking information through timing. + +## Public-key cryptography + +Each party has a key pair: a private key kept secret and a public key shared freely. + +- **Digital signatures**: sign with the private key; anyone can verify with the public key. Used for firmware signing, certificates and software packages. Algorithms: **ECDSA** with P-256, **Ed25519**, and **RSA** (3072 bits or more for new keys). +- **Key exchange**: two parties agree a shared secret over an open channel. **ECDH**, especially **X25519**, is the modern choice. The shared secret then keys a symmetric cipher. +- Public-key operations are much slower than symmetric ones, so protocols such as [TLS](tls-and-dtls.md) use them only to authenticate and agree keys. +- The hard part is trusting that a public key really belongs to who it claims; that is the job of [PKI](pki-and-certificates.md). + +## Key derivation + +A key derivation function (KDF) turns one secret into the keys a protocol needs. + +- **HKDF** derives several keys from a shared secret, as in TLS 1.3. +- **Password hashing** needs a deliberately slow, memory-hard function: **Argon2id** first, otherwise **scrypt**, **bcrypt** or **PBKDF2** with a high iteration count. Always use a unique salt per password. + +## Random numbers + +Keys, nonces and salts must come from a cryptographically secure random number generator. On a computer, use the operating system's (`/dev/urandom`, `getrandom()`, Python's `secrets`). On a microcontroller, use the hardware true random number generator (TRNG) feeding a DRBG (deterministic random bit generator). `rand()` is never acceptable. Weak randomness at first boot, before enough entropy has been gathered, is a known source of duplicate keys in device fleets. + +## Choosing for constrained devices + +Small devices often use AES-CCM, ECDSA or Ed25519, X25519 and SHA-256. Hardware accelerators support them, as do libraries such as Mbed TLS, wolfSSL and the PSA Crypto API. NIST's Ascon (lightweight cryptography standard, 2025) targets the smallest devices. + +## Where next + +- [PKI and certificates](pki-and-certificates.md) and [TLS and DTLS](tls-and-dtls.md) show the primitives in use. +- [Post-quantum cryptography](post-quantum-cryptography.md) covers what will replace today's public-key algorithms. +- The [Standards](../standards/index.md) section lists the algorithms approved here. diff --git a/content/primers/data-formats.md b/content/primers/data-formats.md new file mode 100644 index 0000000..ec861eb --- /dev/null +++ b/content/primers/data-formats.md @@ -0,0 +1,77 @@ +--- +title: Data formats +--- + +# Data formats + +Every interface and configuration file needs a format. The three you meet most are JSON, YAML and Protocol Buffers. They solve different problems, and choosing the wrong one makes life harder for years. + +## JSON + +JSON (JavaScript Object Notation, RFC 8259) is the default format for web APIs. It has objects, arrays, strings, numbers, booleans and `null`, and nothing else. + +```json +{"id": "dev-042", "online": true, "battery": 71, "tags": ["roof", "north"]} +``` + +[[CAPTION:Figure]] A JSON object. + +- Readable, supported everywhere, easy to debug. +- No comments, no date type, no binary data (use base64) and only one number type. Large integers can lose precision in JavaScript. +- JSON Schema describes and validates JSON documents, and OpenAPI uses it. +- Verbose on the wire for small devices; see [CBOR](cbor.md) for a binary equivalent. + +## YAML + +YAML is a superset of JSON aimed at people writing configuration by hand: indentation instead of braces, comments, multi-line strings and anchors for reuse. + +```yaml +device: + id: dev-042 + online: true + tags: [roof, north] # comments are allowed +``` + +[[CAPTION:Figure]] The same data in YAML. + +- Good for configuration files: CI pipelines, Kubernetes, Docker Compose, this site's own `py-doc.yml`. +- Indentation errors change the meaning silently. Unquoted values can surprise: in YAML 1.1 parsers `no` and `on` become booleans and `1.10` becomes a number. Quote strings when in doubt. +- Do not use it for data exchanged between programs; JSON is simpler and safer. + +## Protocol Buffers + +Protocol Buffers (protobuf), from Google, is a binary format defined by a schema file. A compiler generates code for each language. + +```protobuf +syntax = "proto3"; +message Device { + string id = 1; + bool online = 2; + uint32 battery = 3; + repeated string tags = 4; +} +``` + +[[CAPTION:Figure]] A protobuf schema. The numbers are field tags, which go on the wire instead of names. + +- Small and fast; typically a fraction of the size of the same JSON. +- Not self-describing: you need the schema to read a message. +- Schema evolution is built in: add fields with new numbers, never reuse or renumber old ones, and old and new code keep working together. +- The basis of gRPC. `nanopb` brings it to microcontrollers. + +## Choosing + +| Need | Good choice | +| --- | --- | +| Public web API, easy debugging | JSON | +| Hand-written configuration | YAML (or TOML for simpler files) | +| Fast internal RPC with a fixed schema | Protocol Buffers | +| Constrained devices, no schema compiler | CBOR | +| Large analytical data sets | Columnar formats such as Parquet | + +[[CAPTION:Table]] Matching the format to the job. + +## Where next + +- [CBOR and compact binary formats](cbor.md) covers the device side. +- [OpenAPI and AsyncAPI](openapi-and-asyncapi.md) shows how schemas become contracts. diff --git a/content/primers/design-patterns.md b/content/primers/design-patterns.md new file mode 100644 index 0000000..495ddce --- /dev/null +++ b/content/primers/design-patterns.md @@ -0,0 +1,97 @@ +--- +title: Design patterns +--- + +# Design patterns + +A design pattern is a named, reusable solution to a problem that keeps coming up in software design. The name is half the value: saying "put an adapter in front of it" communicates a whole design in four words. + +The best-known catalogue is *Design Patterns: Elements of Reusable Object-Oriented Software* (1994) by Gamma, Helm, Johnson and Vlissides, the "Gang of Four" (GoF). It describes 23 patterns in three groups. + +## Creational patterns + +These control how objects are made, so the code that uses an object does not depend on exactly which class it gets. + +| Pattern | Use it when | +| --- | --- | +| Factory method | A class needs to create objects but subclasses should decide which kind | +| Abstract factory | You need families of related objects that must match (for example, one per hardware board) | +| Builder | An object has many optional parts and a constructor would be unreadable | +| Prototype | Copying an existing object is cheaper or simpler than building one | +| Singleton | There must be exactly one instance. Use sparingly: it is global state with a nicer name | + +[[CAPTION:Table]] The creational patterns. + +## Structural patterns + +These are about composing classes and objects into larger structures. + +| Pattern | Use it when | +| --- | --- | +| Adapter | You need to use a class whose interface does not match what your code expects | +| Bridge | Both an abstraction and its implementation need to vary independently | +| Composite | Clients should treat a single item and a group of items the same way (trees) | +| Decorator | You want to add behaviour to an object without subclassing it | +| Facade | A subsystem is complicated and most callers need a simple front door | +| Flyweight | You have very many similar objects and need to share their common state | +| Proxy | Access to an object needs control: lazy loading, caching, remote calls, permissions | + +[[CAPTION:Table]] The structural patterns. + +## Behavioural patterns + +These are about how objects share responsibility and communicate. + +| Pattern | Use it when | +| --- | --- | +| Chain of responsibility | Several handlers might deal with a request and the sender should not care which | +| Command | You want to treat a request as an object: queue it, log it, undo it | +| Interpreter | You have a small language to evaluate | +| Iterator | Clients need to walk a collection without knowing its structure | +| Mediator | Many objects talk to each other and the wiring has become a tangle | +| Memento | You need to save and restore an object's state without exposing its internals | +| Observer | Other objects must be told when something changes (the basis of event handling) | +| State | An object's behaviour depends on its state and the `if` statements are spreading | +| Strategy | You need to swap one algorithm for another at run time | +| Template method | The steps of an algorithm are fixed but some steps vary | +| Visitor | You need to add operations to a stable set of classes without changing them | + +[[CAPTION:Table]] The behavioural patterns. + +## A small example + +The strategy pattern in Python, choosing how to compress a log file: + +```python +from typing import Protocol +import gzip, lzma + +class Compressor(Protocol): + def compress(self, data: bytes) -> bytes: ... + +class Gzip: + def compress(self, data: bytes) -> bytes: + return gzip.compress(data) + +class Xz: + def compress(self, data: bytes) -> bytes: + return lzma.compress(data) + +def archive(data: bytes, compressor: Compressor) -> bytes: + return compressor.compress(data) +``` + +[[CAPTION:Figure]] The strategy pattern: the caller picks the algorithm. + +## Using patterns well + +- Start from the problem, not the pattern. If you cannot name the force a pattern resolves, you probably do not need it. +- Modern languages absorb some patterns. First-class functions make many strategies and commands a plain function; iterators are built into most languages. +- Patterns in embedded C look different (function pointer tables, static allocation) but the ideas carry over. +- Overuse is its own [anti-pattern](anti-patterns.md): an abstract factory for one product is just extra code. + +## Where next + +- [Design principles](design-principles.md) explains the ideas the patterns rest on. +- Architectural patterns work at a larger scale: see [hexagonal architecture](hexagonal-architecture.md) and [event-driven architecture](event-driven-architecture.md). +- [refactoring.guru](https://refactoring.guru/design-patterns) has a clear illustrated description of every GoF pattern. diff --git a/content/primers/design-principles.md b/content/primers/design-principles.md new file mode 100644 index 0000000..b6cec99 --- /dev/null +++ b/content/primers/design-principles.md @@ -0,0 +1,46 @@ +--- +title: Design principles +--- + +# Design principles + +A handful of principles turn up in almost every conversation about code quality. None of them is a law. Each one is a rule of thumb that pushes against a common way for code to go wrong, and each one can be overdone. + +## SOLID + +Robert C. Martin gathered five principles for object-oriented design under the SOLID acronym. They apply just as well to modules and services. + +| Principle | In one line | What it prevents | +| --- | --- | --- | +| Single responsibility | A module should have one reason to change | Classes that change for unrelated business reasons | +| Open/closed | Extend behaviour by adding code, not by editing working code | Every new case touching the same `switch` statement | +| Liskov substitution | A subtype must work wherever its parent type works | Subclasses that throw "not supported" or quietly change the meaning | +| Interface segregation | Many small interfaces beat one large one | Clients depending on methods they never call | +| Dependency inversion | Depend on abstractions; let the details depend on them too | Business logic that imports the database driver | + +[[CAPTION:Table]] The SOLID principles. + +Dependency inversion is the one with the biggest effect on architecture: it is what makes [hexagonal architecture](hexagonal-architecture.md) and easy unit testing possible. + +## DRY, KISS and YAGNI + +- **DRY (Don't Repeat Yourself)**: every piece of knowledge should have one authoritative home. It is about knowledge, not text. Two functions that look alike but change for different reasons are not duplication, and merging them couples things that should stay apart. +- **KISS (Keep It Simple)**: choose the simplest design that does the job. Clever code costs more to read, test and debug than it saves. +- **YAGNI (You Aren't Gonna Need It)**: do not build for requirements you only imagine. Speculative flexibility has a cost today and usually guesses the future wrong. + +## Other principles worth knowing + +- **Separation of concerns**: keep different kinds of work (presentation, business rules, storage) in different places. +- **High cohesion, low coupling**: things that change together live together; things that change apart talk through narrow interfaces. +- **Composition over inheritance**: build behaviour by combining small objects rather than deep class trees. +- **Least astonishment**: an interface should do what its name suggests. +- **Fail fast**: detect bad input or state at the boundary and stop, rather than carrying it deeper. + +## Using them well + +Principles are for judging trade-offs, not for winning arguments. A useful habit is to name the change you expect: "if we add a second payment provider, how many files change?" That question applies SOLID and DRY without anyone needing to quote them. + +## Where next + +- [Design patterns](design-patterns.md) and [anti-patterns](anti-patterns.md) show the principles applied and ignored. +- The [Guides](../guides/index.md) section covers coding standards in more depth. diff --git a/content/primers/digital-twins-and-device-shadows.md b/content/primers/digital-twins-and-device-shadows.md new file mode 100644 index 0000000..fc6a6c2 --- /dev/null +++ b/content/primers/digital-twins-and-device-shadows.md @@ -0,0 +1,76 @@ +--- +title: Digital twins and device shadows +--- + +# Digital twins and device shadows + +A connected device is often asleep, out of coverage or busy. Applications still need to read its state and change its settings. A device shadow (Microsoft calls it a device twin) solves this with a copy of the device's state in the cloud, which applications can always reach. The device and the shadow catch up with each other whenever the device connects. + +A digital twin is the broader idea: a model of a physical thing, kept in step with it, that can be queried, analysed or simulated. A shadow is the simplest kind of digital twin. + +## Desired and reported state + +The shadow holds two views of the same settings: + +- **Desired**: what applications want the device to be. Written by the cloud side. +- **Reported**: what the device says it is. Written only by the device. + +The difference between them (the delta) is the work the device still has to do. + +```json +{ + "state": { + "desired": { "reportIntervalSeconds": 300, "firmware": "2.4.1" }, + "reported": { "reportIntervalSeconds": 900, "firmware": "2.4.0", "battery": 71 } + }, + "version": 42, + "timestamp": 1760000000 +} +``` + +[[CAPTION:Figure]] A shadow document with a pending change to two settings. + +## The reconciliation loop + +```mermaid +sequenceDiagram + participant App + participant Shadow as Cloud shadow + participant Device + App->>Shadow: set desired.reportIntervalSeconds = 300 + Note over Device: asleep + Device->>Shadow: connects, requests delta + Shadow-->>Device: delta: reportIntervalSeconds = 300 + Device->>Device: apply setting + Device->>Shadow: reported.reportIntervalSeconds = 300 + Shadow-->>App: in sync +``` + +[[CAPTION:Figure]] A setting applied the next time the device wakes. + +The device applies what it can and reports what it actually did. If a setting is invalid, it reports its real value and, ideally, an error field, so the delta stays visible rather than being silently ignored. + +## Getting the details right + +- **Versions**: every update carries the version it was based on, and the service rejects stale updates. This stops two writers overwriting each other. +- **Idempotency**: the device may receive the same delta twice. Applying it again must do no harm. See [idempotency](idempotency.md). +- **Size**: keep the shadow to configuration and a small summary of status. Telemetry belongs in a time-series store, not in the shadow. +- **Ownership**: only the device writes `reported`; only applications write `desired`. Mixing them makes the state impossible to reason about. +- **Commands are different**: "reboot now" is not a state. Use a command or job mechanism for one-off actions. + +## Implementations + +| Option | Notes | +| --- | --- | +| AWS IoT Device Shadow | Shadows over MQTT topics; named shadows for separate concerns | +| Azure IoT Hub device twins | Desired and reported properties, plus tags for querying fleets | +| Eclipse Ditto | Open-source digital twin service for self-hosted platforms | +| LwM2M | The server's view of each device's objects plays the same role; see [LwM2M](lwm2m.md) | +| W3C Web of Things | A standard Thing Description format for describing what a device offers | + +[[CAPTION:Table]] Common ways to implement shadows and twins. + +## Where next + +- [Edge computing for IoT](edge-computing-for-iot.md) covers where the reconciliation logic can run. +- The [Handbook](../handbook/index.md) covers device management architecture in more depth. diff --git a/content/primers/distributed-systems.md b/content/primers/distributed-systems.md new file mode 100644 index 0000000..ea7d768 --- /dev/null +++ b/content/primers/distributed-systems.md @@ -0,0 +1,14 @@ +--- +title: Distributed systems primers +--- + +# Distributed systems primers + +A distributed system is one where the parts run on different machines and talk over a network. Almost every modern system is one, including any IoT platform. The network can be slow, lose messages, deliver them twice or split in two, and clocks on different machines disagree. These primers cover the ideas that follow from that. + +- [CAP and PACELC](cap-and-pacelc.md): what you give up when the network fails, and when it does not. +- [Consistency models](consistency-models.md): strong, eventual and causal consistency, and CRDTs. +- [Idempotency](idempotency.md): making it safe to do the same thing twice. +- [The saga pattern](saga-pattern.md): transactions that span several services. + +The [Handbook](../handbook/index.md) covers how to apply these in a design, and [event-driven architecture](event-driven-architecture.md) is where they matter most. diff --git a/content/primers/docs-as-code.md b/content/primers/docs-as-code.md new file mode 100644 index 0000000..2cc7972 --- /dev/null +++ b/content/primers/docs-as-code.md @@ -0,0 +1,62 @@ +--- +title: Docs as code +--- + +# Docs as code + +Docs as code means writing and managing documentation with the same tools and habits as software. The files are plain text in version control, changes go through reviewed pull requests, and a build pipeline checks and publishes them. Documentation then lives next to the code it describes, changes with it, and gets the same care. + +This site is built that way: the pages are Markdown in a Git repository, checked by a linter, and built into a website, Word files and PDFs by [py-doc](https://github.com/markbac/py-doc). + +## Why it works + +- **Close to the code**: the person changing the code can update the docs in the same pull request, and reviewers can see if they did not. +- **History and review**: every change is tracked, attributed and reviewable. +- **Automation**: spelling, style, broken links and diagrams are checked on every change; publishing needs no manual steps. +- **One source, many outputs**: the same Markdown can build a website, a PDF and a Word document. +- **Familiar tools** for developers, so they are more likely to contribute. + +## The pieces + +```mermaid +flowchart LR + W[Write Markdown and diagram source] --> PR[Pull request] + PR --> L[Lint: style, spelling, links] + PR --> R[Review] + L --> B[Build: site, PDF, Word] + R --> B + B --> P[Publish] +``` + +[[CAPTION:Figure]] A docs-as-code pipeline. + +| Piece | Common choices | +| --- | --- | +| Markup | Markdown (CommonMark or pandoc flavour), AsciiDoc, reStructuredText | +| Diagrams | [PlantUML and Mermaid](plantuml-and-mermaid.md), Structurizr for [C4](c4-model.md) | +| Site generators | MkDocs with the Material theme, Docusaurus, Sphinx, Antora, Astro Starlight | +| Other outputs | pandoc for Word and PDF | +| Style and spelling checks | Vale, markdownlint, cSpell | +| API reference | Generated from [OpenAPI or AsyncAPI](openapi-and-asyncapi.md) descriptions, or from code comments | +| Hosting | GitHub Pages, GitLab Pages, Read the Docs, any static host | + +[[CAPTION:Table]] Typical tools for each part. + +## Good habits + +- **Structure by purpose**. The Diátaxis framework separates four kinds of documentation (tutorials, how-to guides, reference and explanation), because readers need each in a different mood. This site's sections follow similar lines. +- **Write diagrams as text** so they can be reviewed and kept current. +- **Make documentation part of done**: a feature is not finished until its docs are updated. +- **Lint for consistency**: a shared style guide enforced by a tool avoids arguments in review. +- **Keep decisions as [ADRs](architecture-decision-records.md)** in the same repository. +- **Give pages owners**, and review them periodically for accuracy. + +## Limits + +Non-technical contributors may find Git and Markdown a barrier; a web editor that opens pull requests helps. Highly designed documents, such as marketing material, are better made with other tools. + +## Where next + +- [Git practices](git-best-practices.md) covers the version control underneath. +- The [Guides](../guides/index.md) section covers how documentation is written and published here. +- [Docs Like Code](https://www.docslikecode.com/) and [diataxis.fr](https://diataxis.fr/) are good starting points. diff --git a/content/primers/domain-driven-design.md b/content/primers/domain-driven-design.md new file mode 100644 index 0000000..2a16aa5 --- /dev/null +++ b/content/primers/domain-driven-design.md @@ -0,0 +1,54 @@ +--- +title: Domain-driven design +--- + +# Domain-driven design + +Domain-driven design (DDD), from Eric Evans' 2003 book of the same name, is an approach to building software for complicated business areas. It puts the business domain, and the language the experts use for it, at the centre of the design. It has two halves: strategic design, which is about boundaries between parts of a system, and tactical design, which is about the code inside one part. + +## Ubiquitous language + +The team (developers and domain experts together) agrees one vocabulary and uses it everywhere: in conversation, in tickets, in class names and in database tables. If the experts say "commissioning" a device, the code has a `commission()` method, not `activateUnit()`. When a word turns out to mean two things, that is a sign there are two contexts. + +## Strategic design + +- **Bounded context**: a part of the system inside which every term has one meaning. "Device" in the manufacturing context (a serial number, a test result) is not the same as "device" in the operations context (a location, a firmware version, a last-seen time). Each context has its own model and usually its own team and codebase. +- **Context map**: a picture of the bounded contexts and how they relate: which one is upstream, which conforms to which, where an anti-corruption layer translates between models. +- **Subdomains**: the core domain is where the business competes and deserves the best people. Supporting and generic subdomains (billing, user accounts) can often be bought or kept simple. + +```mermaid +flowchart LR + MFG[Manufacturing context] -- device built --> OPS[Operations context] + OPS -- usage records --> BILL[Billing context] + OPS -. anti-corruption layer .-> LEGACY[Legacy asset register] +``` + +[[CAPTION:Figure]] A simple context map. + +Bounded contexts are the usual starting point for service boundaries. See [modular monolith or microservices](modular-monolith-vs-microservices.md). + +## Tactical design + +| Building block | What it is | +| --- | --- | +| Entity | An object with an identity that lasts over time (a device, an order) | +| Value object | An object defined only by its values, and immutable (a temperature, an address) | +| Aggregate | A cluster of objects changed together as one unit, with one root that guards its rules | +| Repository | Loads and saves whole aggregates, hiding storage | +| Domain event | A record of something that happened that the business cares about ("Device commissioned") | +| Domain service | Logic that does not belong to any one entity | + +[[CAPTION:Table]] The tactical building blocks. + +The most useful rule is about aggregates: change only one aggregate per transaction, and refer to other aggregates by their identity. This keeps transactions small and makes it possible to split the system later. + +## When it is worth it + +DDD pays off when the business rules are complicated and the domain experts are available to talk to. For a simple data-entry application it is too much. Many teams use the strategic ideas (language, contexts) everywhere and the tactical patterns only in the core domain. + +## Where next + +- [Event storming](event-storming.md) is a workshop for discovering domain events and contexts. +- [Hexagonal architecture](hexagonal-architecture.md) keeps the domain model free of infrastructure. +- The [Handbook](../handbook/index.md) covers domain modelling and service boundaries in more depth. +- Vaughn Vernon's *Domain-Driven Design Distilled* is the shortest good introduction. diff --git a/content/primers/edge-computing-for-iot.md b/content/primers/edge-computing-for-iot.md new file mode 100644 index 0000000..c2d3114 --- /dev/null +++ b/content/primers/edge-computing-for-iot.md @@ -0,0 +1,68 @@ +--- +title: Edge computing for IoT +--- + +# Edge computing for IoT + +Edge computing means processing data close to where it is produced, on the device or on a gateway nearby, rather than sending everything to the cloud. In an IoT system it answers the question of where each piece of logic should run. + +## Why process at the edge + +- **Latency**: a safety cut-off or a control loop cannot wait for a round trip to a data centre. +- **Bandwidth and cost**: a vibration sensor can produce megabytes a minute. Sending a summary or an alert instead saves airtime, cellular data and cloud storage. +- **Resilience**: a site must keep working when its internet connection fails, and catch up afterwards. +- **Privacy and data rules**: raw video or personal data may need to stay on site, with only anonymised results leaving it. +- **Power**: for battery devices, computing a result locally often uses less energy than transmitting the raw data. + +## The tiers + +```mermaid +flowchart LR + subgraph Device["Device edge"] + S[Sensors and microcontroller: filter, threshold, compress] + end + subgraph Gateway["Local or gateway edge"] + G[Gateway or industrial PC: aggregate, run rules and models, buffer] + end + subgraph Cloud["Cloud"] + C[Fleet management, long-term storage, analytics, model training] + end + S -- local radio or fieldbus --> G + G -- internet --> C + C -- configuration, models, updates --> G +``` + +[[CAPTION:Figure]] Where processing can happen. + +| Tier | Typical hardware | Good for | +| --- | --- | --- | +| Device | Microcontroller, a few hundred kilobytes of RAM | Filtering, thresholds, simple anomaly detection, small machine-learning models (TinyML) | +| Gateway or on-premises | Linux gateway, industrial PC, small server | Protocol translation, aggregation across devices, local dashboards and control, larger models, store and forward | +| Cloud | Elastic compute and storage | Fleet-wide analytics, training models, history, integration with business systems | + +[[CAPTION:Table]] What each tier is good at. + +## Patterns + +- **Filter and aggregate**: send minimum, maximum and mean per minute instead of every sample; send changes rather than repeated values. +- **Store and forward**: buffer data locally during an outage and upload it, with original timestamps, when the link returns. +- **Local control loop**: the edge acts on its own and reports what it did; the cloud sets the rules. +- **Protocol gateway**: translate local protocols (Modbus, BACnet, Bluetooth LE, Zigbee) into [MQTT](mqtt.md) or [LwM2M](lwm2m.md) for the cloud. +- **Inference at the edge, training in the cloud**: models are trained centrally on fleet data and pushed to the edge. +- **Shadow and reconcile**: the cloud holds the desired configuration and the edge converges on it; see [digital twins and device shadows](digital-twins-and-device-shadows.md). + +## The costs + +Edge computing moves complexity rather than removing it. Edge software must be deployed, updated, monitored and secured on hardware you may not be able to reach. Plan for: + +- Remote software deployment and rollback (containers on gateways; [firmware over the air](firmware-over-the-air.md) on devices). +- Physical security: edge hardware can be stolen or opened. Use secure boot and encrypted storage. +- Clock synchronisation, so data from different devices lines up. +- Monitoring of the edge itself, not just the data it sends. + +Platforms such as AWS IoT Greengrass, Azure IoT Edge, KubeEdge, and lightweight Kubernetes distributions (k3s) help run and update software on gateways. + +## Where next + +- [Embedded systems design](embedded-systems-design.md) covers the device tier. +- The [Handbook](../handbook/index.md) covers edge and platform architecture in more depth. diff --git a/content/primers/embedded-systems-design.md b/content/primers/embedded-systems-design.md new file mode 100644 index 0000000..4f33c51 --- /dev/null +++ b/content/primers/embedded-systems-design.md @@ -0,0 +1,65 @@ +--- +title: Embedded systems design +--- + +# Embedded systems design + +Embedded software runs on hardware chosen for cost, size and power, not for the programmer's convenience. A typical microcontroller has tens or hundreds of kilobytes of RAM, a megabyte or two of flash, no memory management unit and no operating system unless you add one. The design has to fit the hardware, meet deadlines, survive faults on its own and, for many products, run for years on a battery. + +## What makes it different + +- **Fixed resources**: memory and CPU are fixed at manufacture. Running out is a design failure, not something to fix by adding a server. +- **Real time**: many tasks have deadlines (sample this sensor every millisecond, stop the motor within 10 ms). Being late is as bad as being wrong. +- **Direct hardware access**: the software drives peripherals through registers and interrupts. +- **No one to help**: a device in the field must detect and recover from faults by itself. +- **Long life**: products stay in service for ten years or more, so code must be maintainable and updatable. + +## A layered structure + +```mermaid +flowchart TB + APP[Application: business logic and state machines] + MW[Middleware: protocols, file system, crypto, update agent] + OS[RTOS or bare-metal scheduler] + HAL[Hardware abstraction layer: drivers with stable interfaces] + BSP[Board support package: pins, clocks, memory map] + HW[Microcontroller and peripherals] + APP --> MW --> OS --> HAL --> BSP --> HW +``` + +[[CAPTION:Figure]] Typical firmware layers. + +The hardware abstraction layer (HAL) is the most important boundary. Application code calls `sensor_read()`, not a register address. That lets the same logic run on a new board, or on a PC for unit tests. It is the embedded form of [hexagonal architecture](hexagonal-architecture.md). + +## Bare metal or RTOS + +Simple devices run a "superloop": one `while (1)` loop that polls inputs and runs a [state machine](finite-state-machines.md), with interrupts for urgent events. Once there are several independent activities with different timing needs, a real-time operating system makes the design clearer. See [real-time operating systems and FreeRTOS](rtos-and-freertos.md). + +## Memory + +- Prefer static allocation. `malloc` on a small heap leads to fragmentation and failures that only appear after weeks of running. +- If dynamic allocation is needed, use fixed-size pools. +- Measure stack use per task and leave a margin. Stack overflow is a common cause of odd crashes. +- Know where every buffer lives and how big it can grow. + +## Power + +Battery life is decided by how long the device sleeps, not by how fast it runs: + +- Do the work quickly, then enter the deepest sleep mode the hardware allows. +- Wake on interrupts and timers instead of polling. +- Turn off peripherals and radios when not in use; the radio is usually the biggest consumer. +- Measure current over a whole duty cycle with a power analyser. Estimates are always optimistic. + +## Reliability + +- Use a [watchdog timer](watchdog-timers.md) to recover from hangs. +- Check every return code; decide what the device does when a peripheral fails. +- Detect brown-outs and protect flash writes from power loss. +- Plan [firmware updates](firmware-over-the-air.md) and a safe [bootloader](bootloaders.md) from the start. +- For safety-related code, follow a coding standard such as MISRA C and use [static analysis](static-code-analysis.md). + +## Where next + +- [Interrupts and DMA](interrupts-and-dma.md) covers how firmware reacts to hardware. +- The [Guides](../guides/index.md) section has embedded coding guidance, and the [Handbook](../handbook/index.md) covers device architecture. diff --git a/content/primers/etsi-en-303-645.md b/content/primers/etsi-en-303-645.md new file mode 100644 index 0000000..dd1fe1c --- /dev/null +++ b/content/primers/etsi-en-303-645.md @@ -0,0 +1,47 @@ +--- +title: ETSI EN 303 645 +--- + +# ETSI EN 303 645 + +ETSI EN 303 645, *Cyber Security for Consumer Internet of Things: Baseline Requirements*, is a European standard. It sets a minimum level of security for connected consumer products such as smart speakers, cameras, toys, appliances and home hubs. It was first published in 2020 and grew out of the UK government's 2018 Code of Practice for consumer IoT security. It is now the usual reference for IoT security regulation and certification. Its requirements are good practice for any connected device, consumer or not. + +## The thirteen provisions + +| No. | Provision | In short | +| --- | --- | --- | +| 5.1 | No universal default passwords | Each device has a unique password, or the user must set one | +| 5.2 | Implement a means to manage reports of vulnerabilities | Publish a vulnerability disclosure policy and act on reports | +| 5.3 | Keep software updated | Secure updates, and a stated period for which they will be provided | +| 5.4 | Securely store sensitive security parameters | Protect keys and credentials; no hard-coded credentials shared across devices | +| 5.5 | Communicate securely | Use current cryptography; encrypt sensitive data in transit | +| 5.6 | Minimise exposed attack surfaces | Disable unused interfaces, services and debug ports | +| 5.7 | Ensure software integrity | Verify software with secure boot | +| 5.8 | Ensure that personal data is secure | Protect personal data in transit and at rest | +| 5.9 | Make systems resilient to outages | Keep working locally when the network or cloud fails | +| 5.10 | Examine system telemetry data | Monitor for security anomalies | +| 5.11 | Make it easy for users to delete user data | Provide a simple way to wipe personal data | +| 5.12 | Make installation and maintenance of devices easy | Secure set-up with minimal steps | +| 5.13 | Validate input data | Check data from interfaces and networks | + +[[CAPTION:Table]] The thirteen cyber security provisions, summarised. + +A further clause covers data protection: telling users what personal data is processed and getting consent where needed. Each provision contains individual requirements, some mandatory and some recommended. ETSI TS 103 701 defines how to assess a product against them. + +## Why it matters: regulation + +The first three provisions became law in the UK under the Product Security and Telecommunications Infrastructure (PSTI) Act, in force since 29 April 2024. Products sold to UK consumers must have no universal default passwords and a published way to report vulnerabilities. They must also state the minimum period for which they will receive security updates. Meeting the corresponding parts of EN 303 645 is a recognised way to comply. + +In the EU, the radio equipment rules have required cybersecurity for internet-connected radio products since August 2025, through the EN 18031 standards. The Cyber Resilience Act goes further: it sets security requirements for almost all products with digital elements, with its main obligations applying from December 2027. EN 303 645 is a good foundation for both. + +## Applying it + +- Treat it as the baseline, not the goal. It sets out what a device must do, not how; pair it with [secure coding practices](secure-coding-practices.md) and a threat model. +- Map each requirement to a design decision and to evidence (test results, documents) early in development. Retro-fitting secure boot or unique credentials after launch is expensive. +- Many provisions need factory and back-end support: unique credentials per device ([PKI and certificates](pki-and-certificates.md)), a [firmware update](firmware-over-the-air.md) service, and a vulnerability handling process. + +## Where next + +- [Bootloaders](bootloaders.md) covers secure boot (5.7), and [TLS and DTLS](tls-and-dtls.md) covers secure communication (5.5). +- The [Standards](../standards/index.md) section covers how these requirements apply here. +- The standard is free to download from [etsi.org](https://www.etsi.org/technologies/consumer-iot-security). diff --git a/content/primers/event-driven-architecture.md b/content/primers/event-driven-architecture.md new file mode 100644 index 0000000..6263cad --- /dev/null +++ b/content/primers/event-driven-architecture.md @@ -0,0 +1,60 @@ +--- +title: Event-driven architecture +--- + +# Event-driven architecture + +In an event-driven architecture (EDA), components announce that something has happened, and other components react if they care. The producer of an event does not know who consumes it. That loose coupling is the main benefit, and also the main source of difficulty. + +## Events, commands and queries + +| Message | Meaning | Example | +| --- | --- | --- | +| Event | A fact about the past; cannot be refused | `DeviceWentOffline` | +| Command | A request for something to happen; can fail or be rejected | `RebootDevice` | +| Query | A request for information | `GetDeviceStatus` | + +[[CAPTION:Table]] Three kinds of message. + +Name events in the past tense. A message called `UpdateDevice` is a command in disguise, and treating it as an event hides who is responsible for the outcome. + +## The building blocks + +```mermaid +flowchart LR + P1[Device service] -- DeviceWentOffline --> B[(Broker or event log)] + B --> C1[Alerting] + B --> C2[Dashboard] + B --> C3[Analytics] +``` + +[[CAPTION:Figure]] One producer, several independent consumers. + +- **Producers** publish events. +- **The channel** carries them: a message broker (MQTT, RabbitMQ, a cloud pub/sub service) or an event log ([Kafka](kafka-event-streaming.md)). A log keeps events so new consumers can replay history; a broker usually forgets a message once it is delivered. +- **Consumers** subscribe and react. Each can be deployed, scaled and fail on its own. + +## Common patterns + +- **Event notification**: the event says only that something changed; the consumer calls back for details. Small events, but more coupling at run time. +- **Event-carried state transfer**: the event carries the data the consumers need, so they keep their own copy and do not call back. +- **Event sourcing**: the events are the system of record. The current state is rebuilt by replaying them. Powerful for audit and debugging, and complicated to run. +- **CQRS (Command Query Responsibility Segregation)**: separate models for writing and reading, often fed by events. Useful when reads and writes have very different shapes or loads. + +## What gets harder + +- **Consistency**: consumers see changes later than the producer. Design for [eventual consistency](consistency-models.md). +- **Duplicates and order**: most brokers deliver at least once and order only within a partition or topic. Make consumers [idempotent](idempotency.md). +- **Tracing**: a business flow is spread across many services. Use correlation IDs and distributed tracing from the start. +- **Schema change**: events are a public contract. Version them, add fields rather than changing them, and keep a schema registry or [AsyncAPI](openapi-and-asyncapi.md) description. +- **Multi-step workflows**: when several services must all succeed or all undo, use a [saga](saga-pattern.md). + +## When it fits + +EDA suits systems where many parties react to the same facts: IoT telemetry, notifications, audit trails, integrations between teams. For a simple request that needs an answer now, a direct API call is easier to understand. + +## Where next + +- [Domain-driven design](domain-driven-design.md) helps find the events that matter. +- The [Handbook](../handbook/index.md) covers messaging and integration patterns. +- Martin Fowler's talk and article [What do you mean by "Event-Driven"?](https://martinfowler.com/articles/201701-event-driven.html) separates the four patterns above. diff --git a/content/primers/event-storming.md b/content/primers/event-storming.md new file mode 100644 index 0000000..651076c --- /dev/null +++ b/content/primers/event-storming.md @@ -0,0 +1,65 @@ +--- +title: Event storming +--- + +# Event storming + +Event storming is a workshop technique, created by Alberto Brandolini, for exploring a business domain quickly with everyone who knows something about it. The group maps out the events that happen in the domain on a long wall using sticky notes of different colours. In a few hours it builds a shared picture that would take weeks of interviews and documents, and it surfaces disagreements and unknowns early. + +## Who and what + +- **People**: domain experts, developers, testers, product owners, operations, anyone with knowledge or questions. Between six and fifteen people works well. +- **Space**: a long wall with a roll of paper, or an online whiteboard such as Miro. +- **A facilitator** who keeps the energy up and the discussion moving, but does not supply the answers. + +## The notes + +| Colour (by convention) | Element | Example | +| --- | --- | --- | +| Orange | Domain event: something that happened, in the past tense | Device installed | +| Blue | Command: the action or decision that caused it | Install device | +| Yellow (small) | Actor: who issued the command | Field engineer | +| Lilac | Policy: a rule that reacts to an event ("whenever...") | When a device is installed, schedule a first reading | +| Pink | External system | Mobile network | +| Green | Read model: information someone needs to decide | Job list for today | +| Large yellow | Aggregate: the thing that accepts commands and produces events | Device | +| Red or hot pink | Hot spot: a problem, question or disagreement | "Who confirms the install?" | + +[[CAPTION:Table]] The usual sticky-note colours and what they mean. + +## The flow of a session + +1. **Chaotic exploration**: everyone writes domain events and puts them on the wall, roughly in time order. No discussion yet. +2. **Enforce the timeline**: arrange the events into a single sequence, merge duplicates and argue about the order. This is where the most learning happens. +3. **Mark hot spots** wherever there is confusion or conflict. Do not resolve them all; record them. +4. **Add commands, actors, policies and external systems** to explain what triggers each event. +5. **Find the boundaries**: look for clusters of events that belong together and pivotal events where the language changes. These suggest [bounded contexts](domain-driven-design.md) and possible service boundaries. + +```mermaid +flowchart LR + A[Field engineer] --> C[Install device] + C --> E1[Device installed] + E1 --> P[Policy: schedule first reading] + P --> C2[Request reading] + C2 --> E2[Reading received] +``` + +[[CAPTION:Figure]] A fragment of a process-level model. + +## Levels + +- **Big picture**: the whole business flow, to understand the landscape and find the hot spots and boundaries. +- **Process modelling**: one process in detail, with commands, policies and read models. +- **Software design**: aggregates and their rules, ready to drive code. + +## Tips + +- Start with events, not systems or screens. "What happens?" is easier to answer than "what do we build?". +- Invite the people who disagree. +- Photograph the wall and keep the hot-spot list; the questions are often the most valuable output. +- Follow up with the actual decisions as [ADRs](architecture-decision-records.md). + +## Where next + +- [Domain-driven design](domain-driven-design.md) and [event-driven architecture](event-driven-architecture.md) turn the results into a design. +- Brandolini's [eventstorming.com](https://www.eventstorming.com/) and his book *Introducing EventStorming* describe the method in full. diff --git a/content/primers/finite-state-machines.md b/content/primers/finite-state-machines.md new file mode 100644 index 0000000..4168d1a --- /dev/null +++ b/content/primers/finite-state-machines.md @@ -0,0 +1,80 @@ +--- +title: Finite state machines +--- + +# Finite state machines + +A finite state machine (FSM) describes behaviour as a fixed set of states, the events that can happen, and the transitions between states that each event causes. Most embedded devices are state machines whether or not anyone drew one: a device that is booting, connecting, connected, updating or sleeping behaves differently in each case. Making the states explicit replaces a tangle of flags and `if` statements with something you can draw, review and test. + +## An example + +```mermaid +stateDiagram-v2 + [*] --> Idle + Idle --> Connecting: report due + Connecting --> Sending: connected + Connecting --> Backoff: failed + Backoff --> Connecting: retry timer + Backoff --> Idle: too many failures + Sending --> Idle: acknowledged + Sending --> Backoff: timeout +``` + +[[CAPTION:Figure]] The connection logic of a reporting device. + +Every combination of state and event has a defined outcome, even if that outcome is "ignore it". Asking "what happens if a timeout arrives while idle?" is how state machines find bugs before the code exists. + +## Implementing one in C + +The simplest form is a `switch` on the current state, with events passed in: + +```c +typedef enum { IDLE, CONNECTING, SENDING, BACKOFF } state_t; +typedef enum { EV_REPORT_DUE, EV_CONNECTED, EV_FAILED, EV_ACK, EV_TIMEOUT, EV_RETRY } event_t; + +static state_t state = IDLE; + +void fsm_handle(event_t ev) +{ + switch (state) { + case IDLE: + if (ev == EV_REPORT_DUE) { modem_connect(); state = CONNECTING; } + break; + case CONNECTING: + if (ev == EV_CONNECTED) { send_report(); state = SENDING; } + else if (ev == EV_FAILED) { start_backoff(); state = BACKOFF; } + break; + case SENDING: + if (ev == EV_ACK) { modem_sleep(); state = IDLE; } + else if (ev == EV_TIMEOUT) { start_backoff(); state = BACKOFF; } + break; + case BACKOFF: + if (ev == EV_RETRY) { modem_connect(); state = CONNECTING; } + break; + } +} +``` + +[[CAPTION:Figure]] A switch-based state machine. + +For larger machines, a table of `{state, event, next state, action}` rows is easier to review, and function pointers (one handler per state) scale better still. + +## Variations + +- **Moore and Mealy machines**: in a Moore machine, outputs depend only on the state; in a Mealy machine, they depend on the state and the event. Firmware usually mixes both. +- **Hierarchical state machines** (statecharts, from David Harel): states can contain sub-states and share transitions, which avoids repeating "on fault, go to error" in every state. Frameworks such as Quantum Leaps' QP implement them. +- **Entry and exit actions**: code that runs whenever a state is entered or left, such as starting or stopping a timer. This keeps the transitions simple. +- **The state pattern**: the object-oriented version, where each state is a class. See [design patterns](design-patterns.md). + +## Good habits + +- Drive the machine with events from a queue, so it handles one event at a time and never re-enters itself. +- Keep actions short; long work belongs in another task that posts an event when it finishes. +- Log every transition during development. A trace of state changes explains most field bugs. +- Test each transition, including the "impossible" ones. +- Keep the diagram in the repository next to the code, as [Mermaid or PlantUML](plantuml-and-mermaid.md), so it stays up to date. + +## Where next + +- [Real-time operating systems and FreeRTOS](rtos-and-freertos.md) shows how state machines fit into tasks. +- Miro Samek's *Practical UML Statecharts in C/C++* covers hierarchical state machines in depth. diff --git a/content/primers/firmware-over-the-air.md b/content/primers/firmware-over-the-air.md new file mode 100644 index 0000000..d7d6b49 --- /dev/null +++ b/content/primers/firmware-over-the-air.md @@ -0,0 +1,63 @@ +--- +title: Firmware over the air +--- + +# Firmware over the air + +Firmware over the air (FOTA) means updating the software on devices in the field through their network connection. Every connected product needs it: to fix bugs, close security holes and add features without sending someone to each device. Regulations increasingly require it too (see [ETSI EN 303 645](etsi-en-303-645.md)). A badly designed update system is also the quickest way to break a whole fleet at once, so it deserves careful design. + +## The pieces + +```mermaid +flowchart LR + CI[Build pipeline] -- signed image --> R[(Image repository)] + R --> C[Campaign service] + C -- notify and schedule --> D[Device update agent] + D -- download in chunks --> R + D --> BL[Bootloader: verify, install, confirm or roll back] + D -- status --> C +``` + +[[CAPTION:Figure]] An end-to-end firmware update system. + +- **Build and signing**: the [CI/CD pipeline](cicd-pipelines.md) builds the image, and a signing service signs it with a key kept in an HSM. +- **Repository**: stores images and their metadata (version, target hardware, minimum version it can update from). +- **Campaign service**: decides which devices get which image and when, and tracks progress. +- **Update agent** on the device: downloads, checks and stores the image. +- **[Bootloader](bootloaders.md)**: verifies the signature, installs the image, and rolls back if it fails. + +## Getting it to the device + +- **Pull or push**: the device can check for updates when it wakes, or the server can notify it. [LwM2M](lwm2m.md) defines both through its firmware update object. +- **Chunks and resume**: download in blocks (HTTP range requests, CoAP block-wise transfer) and resume after a dropped connection rather than starting again. +- **Delta updates**: send only the differences from the installed version. On [NB-IoT](cellular-lpwan.md) and other slow links, a delta of tens of kilobytes instead of a full image of a megabyte makes the difference between practical and impossible. Deltas need the exact base version on the device, so keep full images as a fallback. +- **Compression** of full images helps too. + +## Making it safe + +- **Verify before installing**: check the signature and the hardware compatibility before touching the running image. +- **A/B slots and rollback**: keep the old image until the new one has started, connected and reported healthy. If it does not confirm, the bootloader reverts. +- **Anti-rollback**: refuse to install versions older than a security minimum. +- **Power and battery**: do not start an update with a low battery, and make every step survive power loss. +- **Encrypt the image** if it contains secrets or valuable intellectual property; signing alone does not hide the contents. + +## Rolling out + +Never update a whole fleet at once: + +1. Internal test devices. +2. A small canary group in the field, perhaps 1 %. +3. Widening waves, with automatic pauses if failure or reset rates rise. + +Watch the metrics during each wave: update success rate, reboot counts, connection rate and battery drain. Schedule updates for quiet times and respect devices that are doing something important. + +## Standards and tools + +- The SUIT manifest format (IETF, RFC 9019 architecture and RFC 9124 information model) describes updates for constrained devices in a standard way. +- MCUboot is a widely used secure bootloader for A/B updates. +- Mender, Memfault, the AWS, Azure and Golioth device services, and Eclipse hawkBit provide campaign management. + +## Where next + +- [Bootloaders](bootloaders.md) covers the device side in more depth. +- The [Handbook](../handbook/index.md) covers device management architecture. diff --git a/content/primers/git-best-practices.md b/content/primers/git-best-practices.md new file mode 100644 index 0000000..d0d5fe2 --- /dev/null +++ b/content/primers/git-best-practices.md @@ -0,0 +1,51 @@ +--- +title: Git practices +--- + +# Git practices + +Git is the version control system almost everyone uses. Its basics take an afternoon to learn; using it so that the history is genuinely useful takes a few habits. A good history answers "what changed, when and why?" years later. A bad one is a list of commits called "fix" and "wip". + +## Commits + +- **One logical change per commit**. A commit should build and pass tests on its own, so `git bisect` can find the change that introduced a bug. +- **Write useful messages**: a short summary line (around 50 characters, imperative mood: "Add retry to modem attach"), a blank line, then a body explaining why the change was needed and anything surprising. The diff already shows what changed. +- **Conventional Commits** (`feat:`, `fix:`, `docs:` and so on) add a machine-readable type, which tools can use to generate changelogs and version numbers. +- **Commit often locally, tidy before sharing**: `git commit --fixup` and `git rebase --autosquash` let you clean up a branch before review. +- **Never commit secrets**. Use `.gitignore`, a pre-commit hook and secret scanning in CI. If a secret is committed, rotate it; removing it from history is not enough. + +## Branching + +| Strategy | How it works | Good for | +| --- | --- | --- | +| Trunk-based development | Everyone merges small changes to `main` at least daily; short-lived branches, feature flags for unfinished work | Teams with good automated tests and [CI/CD](cicd-pipelines.md); fastest feedback | +| GitHub flow | A branch per change, a pull request, review, merge to `main`, deploy | Most web services and libraries | +| Git flow | Long-lived `develop` and `main`, plus feature, release and hotfix branches | Products with scheduled releases and several supported versions; adds overhead | +| Release branches | Trunk-based development, with a branch cut for each release and fixes cherry-picked to it | Firmware and products that need to maintain released versions | + +[[CAPTION:Table]] Common branching strategies. + +Whatever the strategy, keep branches short-lived. A branch that lives for weeks means a painful merge and late feedback. + +## Merging + +- **Merge commit**: keeps the branch's commits and records the merge. Full history, but noisy. +- **Squash merge**: the whole branch becomes one commit on `main`. Clean, but loses the individual steps. +- **Rebase and merge**: replays the commits on top of `main` for a linear history. + +Pick one per repository and use it consistently. Do not rewrite history that others have already pulled; `git push --force-with-lease` is safer than `--force` on your own branches. + +## Useful habits + +- **Protect `main`**: require pull requests, reviews and passing checks. +- **Tag releases** with annotated, ideally signed, tags following semantic versioning (`v2.4.1`). +- **Sign commits** where provenance matters, with GPG, SSH or Sigstore keys. +- **Use hooks** (with a tool such as pre-commit) for formatting and quick checks before code leaves your machine. +- **Learn a few power tools**: `git bisect` to find a breaking commit, `git blame` (or better, `git log -L`) to find why a line changed, `git reflog` to recover lost work, `git worktree` to work on two branches at once. +- **Large binary files** belong in Git LFS or an artefact store, not in the repository. + +## Where next + +- [CI/CD pipelines](cicd-pipelines.md) builds on these habits. +- The [Templates](../templates/index.md) section has a pull-request template and a commit-message hook. +- *Pro Git* by Scott Chacon and Ben Straub is free at [git-scm.com/book](https://git-scm.com/book). diff --git a/content/primers/hardware-in-the-loop-testing.md b/content/primers/hardware-in-the-loop-testing.md new file mode 100644 index 0000000..6021d8e --- /dev/null +++ b/content/primers/hardware-in-the-loop-testing.md @@ -0,0 +1,63 @@ +--- +title: Hardware-in-the-loop testing +--- + +# Hardware-in-the-loop testing + +Hardware-in-the-loop (HIL) testing runs real firmware on real hardware, with its inputs and outputs connected to a test rig that a computer controls. The rig can simulate sensors, switch power, disturb the radio and watch every output, so tests that would need a technician at a bench can run automatically, on every change. + +Unit tests on a PC catch logic errors cheaply. They cannot catch timing problems, driver bugs, power behaviour or the way a real modem misbehaves. HIL tests can. + +## Where it fits + +```mermaid +flowchart TB + U[Unit tests on the host: many, fast, every commit] + S[Simulation and emulation: QEMU, Renode, simulated peripherals] + H[HIL tests on real boards: fewer, slower, every merge or nightly] + F[Field trials: real devices in real places] + U --> S --> H --> F +``` + +[[CAPTION:Figure]] HIL in the embedded testing pyramid. + +## A test bench + +A typical bench has: + +- **The device under test**, connected through its debug port (SWD or JTAG) so the test can flash firmware and read state. +- **A programmable power supply** to vary voltage, simulate brown-outs and measure current. +- **Signal generation and measurement**: digital and analogue I/O, a logic analyser, perhaps an oscilloscope. +- **Relays and switches** to connect, disconnect or short signals. +- **Radio equipment**: a shielded box, a programmable attenuator, or a network emulator for cellular tests. +- **A host computer** that runs the tests and controls everything. Python with pytest is a common choice; many instruments speak SCPI, which PyVISA can drive. + +## Fault injection + +The most valuable HIL tests check how the device behaves when things go wrong: + +| Fault | How the bench causes it | What the firmware should do | +| --- | --- | --- | +| Brown-out during a flash write | Drop the supply voltage briefly during an update | Reset cleanly; the file system or update recovers without corruption | +| Stuck I2C bus | Hold the data line low during a sensor read | Detect the timeout, clock the bus free, reinitialise the driver | +| Weak or lost radio | Increase attenuation during a connection | Back off, retry with jitter, then sleep without draining the battery | +| Hung task | Trigger a test hook that blocks a task | The [watchdog](watchdog-timers.md) resets the device and the reset cause is reported | +| Clock failure | Disable the external crystal | Fall back to the internal oscillator and report the fault | + +[[CAPTION:Table]] Examples of automated fault injection. + +## Running it in CI + +- Connect benches to the [CI/CD pipeline](cicd-pipelines.md) as self-hosted runners, and run a short smoke suite on every merge and the full suite nightly. +- Make every test start from a known state: flash, power-cycle, reset the instruments. +- Record logs, current traces and serial output as test artefacts. +- Treat benches as shared infrastructure: label them, monitor them, and have a way to reserve one for debugging. +- Keep tests independent of a particular bench where possible, so benches can be added as the suite grows. + +Open-source tools such as Labgrid and LAVA help manage benches and boards. Commercial HIL systems exist for automotive and industrial work, where they also simulate whole vehicles or plants. + +## Where next + +- [Embedded systems design](embedded-systems-design.md) describes the behaviour these tests verify. +- [Testing strategies](testing-strategies.md) covers testing more generally. +- The [Guides](../guides/index.md) section covers setting up a test bench. diff --git a/content/primers/hexagonal-architecture.md b/content/primers/hexagonal-architecture.md new file mode 100644 index 0000000..f49735f --- /dev/null +++ b/content/primers/hexagonal-architecture.md @@ -0,0 +1,60 @@ +--- +title: Hexagonal architecture +--- + +# Hexagonal architecture + +Hexagonal architecture, also called ports and adapters, was described by Alistair Cockburn in 2005. Its one idea is that the business logic sits in the middle and knows nothing about the outside world. It does not know about the database, the web framework or the message broker. Everything outside talks to it through ports, and adapters connect those ports to real technology. + +The hexagon shape has no meaning. It is just a shape with enough sides to draw several adapters around it. + +## The parts + +- **The core (domain)** holds the rules of the business: entities, calculations, decisions. It is plain code with no framework imports. +- **Ports** are interfaces that the core defines. A driving port (also called inbound) is how the outside asks the core to do something: `RegisterDevice`. A driven port (outbound) is something the core needs from outside: `DeviceRepository`, `NotificationSender`. +- **Adapters** implement the ports for one technology. A REST controller and a command-line tool can both drive the same inbound port. A PostgreSQL adapter and an in-memory test adapter can both implement the same repository port. + +```mermaid +flowchart LR + REST[REST adapter] --> IN((Inbound port)) + CLI[CLI adapter] --> IN + IN --> CORE[Domain core] + CORE --> OUT1((Repository port)) + CORE --> OUT2((Notifier port)) + OUT1 --> PG[PostgreSQL adapter] + OUT1 --> MEM[In-memory adapter for tests] + OUT2 --> MQ[MQTT adapter] +``` + +[[CAPTION:Figure]] Adapters on the outside, the domain in the middle. + +The dependency arrows always point inwards. That is [dependency inversion](design-principles.md) applied to a whole application. + +## Why it helps + +- **Testing**: the core can be tested fast, with in-memory adapters and no database or network. +- **Change**: swapping a database or adding a new interface (a queue consumer next to the REST API) touches only an adapter. +- **Clarity**: the business rules are in one place, not scattered across controllers and SQL. + +## Relatives + +Several later styles share the same idea with different names and layering: + +- **Onion architecture** (Jeffrey Palermo) draws the same thing as concentric rings. +- **Clean architecture** (Robert C. Martin) adds explicit rings for entities and use cases, with the same rule that dependencies point inwards. + +In practice the differences are mostly vocabulary. Pick one set of names and use it consistently. + +## In embedded systems + +The same idea appears in firmware as a hardware abstraction layer: application logic talks to a `Sensor` or `Storage` interface, and board-specific drivers implement it. This lets the logic run on a PC in unit tests and move between microcontrollers. See [embedded systems design](embedded-systems-design.md). + +## When not to use it + +For a small CRUD service with no real business rules, ports and adapters add ceremony for little gain. Use it where the logic is worth protecting. + +## Where next + +- [Domain-driven design](domain-driven-design.md) is about what goes in the core. +- The [Handbook](../handbook/index.md) covers architectural styles and how to choose between them. +- Cockburn's original article is at [alistair.cockburn.us/hexagonal-architecture](https://alistair.cockburn.us/hexagonal-architecture/). diff --git a/content/primers/idempotency.md b/content/primers/idempotency.md new file mode 100644 index 0000000..fea441d --- /dev/null +++ b/content/primers/idempotency.md @@ -0,0 +1,59 @@ +--- +title: Idempotency +--- + +# Idempotency + +An operation is idempotent if doing it twice has the same effect as doing it once. Setting a thermostat to 21 °C is idempotent; turning it up by one degree is not. In a distributed system this matters because messages get retried and duplicated, so anything that can be repeated eventually will be. + +## Why duplicates happen + +- A client sends a request, the server does the work, and the reply is lost. The client cannot tell this from a request that never arrived, so it retries. +- Message brokers usually promise at-least-once delivery. MQTT QoS 1 and most Kafka consumers can see the same message again after a reconnect or a rebalance. +- A device reboots in the middle of an upload and starts again. + +"Exactly once" delivery across a network is not possible in general. What systems actually offer is at-least-once delivery plus idempotent processing, which gives exactly-once effects. + +## Idempotency in HTTP + +| Method | Idempotent? | Notes | +| --- | --- | --- | +| GET, HEAD | Yes | Should not change anything | +| PUT | Yes | Replaces the resource with the given state | +| DELETE | Yes | Deleting twice leaves it deleted (the second call may return 404) | +| POST | No | Usually creates something; repeat it and you get two | +| PATCH | Depends | "Set field to X" is; "append to list" is not | + +[[CAPTION:Table]] Which HTTP methods are idempotent by definition. + +## Techniques + +- **Idempotency keys**: the client generates a unique key per logical operation and sends it with every attempt (for example an `Idempotency-Key` header). The server stores the key with the result and returns the stored result for a repeat. Payment APIs work this way. +- **Natural keys and upserts**: use an identifier from the data itself (device ID plus reading timestamp) and write with "insert or update", so a duplicate overwrites itself. +- **Deduplication on consume**: record the IDs of processed messages and skip ones already seen. Keep the record for longer than the longest retry window. +- **Set, do not increment**: send absolute values ("battery is 71 %") rather than changes ("battery dropped 2 %"). +- **Version checks**: apply an update only if it is newer than the stored version. This also handles messages that arrive out of order. + +```mermaid +sequenceDiagram + participant C as Client + participant S as Server + C->>S: POST /orders (key=abc) + S->>S: create order, store key abc + S--xC: reply lost + C->>S: retry POST /orders (key=abc) + S-->>C: 201, the same order as before +``` + +[[CAPTION:Figure]] An idempotency key turns a retried request into a harmless repeat. + +## Practical points + +- Do the deduplication check and the work in the same transaction where possible; otherwise a crash between them reopens the gap. +- Side effects outside your store (sending an email, calling another API) need their own idempotency, or an outbox that records them first. +- Retries need backoff with jitter, or a fleet of clients retrying at once becomes its own outage. + +## Where next + +- [The saga pattern](saga-pattern.md) relies on idempotent steps and compensations. +- [MQTT](mqtt.md) and [Kafka](kafka-event-streaming.md) describe the delivery guarantees you are designing for. diff --git a/content/primers/index.md b/content/primers/index.md index 34cecb3..cfa6e79 100644 --- a/content/primers/index.md +++ b/content/primers/index.md @@ -5,6 +5,63 @@ author: Mark Bacon # Primers -Short introductions to architecture, cloud, IoT, security, tools and methods: the Tech 101s. +Short introductions to the topics that come up again and again in software, cloud and embedded engineering. Each primer explains what something is, why it matters and the main ideas, in a few minutes' reading, and then points to where to go deeper. They used to be called the Tech 101s. -This section is moving in from technical-documentation's primers and site-web-content-src's Tech 101s. +The primers are introductions, not rules. The [Guidelines](../guidelines/index.md) and [Standards](../standards/index.md) sections say how things are done here; the [Guides](../guides/index.md) and the [Handbook](../handbook/index.md) go into depth; the [Glossary](../glossary/index.md) defines the terms. + +## The areas + +| Area | Covers | +| --- | --- | +| [Architecture](architecture.md) | Decision records, C4, design principles and patterns, anti-patterns, hexagonal architecture, domain-driven design, event-driven architecture, monoliths and microservices, device shadows | +| [Distributed systems](distributed-systems.md) | CAP and PACELC, consistency models and CRDTs, idempotency, sagas | +| [APIs and data](apis-and-data.md) | REST, OpenAPI and AsyncAPI, JSON, YAML, Protocol Buffers, CBOR | +| [Cloud](cloud.md) | CI/CD, infrastructure as code, serverless, Kafka, IoT on Google Cloud | +| [IoT and embedded](iot-and-embedded.md) | Embedded design, RTOS, state machines, interrupts and DMA, watchdogs, bootloaders, MQTT, CoAP, LwM2M, cellular LPWAN, firmware updates, edge computing, hardware-in-the-loop testing | +| [Security](security.md) | Cryptography, PKI, TLS and DTLS, post-quantum cryptography, secure coding, OWASP Top 10, OAuth and OpenID Connect, zero trust, ETSI EN 303 645, ISO/IEC 30141 | +| [Methods and delivery](methods-and-delivery.md) | Life cycle models, agile, Scrum and Kanban, event storming, architecture evaluations, testing, code review, technical debt, docs as code, root cause analysis | +| [Programming and tools](programming-and-tools.md) | Git, static analysis, PlantUML and Mermaid, Python, Rust, the terminal | + +[[CAPTION:Table]] The primer areas. + +## Suggested reading paths + +If you are new to an area, these orders build on each other. + +**Embedded and IoT firmware** + +1. [Embedded systems design](embedded-systems-design.md) +2. [Real-time operating systems and FreeRTOS](rtos-and-freertos.md) +3. [Finite state machines](finite-state-machines.md) +4. [Interrupts and DMA](interrupts-and-dma.md) +5. [Cellular LPWAN](cellular-lpwan.md), then [CoAP](coap.md) or [MQTT](mqtt.md) +6. [CBOR and compact binary formats](cbor.md) +7. [Bootloaders](bootloaders.md) and [firmware over the air](firmware-over-the-air.md) +8. [ETSI EN 303 645](etsi-en-303-645.md) +9. [Hardware-in-the-loop testing](hardware-in-the-loop-testing.md) + +**Cloud and distributed systems** + +1. [Cloud primers](cloud.md) for the service models +2. [Domain-driven design](domain-driven-design.md) +3. [Modular monolith or microservices](modular-monolith-vs-microservices.md) +4. [Event-driven architecture](event-driven-architecture.md) and [Kafka](kafka-event-streaming.md) +5. [CAP and PACELC](cap-and-pacelc.md), [idempotency](idempotency.md) and [sagas](saga-pattern.md) +6. [Infrastructure as code](infrastructure-as-code.md) and [CI/CD pipelines](cicd-pipelines.md) + +**Security** + +1. [Cryptography basics](cryptography-basics.md) +2. [PKI and certificates](pki-and-certificates.md) and [TLS and DTLS](tls-and-dtls.md) +3. [Secure coding practices](secure-coding-practices.md) and the [OWASP Top 10](owasp-top-10.md) +4. [OAuth 2.0 and OpenID Connect](oauth-and-oidc.md) +5. [Zero trust architecture](zero-trust-architecture.md) + +**Technical leadership** + +1. [Agile development](agile-development.md) and [Scrum and Kanban](scrum-vs-kanban.md) +2. [Architecture decision records](architecture-decision-records.md) and [the C4 model](c4-model.md) +3. [Lightweight architecture evaluations](lightweight-architecture-evaluations.md) +4. [Technical debt](technical-debt.md) and [code review](code-review.md) +5. [Root cause analysis](root-cause-analysis.md) +6. [Docs as code](docs-as-code.md) diff --git a/content/primers/infrastructure-as-code.md b/content/primers/infrastructure-as-code.md new file mode 100644 index 0000000..708e3a4 --- /dev/null +++ b/content/primers/infrastructure-as-code.md @@ -0,0 +1,62 @@ +--- +title: Infrastructure as code +--- + +# Infrastructure as code + +Infrastructure as code (IaC) means describing servers, networks, databases and permissions in text files kept in version control, and letting a tool create and change the real resources to match. It replaces clicking through a web console, which is quick once and impossible to repeat exactly. + +## Why it matters + +- **Repeatable**: the same files build development, staging and production, so they really are alike. +- **Reviewable**: infrastructure changes go through pull requests, with a diff and an approver. +- **Recoverable**: if an environment is lost, rebuild it from the files. +- **Documented**: the files are an accurate description of what exists. + +## Declarative and imperative + +- **Declarative** tools take a description of the desired end state and work out what to change. Terraform and OpenTofu, AWS CloudFormation, Azure Bicep and Kubernetes manifests work this way. +- **Imperative** tools run steps in order. Shell scripts and, partly, Ansible playbooks are like this. +- **General-purpose languages**: Pulumi and the AWS CDK let you write infrastructure in TypeScript, Python or Go, and still produce a declarative plan. + +Declarative tools are usually the better default, because running them twice is safe and they can show a plan before changing anything. + +## How Terraform works + +```hcl +resource "google_pubsub_topic" "telemetry" { + name = "device-telemetry" + labels = { + environment = var.environment + } +} +``` + +[[CAPTION:Figure]] A Terraform resource. + +```mermaid +flowchart LR + A[Edit .tf files] --> B[terraform plan] + B --> C[Review the plan in a pull request] + C --> D[terraform apply] + D --> E[(State file)] + E --> B +``` + +[[CAPTION:Figure]] The plan and apply cycle. + +Terraform keeps a state file that maps the code to real resources. Store it remotely (an object storage bucket with locking), never in Git, and never edit it by hand. + +## Good habits + +- Run `plan` and `apply` from the [CI/CD pipeline](cicd-pipelines.md), not from laptops, so every change is recorded. +- Use modules for repeated patterns, but keep them shallow; deeply nested modules are hard to change. +- Keep secrets out of the files. Reference a secrets manager instead. +- Check the files with linters and policy tools (`tflint`, Checkov, Open Policy Agent) to catch insecure settings before they exist. +- Detect drift: schedule a `plan` to spot changes made by hand outside the code. +- Separate configuration management (what is installed on a machine) from provisioning (what machines exist). Better still, build immutable images and replace machines instead of changing them. + +## Where next + +- [CI/CD pipelines](cicd-pipelines.md) runs IaC changes safely. +- The [Guides](../guides/index.md) section covers environment setup in more detail, and the [Tech Radar](../radar/index.md) shows which IaC tools are recommended. diff --git a/content/primers/interrupts-and-dma.md b/content/primers/interrupts-and-dma.md new file mode 100644 index 0000000..73beca3 --- /dev/null +++ b/content/primers/interrupts-and-dma.md @@ -0,0 +1,77 @@ +--- +title: Interrupts and DMA +--- + +# Interrupts and DMA + +Firmware has to respond to hardware: a byte arriving on a serial port, a timer expiring, a button being pressed. Checking each peripheral in a loop (polling) wastes CPU time and power, and can miss events. Interrupts let the hardware tell the CPU that something needs attention. Direct memory access (DMA) goes further, moving data between peripherals and memory with no CPU involvement at all. + +## Interrupts + +When a peripheral raises an interrupt, the CPU finishes its current instruction, saves its context, and jumps to an interrupt service routine (ISR). When the ISR returns, the interrupted code carries on as if nothing had happened. On Arm Cortex-M parts, the Nested Vectored Interrupt Controller (NVIC) handles priorities, so a more urgent interrupt can pre-empt a less urgent one. + +### Rules for interrupt handlers + +- **Keep them short**: acknowledge the hardware, capture the data, signal a task or set a flag, and return. Do the processing outside the ISR. +- **No blocking**: never wait, sleep or take a mutex in an ISR. Use only the interrupt-safe API calls of your RTOS (the `FromISR` functions in FreeRTOS). +- **Share data carefully**: variables written by an ISR and read elsewhere must be `volatile`, and multi-byte updates need protection (briefly disabling interrupts, or atomic operations), or the main code can read half-updated values. +- **Clear the interrupt flag**, or the ISR runs again immediately and the system locks up. +- **Set priorities deliberately**: give the most time-critical source the highest priority, and know your worst-case latency. + +```c +static volatile uint32_t ticks; +static SemaphoreHandle_t rx_ready; + +void USART1_IRQHandler(void) +{ + BaseType_t woken = pdFALSE; + if (USART1->SR & USART_SR_RXNE) { + ring_buffer_put(&rx, USART1->DR); /* reading DR clears the flag */ + xSemaphoreGiveFromISR(rx_ready, &woken); + } + portYIELD_FROM_ISR(woken); +} +``` + +[[CAPTION:Figure]] A serial receive interrupt that defers processing to a task. + +## DMA + +A DMA controller copies data between memory and peripherals (or memory to memory) on its own, and raises an interrupt when a transfer is complete or half complete. The CPU sets up the transfer and is then free to do other work or to sleep. + +Typical uses: + +- Receiving serial data into a buffer without an interrupt per byte. +- Streaming ADC samples into memory at a fixed rate. +- Sending a display frame over SPI. +- Moving audio samples to and from a codec. + +A common pattern is **double buffering** (or a circular buffer with half-transfer interrupts): DMA fills one half while the CPU processes the other. + +```mermaid +sequenceDiagram + participant DMA + participant CPU + DMA->>DMA: fill buffer A + DMA-->>CPU: half-transfer interrupt + par + DMA->>DMA: fill buffer B + and + CPU->>CPU: process buffer A + end + DMA-->>CPU: transfer-complete interrupt +``` + +[[CAPTION:Figure]] Double buffering with DMA. + +### Things to watch + +- **Caches**: on cores with a data cache (Cortex-M7, for example), clean the cache before a DMA transfer out of memory and invalidate it after a transfer in, or place DMA buffers in non-cacheable memory. +- **Alignment and placement**: some DMA controllers can only reach certain memory regions or need aligned buffers. +- **Ownership**: never touch a buffer while DMA is using it. +- **Errors**: handle overrun and transfer-error interrupts, not just completion. + +## Where next + +- [Real-time operating systems and FreeRTOS](rtos-and-freertos.md) shows how ISRs hand work to tasks. +- [Embedded systems design](embedded-systems-design.md) covers power and memory, which interrupts and DMA both help. diff --git a/content/primers/iot-and-embedded.md b/content/primers/iot-and-embedded.md new file mode 100644 index 0000000..3f10760 --- /dev/null +++ b/content/primers/iot-and-embedded.md @@ -0,0 +1,41 @@ +--- +title: IoT and embedded primers +--- + +# IoT and embedded primers + +An embedded system is a computer built into a product to do one job: a microcontroller in a sensor, a thermostat, a motor drive. The Internet of Things (IoT) connects such devices to networks and to services in the cloud. Building them means working with small memories, limited power, unreliable links and hardware that is hard to reach once it is installed, and the devices must stay secure for years. + +```mermaid +flowchart LR + S[Sensors and actuators] --> M[Microcontroller and firmware] + M -- radio or wired link --> G[Gateway or edge] + G -- internet --> C[Cloud platform] + C --> A[Applications and users] +``` + +[[CAPTION:Figure]] The layers of a typical IoT system. + +## On the device + +- [Embedded systems design](embedded-systems-design.md): constraints, layering and power. +- [Real-time operating systems and FreeRTOS](rtos-and-freertos.md): tasks, scheduling and when you need an RTOS. +- [Finite state machines](finite-state-machines.md): the most useful pattern in firmware. +- [Interrupts and DMA](interrupts-and-dma.md): reacting to hardware without wasting the CPU. +- [Watchdog timers](watchdog-timers.md): recovering when software hangs. +- [Bootloaders](bootloaders.md): starting safely and accepting new firmware. + +## Connecting devices + +- [MQTT](mqtt.md): publish and subscribe over TCP. +- [CoAP](coap.md): REST for constrained devices over UDP. +- [LwM2M](lwm2m.md): device management built on CoAP. +- [Cellular LPWAN](cellular-lpwan.md): NB-IoT and LTE-M. +- [Firmware over the air](firmware-over-the-air.md): updating devices in the field. +- [Edge computing for IoT](edge-computing-for-iot.md): processing data close to the devices. + +## Testing + +- [Hardware-in-the-loop testing](hardware-in-the-loop-testing.md): automated tests on real boards. + +Security standards for devices are in [Security primers](security.md), including [ETSI EN 303 645](etsi-en-303-645.md). The [Handbook](../handbook/index.md) covers device and platform architecture, and the [Guides](../guides/index.md) give practical how-tos. diff --git a/content/primers/iot-on-google-cloud.md b/content/primers/iot-on-google-cloud.md new file mode 100644 index 0000000..39c1506 --- /dev/null +++ b/content/primers/iot-on-google-cloud.md @@ -0,0 +1,60 @@ +--- +title: IoT on Google Cloud +--- + +# IoT on Google Cloud + +Google Cloud no longer has a dedicated IoT device service. Cloud IoT Core, which handled device registration and MQTT connections, was retired on 16 August 2023. An IoT platform on Google Cloud today is built from its general services, with device connectivity provided by an MQTT broker you run yourself or by a partner. The same pattern applies to any cloud: the provider's own IoT service, where it has one, is only one option. + +## A typical architecture + +```mermaid +flowchart LR + D[Devices] -- MQTT over TLS --> B[MQTT broker: self-hosted on GKE or a partner service] + B --> PS[Pub/Sub] + PS --> DF[Dataflow: clean, enrich, window] + PS --> CR[Cloud Run: rules and alerts] + DF --> BQ[(BigQuery: analytics)] + DF --> BT[(Bigtable: time series)] + CR --> FS[(Firestore: device state)] + BQ --> L[Looker and Vertex AI] +``` + +[[CAPTION:Figure]] Device data flowing through Google Cloud services. + +| Job | Google Cloud service | +| --- | --- | +| Device connections | An MQTT broker (EMQX, HiveMQ, Mosquitto) on GKE or Compute Engine, or a partner's managed service | +| Ingestion and fan-out | Pub/Sub | +| Stream and batch processing | Dataflow (Apache Beam) | +| Event-driven logic | Cloud Run and Cloud Run functions | +| Analytics | BigQuery | +| High-volume time series | Bigtable | +| Device state and metadata | Firestore or Cloud SQL | +| Machine learning | Vertex AI | +| Edge | Google Distributed Cloud, or your own edge software | + +[[CAPTION:Table]] Which service does which job. + +## What you now own + +Without a managed device service, these jobs are yours: + +- **Device identity**: per-device X.509 certificates or keys, issued in manufacturing or at first connection. See [PKI and certificates](pki-and-certificates.md). +- **Registry and shadows**: a database of devices with their configuration and last state. See [digital twins and device shadows](digital-twins-and-device-shadows.md). +- **Firmware updates**: a campaign service and storage for images. See [firmware over the air](firmware-over-the-air.md). +- **Broker operations**: scaling, high availability and patching of the MQTT broker, or a contract with the partner who runs it. + +This is more work than a managed service, but it avoids being stranded again if a provider withdraws one. + +## Good habits + +- Keep the device protocol standard ([MQTT](mqtt.md) or [LwM2M](lwm2m.md)) so the cloud side can change without touching devices in the field. +- Put a bridge between the broker and Pub/Sub, so the cloud processing knows nothing about device connections. +- Partition data by device and time from the start; BigQuery and Bigtable costs depend heavily on it. +- Use [infrastructure as code](infrastructure-as-code.md) for every resource. + +## Where next + +- [Edge computing for IoT](edge-computing-for-iot.md) covers what to process before data reaches the cloud. +- The [Handbook](../handbook/index.md) covers IoT platform architecture, and Google's [Pub/Sub documentation](https://cloud.google.com/pubsub/docs) is the starting point for the cloud side. diff --git a/content/primers/iso-30141.md b/content/primers/iso-30141.md new file mode 100644 index 0000000..d938929 --- /dev/null +++ b/content/primers/iso-30141.md @@ -0,0 +1,59 @@ +--- +title: ISO/IEC 30141 +--- + +# ISO/IEC 30141 + +ISO/IEC 30141, *Internet of Things (IoT): Reference architecture*, gives a common vocabulary and a set of standard views for describing IoT systems. It does not prescribe technologies. Its value is in giving architects, suppliers and customers the same names for the same things, and a checklist of concerns that an IoT design should address. The first edition was published in 2018 and a revised edition in 2024. + +## What it contains + +- **A conceptual model**: the main entities in any IoT system (physical entities, IoT devices with sensors and actuators, gateways, services, networks, users) and how they relate. +- **Characteristics**: the qualities an IoT system needs, grouped into trustworthiness (security, privacy, safety, reliability, resilience), architecture (modularity, scalability, heterogeneity) and function (accuracy, interoperability, real-time behaviour). +- **Reference models and architecture views**: functional, system, communications, information, usage and deployment views, in the style of ISO/IEC/IEEE 42010 architecture descriptions. + +## The domains + +The standard groups the parts of an IoT system into domains. The six-domain model from the 2018 edition is the best-known part: + +```mermaid +flowchart TB + UD[User domain: people and applications that use the system] + OMD[Operations and management domain: provisioning, monitoring, policy] + ASD[Application and service domain: business services, analytics] + RAID[Resource access and interchange domain: APIs and data sharing with outside parties] + SCD[Sensing and controlling domain: devices, gateways, local networks] + PED[Physical entity domain: the things being sensed or controlled] + UD --- ASD + OMD --- ASD + ASD --- RAID + ASD --- SCD + SCD --- PED +``` + +[[CAPTION:Figure]] The six domains of the ISO/IEC 30141 reference model. + +| Domain | Typical contents | +| --- | --- | +| Physical entity | Buildings, machines, vehicles, environments | +| Sensing and controlling | Sensors, actuators, microcontrollers, gateways, local networks | +| Application and service | Data processing, business logic, analytics, dashboards | +| Operations and management | Device registry, provisioning, updates, monitoring, security management | +| Resource access and interchange | External APIs, data sharing with partners, access control | +| User | People, organisations and systems that use the services | + +[[CAPTION:Table]] What belongs in each domain. + +## Using it + +- **As a checklist**: walk through each domain and characteristic when reviewing a design. Gaps in operations and management (provisioning, [updates](firmware-over-the-air.md), monitoring) are the most common. +- **As a shared vocabulary** in tenders and supplier documents, so "gateway" or "platform" means the same to everyone. +- **Alongside other models**: it complements the [C4 model](c4-model.md), which describes one specific system, and fits with security standards such as [ETSI EN 303 645](etsi-en-303-645.md) and the IEC 62443 series for industrial systems. + +It is a large, abstract document. For most teams, the domain model and the list of characteristics give most of the value; the full views are useful for formal architecture descriptions. + +## Where next + +- [Edge computing for IoT](edge-computing-for-iot.md) maps closely onto the sensing and application domains. +- The [Handbook](../handbook/index.md) applies these ideas to IoT platform design. +- The standard is available from [iso.org](https://www.iso.org/search.html?q=30141). diff --git a/content/primers/kafka-event-streaming.md b/content/primers/kafka-event-streaming.md new file mode 100644 index 0000000..8420676 --- /dev/null +++ b/content/primers/kafka-event-streaming.md @@ -0,0 +1,65 @@ +--- +title: Event streaming with Kafka +--- + +# Event streaming with Kafka + +Apache Kafka is a distributed event log. Producers append records to it; consumers read them at their own pace and can re-read them later. Unlike a traditional message queue, Kafka does not delete a message when it is read. It keeps records for a configured time (or forever), which makes it a shared, replayable history of what happened. + +Kafka was built at LinkedIn and open-sourced in 2011. It now underpins many [event-driven architectures](event-driven-architecture.md) and data pipelines. + +## Core ideas + +- **Topic**: a named stream of records, such as `device-telemetry`. +- **Partition**: each topic is split into partitions, each an ordered, append-only log. Partitions are how Kafka scales: they spread across brokers and consumers. +- **Key**: records with the same key go to the same partition, so they stay in order. Use the device ID as the key and each device's readings arrive in order. +- **Offset**: a record's position in its partition. Consumers track the offset they have reached. +- **Broker**: a Kafka server. A cluster has several, and each partition is replicated to more than one for safety. +- **Consumer group**: consumers that share the work of reading a topic. Each partition is read by one member of the group at a time. Different groups read the same topic independently. + +```mermaid +flowchart LR + P[Producers] --> T0[Partition 0] + P --> T1[Partition 1] + P --> T2[Partition 2] + subgraph G1[Consumer group: alerts] + C1[Consumer A] + C2[Consumer B] + end + subgraph G2[Consumer group: analytics] + C3[Consumer C] + end + T0 --> C1 + T1 --> C1 + T2 --> C2 + T0 --> C3 + T1 --> C3 + T2 --> C3 +``` + +[[CAPTION:Figure]] One topic, three partitions, two independent consumer groups. + +Since Kafka 4.0, clusters manage their own metadata with the KRaft protocol; ZooKeeper is no longer used. + +## Delivery guarantees + +- Producers choose how many replicas must confirm a write (`acks=all` for safety). +- Consumers get at-least-once delivery by default: if a consumer crashes before committing its offset, it reads some records again. Make processing [idempotent](idempotency.md). +- Idempotent producers and transactions give exactly-once processing within Kafka (read from one topic, write to another), but not for side effects outside it. + +## Ecosystem + +- **Kafka Connect** moves data between Kafka and databases, object stores and other systems without custom code. +- **Kafka Streams** and **Apache Flink** process streams: filtering, joining, windowed aggregation. +- **Schema registries** keep producers and consumers agreed on Avro, Protobuf or JSON Schema formats. +- Managed services (Confluent Cloud, Amazon MSK, Azure Event Hubs with its Kafka interface) remove most of the operational work. + +## When to use it + +Kafka suits high-volume streams, many independent consumers, replay, and feeding analytics. It is heavy for a few hundred messages a minute or for request-response work. It is also not a device protocol: devices usually speak [MQTT](mqtt.md) to a broker, and a bridge forwards their messages into Kafka. + +## Where next + +- [Event-driven architecture](event-driven-architecture.md) and [consistency models](consistency-models.md) give the design background. +- The [Handbook](../handbook/index.md) covers messaging platform choices. +- [kafka.apache.org/documentation](https://kafka.apache.org/documentation/) is the reference. diff --git a/content/primers/lightweight-architecture-evaluations.md b/content/primers/lightweight-architecture-evaluations.md new file mode 100644 index 0000000..92034a1 --- /dev/null +++ b/content/primers/lightweight-architecture-evaluations.md @@ -0,0 +1,67 @@ +--- +title: Lightweight architecture evaluations +--- + +# Lightweight architecture evaluations + +An architecture evaluation checks whether a design will meet the qualities that matter most (performance, availability, security, modifiability, cost) before the team commits to it. Formal methods such as ATAM take several days and a trained team. Lightweight evaluations keep the core idea and fit into a few hours, so they can be done regularly rather than once. + +## The core idea + +Most architecture problems come from qualities nobody stated precisely. "It must be fast" cannot be checked. A quality attribute scenario can: + +| Part | Example | +| --- | --- | +| Source | 50,000 devices | +| Stimulus | reconnect at once after a regional network outage | +| Environment | normal operation, peak hour | +| Artefact | the device connection service | +| Response | accepts all connections, queues their data, sheds no data | +| Measure | all devices reconnected within 10 minutes; no data lost | + +[[CAPTION:Table]] A quality attribute scenario. + +The evaluation walks the design through a handful of the most important scenarios and asks: does it handle this, and how do we know? + +## A simple process + +1. **Prepare** (an hour or two): the architect summarises the business drivers and the design, with [C4](c4-model.md) context and container diagrams and the key [ADRs](architecture-decision-records.md). +2. **Gather scenarios** with stakeholders, and pick the top five to ten by importance and difficulty. A quality attribute tree (quality, then refinement, then scenario) helps organise them. +3. **Analyse**: for each scenario, trace how the design responds. Record: + - **Risks**: decisions that may cause problems. + - **Non-risks**: decisions that are sound, and why. + - **Sensitivity points**: parameters that strongly affect a quality (queue size, timeout). + - **Trade-offs**: decisions that help one quality and hurt another. +4. **Report**: a short list of risks, grouped into themes, each with an owner and a next step. + +```mermaid +flowchart LR + D[Business drivers] --> S[Scenarios] + A[Architecture and decisions] --> AN[Analysis] + S --> AN + AN --> R[Risks and trade-offs] + R --> N[Actions, ADRs, spikes] +``` + +[[CAPTION:Figure]] The flow of a lightweight evaluation. + +## Formats + +- **Mini-ATAM**: the steps above in half a day. +- **Architecture review against a checklist**: quick and repeatable, though it can miss problems specific to the system. +- **Risk storming** (from Simon Brown): each participant marks risks on the architecture diagrams independently, then the group compares and discusses. +- **Decision review**: review each significant ADR when it is proposed. +- **Fitness functions**: automated tests that check an architectural quality continuously (dependency rules, response-time budgets) in the [CI pipeline](cicd-pipelines.md). + +## Tips + +- Do it early, when changes are cheap, and again when the design changes significantly. +- Include people who will build and run the system, not only architects. +- Separate finding risks from solving them; an evaluation that turns into a redesign session runs out of time. +- Follow up: a risk with no owner is not managed. + +## Where next + +- The [Handbook](../handbook/index.md) covers quality attributes and the full review process. +- The [Templates](../templates/index.md) section has an architecture review checklist. +- *Software Architecture in Practice* (Bass, Clements and Kazman) is the standard text on ATAM and quality attribute scenarios. diff --git a/content/primers/lwm2m.md b/content/primers/lwm2m.md new file mode 100644 index 0000000..d70dd85 --- /dev/null +++ b/content/primers/lwm2m.md @@ -0,0 +1,70 @@ +--- +title: LwM2M +--- + +# LwM2M + +LwM2M (Lightweight Machine to Machine) is a device management protocol from the Open Mobile Alliance, now maintained by OMA SpecWorks. It runs on [CoAP](coap.md) and gives a standard way for a server to provision, configure, monitor and update constrained devices. Where [MQTT](mqtt.md) moves messages and leaves their meaning to you, LwM2M defines both the transport and a data model for common device functions. + +Versions 1.0 (2017), 1.1 (2018) and 1.2 (2020) are in use. 1.1 added non-IP transports and LwM2M over TCP; 1.2 added gateway support and MQTT and HTTP transports. + +## The parts + +```mermaid +flowchart LR + D[LwM2M client on the device] -- 1. bootstrap --> BS[Bootstrap server] + BS -- credentials and server address --> D + D -- 2. register, update, report --> S[LwM2M server] + S -- read, write, execute, observe --> D +``` + +[[CAPTION:Figure]] Bootstrap and registration. + +- **Client**: runs on the device and exposes its resources. +- **Bootstrap server**: gives a new device the address and credentials of the management server it should use. This lets the same factory image work for any customer or region. +- **LwM2M server**: manages the device once registered. + +## The object model + +Everything on the device is addressed as `/object/instance/resource`. Objects are standardised and numbered: + +| Object | ID | Example resource | +| --- | --- | --- | +| Security | 0 | Server URI and keys | +| Server | 1 | Registration lifetime | +| Device | 3 | `/3/0/0` manufacturer, `/3/0/9` battery level, `/3/0/4` reboot | +| Connectivity monitoring | 4 | `/4/0/2` signal strength | +| Firmware update | 5 | `/5/0/1` package URI, `/5/0/2` update | +| Location | 6 | Latitude and longitude | +| Temperature (IPSO) | 3303 | `/3303/0/5700` sensor value | + +[[CAPTION:Table]] Some standard LwM2M objects. + +Hundreds of further objects are registered by OMA and other bodies, including the IPSO smart objects for common sensors. Vendors can define their own. + +## Operations + +- **Register and update**: the device announces itself and its objects, then refreshes its registration before its lifetime expires. A sleepy device uses queue mode, and the server holds requests until the device next checks in. +- **Read, write, execute**: the server reads values, changes configuration, or triggers actions such as a reboot. +- **Observe and notify**: the server subscribes to resources, and the device reports changes, with attributes such as minimum period or "only if it changes by more than 0.5". +- **Send** (1.1 and later): the device pushes data without being asked. + +Payloads use compact formats: TLV, SenML in JSON or [CBOR](cbor.md), and LwM2M CBOR in 1.2. + +## Firmware updates + +Object 5 defines a standard update state machine. The server gives the device a package, either by pushing it with block-wise transfer or by giving a URI to pull from. The device downloads and verifies it, the server triggers the update, and the device reports the result. Every compliant server and client agrees on this flow, which is one of LwM2M's main attractions. See [firmware over the air](firmware-over-the-air.md). + +## Security + +LwM2M uses DTLS with pre-shared keys, raw public keys or certificates, or OSCORE. Credentials are delivered at bootstrap, so devices can be re-keyed later without visiting them. See [TLS and DTLS](tls-and-dtls.md). + +## When to choose it + +LwM2M suits large fleets of constrained, often cellular, devices that need standard management, especially where devices and servers come from different suppliers. Open-source implementations include Eclipse Leshan (server), Eclipse Wakaama and Anjay (clients), and the client in Zephyr. + +## Where next + +- [CoAP](coap.md) covers the transport underneath. +- The [Handbook](../handbook/index.md) covers device management architecture, and the [Glossary](../glossary/index.md) defines the terms. +- OMA SpecWorks publishes the specifications and the object registry at [openmobilealliance.org](https://www.openmobilealliance.org/specifications/lwm2m). diff --git a/content/primers/methods-and-delivery.md b/content/primers/methods-and-delivery.md new file mode 100644 index 0000000..7e18cfb --- /dev/null +++ b/content/primers/methods-and-delivery.md @@ -0,0 +1,28 @@ +--- +title: Methods and delivery primers +--- + +# Methods and delivery primers + +How a team works decides what it can deliver as much as the technology it uses. These primers introduce the common ways of planning, building, checking and learning. + +## Planning and delivery + +- [Software development life cycles](sdlc-models.md): waterfall, the V-model, spiral and iterative approaches. +- [Agile development](agile-development.md): the values and practices behind short, feedback-driven cycles. +- [Scrum and Kanban](scrum-vs-kanban.md): the two most common ways to run an agile team. + +## Design and quality + +- [Event storming](event-storming.md): a workshop for exploring a business domain. +- [Lightweight architecture evaluations](lightweight-architecture-evaluations.md): checking a design against what matters. +- [Testing strategies](testing-strategies.md): what to test, at which level. +- [Code review](code-review.md): getting value from a second pair of eyes. +- [Technical debt](technical-debt.md): naming it, measuring it and paying it down. +- [Docs as code](docs-as-code.md): treating documentation like software. + +## Learning from problems + +- [Root cause analysis](root-cause-analysis.md): finding out why something went wrong, not just what. + +The [Guides](../guides/index.md) give step-by-step practice, and the [Templates](../templates/index.md) section has checklists for definitions of ready and done, reviews and postmortems. diff --git a/content/primers/modular-monolith-vs-microservices.md b/content/primers/modular-monolith-vs-microservices.md new file mode 100644 index 0000000..b986d80 --- /dev/null +++ b/content/primers/modular-monolith-vs-microservices.md @@ -0,0 +1,60 @@ +--- +title: Modular monolith or microservices +--- + +# Modular monolith or microservices + +How a system is split into deployable units is one of the most expensive decisions to reverse. The two main options are a modular monolith (one deployable unit with strong internal boundaries) and microservices (many small units, each deployed on its own). The right choice depends far more on the organisation than on the technology. + +## The options + +- **Monolith**: one codebase, one build, one deployment. Often blamed for problems that are really the lack of internal structure. +- **Modular monolith**: still one deployment, but split into modules with clear public interfaces, each owning its own data. Modules call each other in process. Boundaries are enforced by the build or by architecture tests. +- **Microservices**: each service has its own codebase or folder, its own data store and its own deployment, and services talk over the network. + +```mermaid +flowchart TB + subgraph Monolith["Modular monolith: one process"] + A[Devices module] --> B[Billing module] + A --> C[Alerts module] + end + subgraph Services["Microservices: many processes"] + D[Device service] -- HTTP or events --> E[Billing service] + D -- HTTP or events --> F[Alert service] + end +``` + +[[CAPTION:Figure]] The same three parts, in process and over the network. + +## The trade-offs + +| Concern | Modular monolith | Microservices | +| --- | --- | --- | +| Deployment | One pipeline; everything ships together | Independent releases per service | +| Scaling | Scale the whole application | Scale each service to its own load | +| Team autonomy | Teams share one release train | Teams own and release their services | +| Consistency | Local transactions across modules | Distributed; needs events and [sagas](saga-pattern.md) | +| Failure | One process; a crash takes everything down | Partial failure is normal and must be designed for | +| Operations | Simple: one thing to monitor | Service discovery, tracing, many pipelines, more infrastructure | +| Refactoring boundaries | Cheap: move code between modules | Expensive: change APIs and migrate data | + +[[CAPTION:Table]] How the two styles compare. + +## A sensible default + +Start with a modular monolith unless you have a clear reason not to. Get the module boundaries right (they usually follow [bounded contexts](domain-driven-design.md)), keep each module's data private, and make modules talk through interfaces or in-process events. When a module needs to scale, release or fail independently, or a separate team owns it, extract it as a service. Good modules make that a mechanical step. + +Reasons to start with microservices include several teams working at once from day one, parts with very different scaling or technology needs, or strict isolation requirements. + +## Warning signs + +- **The distributed monolith**: services that must be deployed together, or that share a database. You pay the costs of both styles and get the benefits of neither. +- **Nano-services**: services so small that every feature needs changes to several of them. +- **No platform**: microservices without automated deployment, monitoring and tracing become very hard to run. + +Conway's law applies: a system's structure tends to mirror the communication structure of the organisation that builds it. Choose boundaries that match the teams, or change the teams. + +## Where next + +- [Event-driven architecture](event-driven-architecture.md) is the usual way services stay loosely coupled. +- The [Handbook](../handbook/index.md) discusses decomposition and service boundaries in more depth. diff --git a/content/primers/mqtt.md b/content/primers/mqtt.md new file mode 100644 index 0000000..2402558 --- /dev/null +++ b/content/primers/mqtt.md @@ -0,0 +1,85 @@ +--- +title: MQTT +--- + +# MQTT + +MQTT is a lightweight publish-subscribe messaging protocol, and the most common way for IoT devices to talk to a cloud platform. Devices and applications connect to a central broker over TCP; publishers send messages to named topics; the broker forwards each message to every client that has subscribed to that topic. Senders and receivers never need to know about each other. + +It was designed by Andy Stanford-Clark (IBM) and Arlen Nipper in 1999 for monitoring oil pipelines over satellite links. It is now an OASIS standard: version 3.1.1 (also ISO/IEC 20922) and version 5.0 (2019). The name originally stood for MQ Telemetry Transport, but the standard now treats MQTT as just a name. + +## How it works + +```mermaid +flowchart LR + S1[Sensor] -- publish site/7/temp --> B{Broker} + S2[Sensor] -- publish site/9/temp --> B + B -- site/+/temp --> D[Dashboard] + B -- site/7/# --> A[Alerting for site 7] +``` + +[[CAPTION:Figure]] Publishers, a broker and subscribers with wildcard topics. + +- **Topics** are strings with levels separated by `/`, such as `site/7/temp`. They are created simply by publishing to them. +- **Wildcards**: `+` matches one level (`site/+/temp`), `#` matches everything below (`site/7/#`). Only subscribers can use wildcards. +- **The broker** handles connections, subscriptions, delivery and authentication. Mosquitto, EMQX, HiveMQ and VerneMQ are common; most clouds offer a managed one. + +## Quality of service + +| QoS | Guarantee | How | Cost | +| --- | --- | --- | --- | +| 0 | At most once | Send and forget | Cheapest; messages can be lost | +| 1 | At least once | Sender repeats until it gets `PUBACK` | Duplicates possible, so receivers must be [idempotent](idempotency.md) | +| 2 | Exactly once | Four-step exchange (`PUBREC`, `PUBREL`, `PUBCOMP`) | Most traffic and state; rarely worth it | + +[[CAPTION:Table]] MQTT quality of service levels. + +QoS applies to each hop separately: publisher to broker, and broker to each subscriber. + +## Features worth knowing + +- **Retained messages**: the broker keeps the last message on a topic and sends it to new subscribers at once. Useful for "current status". +- **Last will and testament**: a message the broker publishes if a client disconnects without saying goodbye, commonly used to mark a device offline. +- **Persistent sessions**: the broker remembers a client's subscriptions and queues messages while it is offline. +- **Keep-alive**: client and broker exchange pings so dead connections are detected. +- **MQTT 5 additions**: reason codes on every acknowledgement, user properties (headers), message expiry, shared subscriptions for load-balancing consumers, topic aliases to save bytes, and request-response support. + +## A Python client + +```python +import paho.mqtt.client as mqtt + +def on_connect(client, userdata, flags, reason_code, properties): + print("connected:", reason_code) + client.subscribe("site/+/temp", qos=1) + +def on_message(client, userdata, msg): + print(msg.topic, msg.payload.decode()) + +client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2) +client.on_connect = on_connect +client.on_message = on_message +client.tls_set() # use TLS in anything real +client.connect("test.mosquitto.org", 8883) +client.loop_forever() +``` + +[[CAPTION:Figure]] Subscribing with Eclipse Paho (version 2 of the Python library). + +## Good habits + +- Always use TLS (port 8883) and authenticate each device individually, ideally with a client certificate. +- Restrict each device to its own topics with broker access control lists. +- Design the topic tree up front: put the most general level first and the device identity early (`tenant/site/device/telemetry`). Do not put personal data in topic names. +- Use QoS 1 with idempotent consumers for most telemetry; QoS 0 for high-rate data where a lost sample does not matter. +- Keep payloads compact (consider [CBOR](cbor.md)) and include a timestamp and schema version. +- Reconnect with exponential backoff and jitter, so a broker restart does not cause a stampede. + +## MQTT and its neighbours + +MQTT needs TCP and a fairly steady connection. For very constrained devices or sleepy cellular links, [CoAP](coap.md) over UDP is lighter, and MQTT-SN is a variant for non-TCP networks. For device management (configuration, firmware updates) on top of messaging, see [LwM2M](lwm2m.md). For high-volume processing in the cloud, a bridge usually forwards MQTT into [Kafka](kafka-event-streaming.md). + +## Where next + +- The [Guides](../guides/index.md) and [Handbook](../handbook/index.md) cover messaging design in more depth; the [Glossary](../glossary/index.md) defines the terms. +- The [MQTT site](https://mqtt.org/) links to the specifications and to client libraries. diff --git a/content/primers/oauth-and-oidc.md b/content/primers/oauth-and-oidc.md new file mode 100644 index 0000000..8bc86d1 --- /dev/null +++ b/content/primers/oauth-and-oidc.md @@ -0,0 +1,75 @@ +--- +title: OAuth 2.0 and OpenID Connect +--- + +# OAuth 2.0 and OpenID Connect + +OAuth 2.0 (RFC 6749) is a framework for delegated authorisation. It lets an application act on a user's behalf, with limited permissions, without ever seeing the user's password. OpenID Connect (OIDC) is a thin layer on top that adds authentication, telling the application who the user is. Together they underpin "sign in with" buttons, single sign-on in organisations, and access control for most modern APIs. + +The distinction matters: OAuth answers "what may this client do?", and OIDC answers "who is the user?". + +## The roles + +- **Resource owner**: the user who owns the data. +- **Client**: the application that wants access. +- **Authorisation server**: signs the user in, asks for consent and issues tokens. Often an identity provider such as Microsoft Entra ID, Okta, Auth0 or Keycloak. +- **Resource server**: the API that accepts tokens. + +## The authorisation code flow with PKCE + +This is the flow to use for web, mobile and desktop applications. + +```mermaid +sequenceDiagram + participant U as User's browser + participant C as Client application + participant A as Authorisation server + participant R as API + C->>U: redirect to authorisation server (with PKCE challenge) + U->>A: sign in and consent + A->>U: redirect back with a one-time code + U->>C: code + C->>A: exchange code (with PKCE verifier) + A->>C: access token, refresh token, ID token + C->>R: request with access token + R->>C: data +``` + +[[CAPTION:Figure]] The authorisation code flow. + +PKCE (Proof Key for Code Exchange, RFC 7636) stops an intercepted code from being used by anyone else. It is required for public clients and recommended for all. + +## Other flows + +| Flow | Use | +| --- | --- | +| Client credentials | Service-to-service calls with no user | +| Device authorisation (RFC 8628) | Devices without a browser or keyboard, such as a TV or an IoT gateway: the user approves on their phone using a short code | +| Refresh token | Getting a new access token without asking the user again | +| Implicit, resource owner password | Deprecated; do not use | + +[[CAPTION:Table]] Which OAuth flow to use when. + +## Tokens + +- **Access token**: presented to the API. Often a signed JWT (JSON Web Token) containing the user, the client, the scopes and an expiry. Keep lifetimes short (minutes). +- **Refresh token**: a long-lived credential for getting new access tokens. Store it securely and rotate it on use. +- **ID token** (OIDC only): a JWT for the client that describes the sign-in: who the user is (`sub`), who issued it (`iss`), for which client (`aud`), and when. + +**Scopes** limit what an access token allows (`devices:read`). OIDC adds standard scopes (`openid`, `profile`, `email`) and a UserInfo endpoint. + +## Common mistakes + +- Using an ID token to call an API, or an access token to identify a user. +- Not validating JWTs properly on the API: check the signature, issuer, audience and expiry, and accept only the expected algorithm. +- Storing tokens where scripts can read them in a browser. Prefer a back-end-for-front-end pattern that keeps tokens on the server. +- Redirect URIs that are not matched exactly, which lets an attacker capture codes. +- Writing your own authorisation server. Use a mature product or library. + +The OAuth 2.0 Security Best Current Practice (RFC 9700, 2025) and the draft OAuth 2.1 gather the current advice in one place. + +## Where next + +- [Zero trust architecture](zero-trust-architecture.md) builds on strong identity. +- The [Guides](../guides/index.md) section covers integrating with an identity provider. +- [oauth.net](https://oauth.net/2/) links to all the specifications. diff --git a/content/primers/openapi-and-asyncapi.md b/content/primers/openapi-and-asyncapi.md new file mode 100644 index 0000000..4695a52 --- /dev/null +++ b/content/primers/openapi-and-asyncapi.md @@ -0,0 +1,95 @@ +--- +title: OpenAPI and AsyncAPI +--- + +# OpenAPI and AsyncAPI + +An interface description is a machine-readable file that says exactly what an API offers: its operations or channels, the messages and their schemas, and how to authenticate. Writing it down as a file makes the API a reviewable contract and lets tools generate documentation, client code, mock servers and tests from one source. + +- **OpenAPI** describes request-response HTTP APIs. It started as Swagger and is now run by the OpenAPI Initiative under the Linux Foundation. Version 3.1 aligned its schemas with JSON Schema. +- **AsyncAPI** describes event-driven and message-based interfaces: MQTT, Kafka, AMQP, WebSockets. It deliberately looks like OpenAPI, so the skills transfer. + +## OpenAPI in brief + +```yaml +openapi: 3.1.0 +info: + title: Device API + version: 1.2.0 +paths: + /devices/{id}: + get: + summary: Read one device + parameters: + - name: id + in: path + required: true + schema: {type: string} + responses: + "200": + description: The device + content: + application/json: + schema: {$ref: "#/components/schemas/Device"} + "404": + description: No such device +components: + schemas: + Device: + type: object + required: [id, status] + properties: + id: {type: string} + status: {type: string, enum: [online, offline]} +``` + +[[CAPTION:Figure]] A minimal OpenAPI 3.1 description. + +## AsyncAPI in brief + +AsyncAPI 3.0 separates channels (where messages go), operations (whether this application sends or receives) and messages (what is sent): + +```yaml +asyncapi: 3.0.0 +info: + title: Device telemetry + version: 1.0.0 +channels: + telemetry: + address: devices/{deviceId}/telemetry + parameters: + deviceId: {} + messages: + reading: + payload: + type: object + properties: + temperature: {type: number} + timestamp: {type: string, format: date-time} +operations: + receiveTelemetry: + action: receive + channel: {$ref: "#/channels/telemetry"} +``` + +[[CAPTION:Figure]] A minimal AsyncAPI 3.0 description of an MQTT topic. + +## Design first or code first + +- **Design first**: write the description, review it with the consumers, then build. Better APIs, because the contract gets attention before anything depends on it. +- **Code first**: annotate the code and generate the description. Quicker to start, but the API tends to reflect the implementation. + +For any API used outside the team, design first is worth the effort. Either way, keep the file in version control and check it in CI. + +## Tooling + +- Linters such as Spectral check descriptions against style rules. +- Breaking-change detectors (`oasdiff` for OpenAPI) compare versions in a pull request. +- Generators produce client libraries and server stubs; Redocly, Swagger UI and the AsyncAPI generator produce documentation. +- Mock servers (Prism, Microcks) let consumers start before the API exists. + +## Where next + +- [REST APIs](rest-apis.md) and [event-driven architecture](event-driven-architecture.md) give the background. +- The [Guidelines](../guidelines/index.md) section sets the API conventions used here. +- The specifications are at [spec.openapis.org](https://spec.openapis.org/) and [asyncapi.com](https://www.asyncapi.com/docs/reference/specification/latest). diff --git a/content/primers/owasp-top-10.md b/content/primers/owasp-top-10.md new file mode 100644 index 0000000..802c40e --- /dev/null +++ b/content/primers/owasp-top-10.md @@ -0,0 +1,46 @@ +--- +title: OWASP Top 10 +--- + +# OWASP Top 10 + +The OWASP Top 10 is a list of the most critical security risks to web applications, published by the Open Worldwide Application Security Project (OWASP), a non-profit community. It is based on data from real application testing plus a survey of practitioners, and it is updated every few years. It is an awareness document, not a complete standard: an application that avoids all ten can still be insecure. For a fuller checklist, use the OWASP Application Security Verification Standard (ASVS). + +## The 2021 list + +| Rank | Risk | In short | Main defences | +| --- | --- | --- | --- | +| A01 | Broken access control | Users can act outside their permissions, such as reading another user's data by changing an ID | Deny by default; check authorisation on the server for every request; test it | +| A02 | Cryptographic failures | Sensitive data exposed through missing or weak encryption | TLS everywhere; strong algorithms; encrypt sensitive data at rest; proper key management | +| A03 | Injection | Untrusted data interpreted as code: SQL, command, cross-site scripting | Parameterised queries; output encoding; input validation | +| A04 | Insecure design | Flaws in the design itself, which perfect code cannot fix | Threat modelling; secure design patterns; abuse cases | +| A05 | Security misconfiguration | Default accounts, verbose errors, open cloud storage, unnecessary features | Hardened, automated configuration; minimal installs; regular review | +| A06 | Vulnerable and outdated components | Libraries and frameworks with known vulnerabilities | Inventory (SBOM); dependency scanning; prompt patching | +| A07 | Identification and authentication failures | Weak passwords, credential stuffing, poor session handling | Multi-factor authentication; rate limiting; secure session management | +| A08 | Software and data integrity failures | Unsigned updates, compromised build pipelines, unsafe deserialisation | Signed artefacts; protected CI/CD; verified dependencies | +| A09 | Security logging and monitoring failures | Attacks go unnoticed | Log security events; alert on them; practise incident response | +| A10 | Server-side request forgery (SSRF) | The server is tricked into fetching a URL chosen by the attacker, often reaching internal systems | Allow-list destinations; block internal addresses; separate networks | + +[[CAPTION:Table]] The OWASP Top 10, 2021 edition. + +## The 2025 edition + +OWASP released a new edition in late 2025. Broken access control stays first. Misconfiguration rises to second, reflecting how much behaviour now lives in configuration. Vulnerable components broaden into software supply chain failures, covering build systems and distribution as well as dependencies. SSRF is folded into access control, and a new category covers the mishandling of exceptional conditions: errors that fail open or leak information. The ideas above still apply; check [owasp.org/Top10](https://owasp.org/Top10/) for the current text. + +## Other OWASP lists + +- **API Security Top 10**: risks specific to APIs, led by broken object-level authorisation, the API form of A01. +- **Mobile Top 10** and **IoT Top 10**: the same idea for mobile apps and connected devices. The IoT list (2018) opens with weak or hard-coded passwords, which [ETSI EN 303 645](etsi-en-303-645.md) also tackles first. +- **Top 10 for LLM Applications**: risks such as prompt injection in systems built on large language models. + +## Using it + +- Use it to train developers and to structure security reviews and tests. +- Map each category to concrete controls in your own standards and to automated checks in the [CI/CD pipeline](cicd-pipelines.md). +- Do not treat it as the whole job. Threat modelling, [secure coding practices](secure-coding-practices.md) and testing are still needed. + +## Where next + +- [Secure coding practices](secure-coding-practices.md) covers the habits that prevent most of these risks. +- [OAuth 2.0 and OpenID Connect](oauth-and-oidc.md) addresses much of A07. +- The [Guidelines](../guidelines/index.md) section sets the security rules used here. diff --git a/content/primers/pki-and-certificates.md b/content/primers/pki-and-certificates.md new file mode 100644 index 0000000..588ea64 --- /dev/null +++ b/content/primers/pki-and-certificates.md @@ -0,0 +1,66 @@ +--- +title: PKI and certificates +--- + +# PKI and certificates + +Public-key cryptography lets anyone verify a signature with a public key, but it does not say whose key it is. A public key infrastructure (PKI) solves that. Trusted certificate authorities (CAs) sign certificates that bind a public key to an identity, such as a domain name or a device serial number. If you trust the CA, you can trust the binding. + +## X.509 certificates + +Almost all certificates use the X.509 format. The important fields are: + +| Field | Meaning | +| --- | --- | +| Subject | Who the certificate is for: a domain, an organisation, a device ID | +| Subject Alternative Name | The names the certificate is actually valid for (browsers check this, not the subject) | +| Public key | The key being certified | +| Issuer | The CA that signed it | +| Validity | Not before and not after dates | +| Serial number | Unique per issuer; used for revocation | +| Key usage, extended key usage | What the key may do: sign, encrypt, act as a TLS server or client | +| Basic constraints | Whether this certificate may itself sign certificates (is a CA) | +| Signature | The issuer's signature over all of the above | + +[[CAPTION:Table]] The main fields of an X.509 certificate. + +`openssl x509 -in cert.pem -text -noout` prints them all. + +## Chains of trust + +```mermaid +flowchart TB + R[Root CA: self-signed, kept offline, long-lived] --> I[Intermediate CA: online, issues certificates] + I --> S[Server certificate: api.example.com] + I --> D[Device certificate: serial 0042] +``` + +[[CAPTION:Figure]] A root, an intermediate and two end-entity certificates. + +A verifier holds the root certificate in its trust store. The other side presents its own certificate and the intermediates; the verifier checks each signature up to the root, plus dates, names and key usage. Keeping the root offline and issuing from intermediates means a compromised intermediate can be replaced without changing every trust store. + +## The lifecycle + +1. **Key generation**: ideally on the device or server itself, inside secure hardware where possible, so the private key never leaves it. +2. **Certificate signing request (CSR)**: a request containing the public key and the requested identity, signed with the private key to prove possession. +3. **Issuance**: the CA checks the request and signs the certificate. +4. **Use**: for [TLS](tls-and-dtls.md) authentication, code signing, or signing data. +5. **Renewal**: before expiry, ideally automatically. Short lifetimes limit the damage from a stolen key; public web certificates are moving to lifetimes of weeks. +6. **Revocation**: if a key is compromised, the CA revokes the certificate. Verifiers learn this from a certificate revocation list (CRL) or by asking an OCSP (Online Certificate Status Protocol) responder. Constrained devices often cannot check either, which is another reason to keep lifetimes short. + +## Automation protocols + +- **ACME** (RFC 8555): automated issuance and renewal, used by Let's Encrypt for web servers. +- **EST** (Enrollment over Secure Transport, RFC 7030): enrolment and renewal over HTTPS, common for devices. +- **CMP** (Certificate Management Protocol, RFC 4210 and its 2023 update RFC 9480): a fuller protocol used in industrial and telecoms PKI. +- **SCEP**: older and still common in enterprise device management. + +## PKI for device fleets + +Each device gets its own key pair and certificate, often in the factory, giving it an identity that cannot be copied to another device. Typical designs use a manufacturing CA to issue birth certificates, then enrol the device with an operational CA when it is first installed. Keep CA private keys in hardware security modules, and plan for how a root will be replaced long before it expires. + +## Where next + +- [Cryptography basics](cryptography-basics.md) explains signatures and key pairs. +- [TLS and DTLS](tls-and-dtls.md) shows certificates in use. +- The [Guides](../guides/index.md) section covers running a device PKI in practice. diff --git a/content/primers/plantuml-and-mermaid.md b/content/primers/plantuml-and-mermaid.md new file mode 100644 index 0000000..f1e9953 --- /dev/null +++ b/content/primers/plantuml-and-mermaid.md @@ -0,0 +1,75 @@ +--- +title: PlantUML and Mermaid +--- + +# PlantUML and Mermaid + +PlantUML and Mermaid draw diagrams from text. You write a short description of the boxes and arrows; the tool lays them out and renders an image. Because the source is plain text, diagrams live in version control next to the code and changes show up as readable diffs in review. Nobody needs a drawing tool licence to fix a label. + +Every diagram on this site is written this way. + +## Mermaid + +Mermaid is written in JavaScript and renders in the browser. GitHub, GitLab, Azure DevOps, Notion, Obsidian and MkDocs Material all display Mermaid blocks in Markdown directly, which makes it the easiest choice for documentation in repositories. + +````markdown +```mermaid +sequenceDiagram + participant D as Device + participant B as Broker + D->>B: CONNECT + B-->>D: CONNACK + D->>B: PUBLISH telemetry +``` +```` + +[[CAPTION:Figure]] A Mermaid sequence diagram in a Markdown file. + +It supports flowcharts, sequence, class, state, entity-relationship, Gantt, pie, mind map, timeline, Git graph and quadrant charts, and has experimental C4 and architecture diagrams. + +## PlantUML + +PlantUML is written in Java and uses Graphviz for some layouts. It is older, covers more of UML, and gives finer control over layout and styling. + +```plantuml +@startuml +[*] --> Idle +Idle --> Connecting : report due +Connecting --> Sending : connected +Connecting --> Backoff : failed +Backoff --> Connecting : retry +Sending --> Idle : acknowledged +@enduml +``` + +[[CAPTION:Figure]] A PlantUML state diagram. + +Beyond UML (sequence, use case, class, activity, component, state, deployment, timing), it draws network diagrams, Gantt charts, mind maps, wireframes, and JSON and YAML data. Its standard library adds [C4](c4-model.md) diagrams and cloud provider icons. + +## Choosing + +| | Mermaid | PlantUML | +| --- | --- | --- | +| Rendering | In the browser, or `mmdc` on the command line | Java (`plantuml.jar`), a server, or a container | +| Native support in GitHub and GitLab | Yes | No; needs a build step or a proxy server | +| Diagram range | Broad and growing | Broader, especially UML and C4 | +| Layout control | Limited | More options (direction, grouping, hidden links, skin parameters) | +| Best for | Diagrams in READMEs and docs viewed on code hosts | Detailed architecture and design documents built by a pipeline | + +[[CAPTION:Table]] Mermaid and PlantUML compared. + +Many teams use both: Mermaid for quick diagrams in repositories, PlantUML for richer architecture documents. Graphviz, D2 and Structurizr are other diagrams-as-code tools worth knowing. + +## Good habits + +- **One idea per diagram**, with a title. If it needs a legend to decode, it is probably two diagrams. +- **Let the tool lay out the diagram**; fighting the layout engine wastes time. Change the structure or the direction instead. +- **Keep styling in shared include files or themes**, not repeated in every diagram. +- **Check diagrams in CI** so a syntax error fails the build rather than the published page. +- **Render to SVG** for the web and PNG for documents. + +## Where next + +- [Docs as code](docs-as-code.md) puts diagrams into a documentation pipeline. +- [The C4 model](c4-model.md) gives a structure for architecture diagrams. +- [mermaid.js.org](https://mermaid.js.org/) and [plantuml.com](https://plantuml.com/) have the full syntax. diff --git a/content/primers/post-quantum-cryptography.md b/content/primers/post-quantum-cryptography.md new file mode 100644 index 0000000..04a297d --- /dev/null +++ b/content/primers/post-quantum-cryptography.md @@ -0,0 +1,46 @@ +--- +title: Post-quantum cryptography +--- + +# Post-quantum cryptography + +A large enough quantum computer could break the public-key algorithms used today. RSA, elliptic-curve Diffie-Hellman and ECDSA all rest on problems (factoring and discrete logarithms) that Shor's algorithm solves efficiently on such a machine. No quantum computer that can do this exists yet, but data and devices last a long time, so the change to new algorithms has already started. Post-quantum cryptography (PQC) means algorithms that run on ordinary computers and are believed to resist quantum attacks. + +## What is affected + +| Primitive | Quantum impact | Action | +| --- | --- | --- | +| RSA, ECDH, ECDSA, Ed25519 | Broken | Replace with post-quantum algorithms | +| AES | Weakened (Grover's algorithm roughly halves the effective key length) | Use 256-bit keys for long-term protection | +| SHA-2, SHA-3 | Slightly weakened | Existing output sizes (256 bits or more) remain fine | + +[[CAPTION:Table]] How quantum computing affects common primitives. + +## The new standards + +NIST published the first post-quantum standards in August 2024: + +- **ML-KEM** (FIPS 203, based on CRYSTALS-Kyber): key encapsulation, which replaces ECDH-style key exchange. +- **ML-DSA** (FIPS 204, based on CRYSTALS-Dilithium): the general-purpose signature scheme. +- **SLH-DSA** (FIPS 205, based on SPHINCS+): a hash-based signature scheme with conservative security assumptions, but large signatures. + +A further signature scheme based on FALCON (FN-DSA) and a backup key encapsulation scheme (HQC) are on the way. For firmware signing, the stateful hash-based schemes **LMS** and **XMSS** (NIST SP 800-208) are already approved, and suit [bootloaders](bootloaders.md) well because verification is simple. The signer must never reuse a state, which needs careful key management. + +The new algorithms have much larger keys and signatures than elliptic curves: an ML-DSA signature is a few kilobytes against 64 bytes for Ed25519. That matters for certificates, constrained devices and [narrow radio links](cellular-lpwan.md). + +## Harvest now, decrypt later + +An attacker can record encrypted traffic today and decrypt it once a quantum computer exists. Any data that must stay secret for ten years or more is already at risk. That is why key exchange is moving first. Hybrid key exchange combines X25519 with ML-KEM, so the connection is safe if either holds. It is already the default in major browsers and in recent OpenSSH and TLS libraries. + +## Preparing + +- **Inventory**: find where public-key cryptography is used: TLS, VPNs, code signing, device identity, stored data. +- **Crypto agility**: make algorithms configurable and updatable rather than fixed in code or hardware. A device that cannot change its signature algorithm through an update may have to be replaced. +- **Prioritise**: long-lived secrets and long-lived devices first. A product designed today may still be in service when quantum attacks become practical. +- **Follow guidance**: the UK National Cyber Security Centre has set out a timeline with migration complete by 2035, and similar plans exist in the US and EU. + +## Where next + +- [Cryptography basics](cryptography-basics.md) explains the primitives being replaced. +- The [Standards](../standards/index.md) section will record the algorithms approved here. +- NIST's [post-quantum cryptography project](https://csrc.nist.gov/projects/post-quantum-cryptography) has the standards and the status of the remaining candidates. diff --git a/content/primers/programming-and-tools.md b/content/primers/programming-and-tools.md new file mode 100644 index 0000000..ddf82be --- /dev/null +++ b/content/primers/programming-and-tools.md @@ -0,0 +1,16 @@ +--- +title: Programming and tools primers +--- + +# Programming and tools primers + +Good tools remove friction and catch mistakes before anyone else sees them. These primers introduce the languages and tools that come up most often in this site's other sections. + +- [Git practices](git-best-practices.md): commits, branches and history that help rather than hinder. +- [Static code analysis](static-code-analysis.md): finding bugs without running the code. +- [PlantUML and Mermaid](plantuml-and-mermaid.md): diagrams as text. +- [Python for automation](python-for-automation.md): scripting the repetitive parts of engineering. +- [Rust for systems programming](rust-for-systems.md): memory safety without a garbage collector. +- [A modern terminal](terminal-modernisation.md): Oh My Zsh, Oh My Posh and friends. + +The [Tech Radar](../radar/index.md) shows which tools are recommended, and the [Guides](../guides/index.md) give step-by-step set-up instructions. diff --git a/content/primers/python-for-automation.md b/content/primers/python-for-automation.md new file mode 100644 index 0000000..1f27d61 --- /dev/null +++ b/content/primers/python-for-automation.md @@ -0,0 +1,82 @@ +--- +title: Python for automation +--- + +# Python for automation + +Python is the default language for automating engineering work: build and release scripts, test benches, data processing, talking to devices over serial ports and networks, and gluing tools together. It is readable, comes with a large standard library, and has a package for almost everything. For a script that saves someone ten minutes a day, it is hard to beat. + +## Typical jobs + +| Job | Useful libraries | +| --- | --- | +| Files, paths and processes | `pathlib`, `shutil`, `subprocess` (standard library) | +| Command-line tools | `argparse` (standard library), Typer, Click | +| HTTP and REST APIs | `httpx`, `requests` | +| Serial ports and devices | `pyserial`, PyVISA for test instruments | +| MQTT and messaging | `paho-mqtt`, `aiomqtt` | +| Data files | `json`, `csv`, `tomllib` (standard library), PyYAML, pandas, Polars | +| SSH and remote machines | Paramiko, Fabric | +| Testing and test benches | pytest | +| Cloud | boto3 (AWS), the Google Cloud and Azure SDKs | +| Scheduling and logging | `logging`, `sched` (standard library); cron or CI for running on a timetable | + +[[CAPTION:Table]] Common automation jobs and the libraries that help. + +## A small, solid script + +```python +"""Check that every device in a CSV file answered within the last hour.""" +import argparse +import csv +import logging +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +log = logging.getLogger("stale-devices") + +def stale(rows, now, limit): + for row in rows: + last_seen = datetime.fromisoformat(row["last_seen"]) + if now - last_seen > limit: + yield row["device_id"], last_seen + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("csv_file", type=Path) + parser.add_argument("--hours", type=float, default=1.0) + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + + with args.csv_file.open(newline="") as f: + found = list(stale(csv.DictReader(f), datetime.now(timezone.utc), timedelta(hours=args.hours))) + for device_id, last_seen in found: + log.warning("%s last seen %s", device_id, last_seen.isoformat()) + return 1 if found else 0 + +if __name__ == "__main__": + sys.exit(main()) +``` + +[[CAPTION:Figure]] A script with arguments, logging and an exit code that CI can use. + +The habits it shows: a docstring, arguments instead of hard-coded paths, logging instead of `print`, logic in a testable function, and a meaningful exit code. + +## Good habits + +- **Use a virtual environment per project**, and pin dependencies. `uv` makes this fast: `uv init`, `uv add httpx`, `uv run script.py`. `pip` with `venv` works everywhere. +- **Use a `pyproject.toml`** for anything beyond a single file, so the tool can be installed with its dependencies. +- **Add type hints** and check them with mypy or Pyright; they catch many mistakes in scripts that grow. +- **Format and lint with Ruff**. +- **Test with pytest**, even a few tests, once a script is used by other people. +- **Keep secrets out of the code**: read them from environment variables or a secrets manager. +- **Handle failure**: time-outs on every network call, retries with backoff, and clear error messages. +- **Know when to stop**: a script that has grown to thousands of lines and a dozen options has become a product; give it a proper structure, tests and an owner. + +Use a supported Python version (3.10 or later as of 2026) and check the end-of-life dates at [devguide.python.org/versions](https://devguide.python.org/versions/). + +## Where next + +- [Hardware-in-the-loop testing](hardware-in-the-loop-testing.md) is a common home for Python automation. +- The [Guides](../guides/index.md) section covers Python project set-up and coding standards. diff --git a/content/primers/rest-apis.md b/content/primers/rest-apis.md new file mode 100644 index 0000000..1a7b5a4 --- /dev/null +++ b/content/primers/rest-apis.md @@ -0,0 +1,64 @@ +--- +title: REST APIs +--- + +# REST APIs + +REST (Representational State Transfer) is the architectural style Roy Fielding described in his 2000 doctoral thesis, based on how the web itself works. In everyday use, a "REST API" means an HTTP API organised around resources, using the HTTP methods and status codes for what they mean. Few APIs follow every one of Fielding's constraints, and that is fine; the conventions are what make an API predictable. + +## Resources and URLs + +A resource is a thing the API exposes: a device, a reading, a user. Each has a URL. Use nouns and plural collections: + +```text +GET /devices list devices +POST /devices create a device +GET /devices/{id} read one device +PATCH /devices/{id} change some fields +DELETE /devices/{id} remove it +GET /devices/{id}/readings a sub-collection +``` + +[[CAPTION:Figure]] Resource-oriented URLs. + +Actions that do not fit create, read, update and delete can be modelled as a resource of their own (`POST /devices/{id}/reboots`) rather than a verb in the URL. + +## Methods + +| Method | Meaning | Safe | [Idempotent](idempotency.md) | +| --- | --- | --- | --- | +| GET | Read | Yes | Yes | +| POST | Create, or start a process | No | No | +| PUT | Replace entirely | No | Yes | +| PATCH | Change part of it | No | Not necessarily | +| DELETE | Remove | No | Yes | + +[[CAPTION:Table]] HTTP methods as REST uses them. + +## Status codes + +Use the right class so that clients and proxies behave correctly: + +- **2xx success**: 200 OK, 201 Created (with a `Location` header), 202 Accepted (work will happen later), 204 No Content. +- **4xx client error**: 400 Bad Request, 401 Unauthorized (not authenticated), 403 Forbidden (authenticated but not allowed), 404 Not Found, 409 Conflict, 422 Unprocessable Content, 429 Too Many Requests. +- **5xx server error**: 500 Internal Server Error, 502 Bad Gateway, 503 Service Unavailable, 504 Gateway Timeout. + +Return errors in one consistent body format. RFC 9457 (Problem Details for HTTP APIs) defines a standard one. + +## Design habits that pay off + +- **Statelessness**: each request carries everything needed to process it, including authentication. This makes servers easy to scale. +- **Pagination** on every collection that can grow, preferably with cursors rather than page numbers. +- **Filtering and sorting** through query parameters: `/devices?status=offline&sort=-lastSeen`. +- **Versioning** from day one, in the path (`/v1/`) or a header, and a policy for how long old versions live. +- **Caching** with `ETag` and `Cache-Control`; conditional requests (`If-Match`) also prevent lost updates. +- **Security**: HTTPS only, tokens such as [OAuth 2.0](oauth-and-oidc.md) access tokens, rate limits, and input validation. See [OWASP Top 10](owasp-top-10.md). + +## REST and the alternatives + +REST suits public and resource-shaped APIs. gRPC (Protocol Buffers over HTTP/2) suits fast internal service-to-service calls. GraphQL suits clients that need flexible queries across many resources. Event-driven interfaces suit notifications and streams. For constrained devices, CoAP brings the REST model to UDP; see [CoAP](coap.md). + +## Where next + +- [OpenAPI and AsyncAPI](openapi-and-asyncapi.md) shows how to describe an API as a contract. +- The [Guidelines](../guidelines/index.md) section has the API design rules for this site. diff --git a/content/primers/root-cause-analysis.md b/content/primers/root-cause-analysis.md new file mode 100644 index 0000000..313c3b4 --- /dev/null +++ b/content/primers/root-cause-analysis.md @@ -0,0 +1,60 @@ +--- +title: Root cause analysis +--- + +# Root cause analysis + +Root cause analysis (RCA) is a structured way of finding out why a problem happened, so that it can be prevented rather than just fixed. Restarting a crashed service fixes the symptom. Finding that a log file filled the disk because new servers never had log rotation, and fixing the provisioning so it cannot happen again, fixes the cause. + +## Principles + +- **Blameless**: look for weaknesses in systems and processes, not for someone to blame. People hide information when they fear blame, and the next person in the same position would probably have made the same mistake. +- **Evidence over opinion**: build a timeline from logs, metrics and records before drawing conclusions. +- **Usually more than one cause**: serious incidents typically need several things to go wrong together. Stopping at the first cause found misses the others. +- **Actions, not just findings**: an RCA is only finished when the preventive actions have owners and dates. + +## The process + +```mermaid +flowchart LR + A[Define the problem and its impact] --> B[Build a timeline from evidence] + B --> C[Identify causes and contributing factors] + C --> D[Agree corrective and preventive actions] + D --> E[Track actions to completion] + E --> F[Share what was learned] +``` + +[[CAPTION:Figure]] The steps of a root cause analysis. + +## Techniques + +- **Five whys**: ask "why?" repeatedly, each answer becoming the next question, until you reach something you can change in the system. Five is a guide, not a rule. It is quick but follows a single chain, so it can miss contributing causes; run it on more than one branch. +- **Fishbone (Ishikawa) diagram**: list possible causes under headings such as people, process, tools, environment, materials and measurement. Good for making sure the group considers every kind of cause. +- **Pareto analysis**: when there are many incidents, count them by cause and focus on the few causes behind most of them. +- **Fault tree analysis**: start from the failure and work down through AND and OR gates to the basic events that can produce it. Common in safety engineering, where it can be quantified with failure rates. +- **Change analysis**: ask what changed just before the problem appeared: a deployment, a configuration, a supplier, a firmware version. + +## An example + +A fleet of devices stops reporting for six hours. + +1. Why did they stop? Their certificates expired. +2. Why did they expire? Automatic renewal failed on 30 % of devices. +3. Why did renewal fail? Those devices ran an older firmware version with a renewal bug. +4. Why was that version still in the field? A [firmware update](firmware-over-the-air.md) campaign had stalled and nobody noticed. +5. Why did nobody notice? Campaign progress was not monitored and there was no alert for certificates close to expiry. + +Actions: fix the renewal bug (done), finish the campaign, add certificate-expiry monitoring, add campaign progress alerts, and review other time-limited credentials. Note that the answer to the first why was technical, but the most useful actions were about monitoring and process. + +## In practice + +- Run an RCA for every significant incident, and for near misses that could have been significant. +- Write it up as a postmortem: summary, impact, timeline, causes, what went well, actions. The [Templates](../templates/index.md) section has a postmortem template and a debugging checklist. +- Review open actions regularly. A backlog of unfinished preventive actions is a leading indicator of the next incident. +- Look across RCAs for repeated themes. + +## Where next + +- [Testing strategies](testing-strategies.md) and [code review](code-review.md) prevent many problems from reaching production. +- The [Guides](../guides/index.md) section covers incident handling and defect management. +- Google's *Site Reliability Engineering* book has a good chapter on [postmortem culture](https://sre.google/sre-book/postmortem-culture/). diff --git a/content/primers/rtos-and-freertos.md b/content/primers/rtos-and-freertos.md new file mode 100644 index 0000000..b7ea54e --- /dev/null +++ b/content/primers/rtos-and-freertos.md @@ -0,0 +1,75 @@ +--- +title: Real-time operating systems and FreeRTOS +--- + +# Real-time operating systems and FreeRTOS + +A real-time operating system (RTOS) lets firmware be written as several independent tasks, each with a priority, and guarantees that the highest-priority task that is ready always runs. "Real time" means predictable, not fast: the point is a known worst-case response time. + +FreeRTOS is the most widely used RTOS on microcontrollers. It is small (a few kilobytes of flash), MIT-licensed, ported to dozens of architectures, and maintained by Amazon Web Services since 2017. Zephyr, ThreadX (now Eclipse ThreadX) and the RTOS in Mbed and NuttX are common alternatives. + +## Bare metal or RTOS + +| Approach | Good for | Drawbacks | +| --- | --- | --- | +| Superloop with interrupts | Simple devices, one main activity, tight resources | Timing of each job depends on all the others; long jobs block everything | +| Cooperative scheduler | A few jobs that yield willingly | One misbehaving job still blocks the rest | +| Pre-emptive RTOS | Several activities with different deadlines; networking stacks | More RAM (a stack per task), more ways to get concurrency wrong | + +[[CAPTION:Table]] Ways to structure firmware. + +## Core concepts + +- **Task**: a function with its own stack and priority that normally loops forever. A task is running, ready, blocked (waiting for something) or suspended. +- **Scheduler**: picks the highest-priority ready task. Equal-priority tasks can share time in turns. +- **Queue**: the main way tasks pass data. A task blocks on an empty queue without using CPU time. +- **Semaphore**: a signal or a count, often given from an interrupt to wake a task. +- **Mutex**: protects a shared resource. FreeRTOS mutexes use priority inheritance to limit priority inversion. +- **Event groups and task notifications**: lighter ways to signal one or more tasks. +- **Software timers**: run a callback after a delay or periodically. + +```c +static QueueHandle_t readings; + +static void sensor_task(void *arg) +{ + for (;;) { + int16_t value = sensor_read(); + xQueueSend(readings, &value, 0); + vTaskDelay(pdMS_TO_TICKS(100)); + } +} + +static void report_task(void *arg) +{ + int16_t value; + for (;;) { + if (xQueueReceive(readings, &value, portMAX_DELAY) == pdPASS) { + report(value); + } + } +} + +void app_main(void) +{ + readings = xQueueCreate(16, sizeof(int16_t)); + xTaskCreate(sensor_task, "sensor", 256, NULL, 2, NULL); + xTaskCreate(report_task, "report", 512, NULL, 1, NULL); + vTaskStartScheduler(); +} +``` + +[[CAPTION:Figure]] Two FreeRTOS tasks connected by a queue. + +## Common mistakes + +- **Priority inversion**: a low-priority task holds a mutex that a high-priority task needs, while a medium-priority task runs. Use mutexes (not binary semaphores) for shared resources and keep critical sections short. +- **Deadlock**: two tasks each hold a lock the other needs. Always take locks in the same order, or avoid sharing. +- **Stack overflow**: enable stack checking (`configCHECK_FOR_STACK_OVERFLOW`) during development and measure the high-water mark. +- **Doing work in interrupts**: keep interrupt handlers short and defer the work to a task with a semaphore or notification. Only call the `FromISR` versions of API functions inside interrupts. +- **Too many tasks**: each one costs a stack and adds interactions. Group related work; use [state machines](finite-state-machines.md) within a task. + +## Where next + +- [Interrupts and DMA](interrupts-and-dma.md) and [embedded systems design](embedded-systems-design.md) give the wider picture. +- The [FreeRTOS documentation](https://www.freertos.org/Documentation/00-Overview) is the reference, and the free book *Mastering the FreeRTOS Real Time Kernel* is a good tutorial. diff --git a/content/primers/rust-for-systems.md b/content/primers/rust-for-systems.md new file mode 100644 index 0000000..b13de0b --- /dev/null +++ b/content/primers/rust-for-systems.md @@ -0,0 +1,56 @@ +--- +title: Rust for systems programming +--- + +# Rust for systems programming + +Rust is a systems programming language that gives the performance and low-level control of C and C++ while preventing most memory safety bugs at compile time. It has no garbage collector, so it suits operating systems, embedded firmware, network services and anything where predictable performance matters. Microsoft and Google have both reported that around 70 % of their serious security bugs are memory safety issues. That is why governments and large companies now encourage memory-safe languages for new code. + +## How it achieves safety + +- **Ownership**: every value has exactly one owner, and the value is freed when its owner goes out of scope. No manual `free`, no double free, no leaks from forgetting. +- **Borrowing**: code can borrow a value either as any number of shared, read-only references or as one mutable reference, never both at once. The compiler's borrow checker enforces this, which rules out use-after-free, dangling pointers and data races. +- **No null**: a value that might be missing is an `Option`, which the compiler forces you to handle. +- **Errors as values**: functions that can fail return `Result`, and the `?` operator passes errors up neatly. Errors cannot be ignored by accident. +- **`unsafe`**: low-level operations the compiler cannot check (raw pointers, hardware registers, calling C) are allowed inside `unsafe` blocks. This keeps the risky code small, visible and reviewable. + +```rust +fn parse_reading(frame: &[u8]) -> Result { + let bytes = frame.get(2..4).ok_or("frame too short")?; + Ok(u16::from_be_bytes([bytes[0], bytes[1]])) +} +``` + +[[CAPTION:Figure]] Bounds-checked parsing: a short frame is an error, not a buffer overrun. + +## The ecosystem + +- **Cargo** builds, tests, documents and manages dependencies (crates) out of the box. +- **Clippy** (linter) and **rustfmt** (formatter) are standard. +- **Async** networking with Tokio; web services with Axum. +- **Interoperability**: Rust can call C and be called from C, so it can be introduced one module at a time into an existing codebase. + +## On microcontrollers + +Embedded Rust is mature on Arm Cortex-M, RISC-V and the ESP32 family: + +- `no_std` code runs without an operating system or heap. +- **Peripheral access crates** are generated from vendor register descriptions, and **HAL crates** implement the shared `embedded-hal` traits, so drivers work across chips. +- **Embassy** provides async tasks on bare metal; **RTIC** provides interrupt-driven concurrency checked at compile time. +- **probe-rs** handles flashing and debugging. +- **Ferrocene** is a Rust toolchain qualified for safety standards such as ISO 26262 and IEC 61508. + +## Trade-offs + +- The learning curve is real, especially the borrow checker; expect weeks before a C programmer feels productive. +- Compile times are longer than C. +- Vendor support (SDKs, examples, certified stacks) is still mostly C. Check that the chips and protocol stacks you need are covered. +- Hiring and team experience may be limited. + +A sensible way to start is a new, self-contained component (a parser, a protocol handler, a command-line tool) rather than rewriting working C. + +## Where next + +- [Secure coding practices](secure-coding-practices.md) explains the bugs Rust prevents. +- The [Tech Radar](../radar/index.md) records where Rust is recommended. +- *The Rust Programming Language* ([doc.rust-lang.org/book](https://doc.rust-lang.org/book/)) and *The Embedded Rust Book* ([docs.rust-embedded.org/book](https://docs.rust-embedded.org/book/)) are free. diff --git a/content/primers/saga-pattern.md b/content/primers/saga-pattern.md new file mode 100644 index 0000000..d715095 --- /dev/null +++ b/content/primers/saga-pattern.md @@ -0,0 +1,59 @@ +--- +title: The saga pattern +--- + +# The saga pattern + +When one business operation changes data in several services, a single database transaction is no longer available. Distributed transactions (two-phase commit) exist but are slow, fragile and poorly supported by message brokers and cloud services. The saga pattern is the common alternative: a sequence of local transactions, each with a compensating action that undoes it if a later step fails. + +The idea comes from a 1987 paper by Garcia-Molina and Salem on long-lived database transactions. + +## An example + +Provisioning a new device might involve three services: + +1. Inventory reserves a SIM card. +2. Billing creates a subscription. +3. Device management registers the device. + +If step 3 fails, the saga runs the compensations in reverse: cancel the subscription, then release the SIM. + +```mermaid +sequenceDiagram + participant O as Orchestrator + participant I as Inventory + participant B as Billing + participant D as Device management + O->>I: reserve SIM + I-->>O: reserved + O->>B: create subscription + B-->>O: created + O->>D: register device + D-->>O: failed + O->>B: cancel subscription + O->>I: release SIM +``` + +[[CAPTION:Figure]] An orchestrated saga that fails at the last step and compensates. + +## Orchestration or choreography + +- **Orchestration**: one component (the orchestrator) tells each service what to do and tracks progress. The flow is in one place, easy to read and monitor. The orchestrator must be reliable and keeps its state durably. Workflow engines such as Temporal, Camunda or AWS Step Functions provide this. +- **Choreography**: there is no coordinator. Each service listens for events and publishes its own: `SimReserved` triggers billing, `SubscriptionCreated` triggers registration. Fewer moving parts, but the flow is spread across services and becomes hard to follow beyond three or four steps. + +## Designing compensations + +- A compensation is a new business action, not a rollback. A cancelled subscription may still leave an invoice line that is credited. Some actions (an email already sent) cannot be undone at all; put them last or accept them. +- Every step and every compensation must be [idempotent](idempotency.md), because they will be retried. +- Other operations can see the intermediate state. Use explicit states ("pending activation") so that half-finished work is visible and not mistaken for done. +- Decide what happens when a compensation fails. Usually: retry, then raise an alert for a person. + +## The outbox + +A service that updates its database and publishes an event can crash between the two. The transactional outbox pattern writes the event to an outbox table in the same transaction as the data change; a separate process then publishes from the table. This makes each saga step reliable. + +## Where next + +- [Event-driven architecture](event-driven-architecture.md) and [consistency models](consistency-models.md) give the background. +- The [Handbook](../handbook/index.md) covers workflow and integration patterns in more depth. +- Chris Richardson's [microservices.io](https://microservices.io/patterns/data/saga.html) describes the saga and outbox patterns with examples. diff --git a/content/primers/scrum-vs-kanban.md b/content/primers/scrum-vs-kanban.md new file mode 100644 index 0000000..9853372 --- /dev/null +++ b/content/primers/scrum-vs-kanban.md @@ -0,0 +1,61 @@ +--- +title: Scrum and Kanban +--- + +# Scrum and Kanban + +Scrum and Kanban are the two most widely used ways to organise an [agile](agile-development.md) team. Scrum works in fixed-length iterations with defined roles and events. Kanban manages a continuous flow of work by making it visible and limiting how much is in progress. Both aim at the same thing: delivering value steadily and improving how the team works. + +## Scrum + +Scrum is defined in the *Scrum Guide* by Ken Schwaber and Jeff Sutherland, last revised in 2020. + +- **Sprint**: a fixed period of one month or less, often two weeks, producing a usable increment. +- **Roles (accountabilities)**: the product owner orders the backlog and maximises value; the developers build the increment; the Scrum Master helps the team use Scrum well and removes impediments. +- **Events**: sprint planning (choose a sprint goal and the work), the daily scrum (15 minutes on progress towards the goal), the sprint review (show the increment to stakeholders) and the sprint retrospective (improve the way of working). +- **Artefacts**: the product backlog, the sprint backlog and the increment, each with a commitment: the product goal, the sprint goal and the definition of done. + +## Kanban + +Kanban for knowledge work was developed by David J. Anderson in the 2000s, drawing on lean manufacturing. + +- **Visualise the work** on a board with a column for each stage: for example, ready, in progress, review, test, done. +- **Limit work in progress (WIP)**: each column has a maximum number of items. When a column is full, people help finish existing work rather than start new work. +- **Manage flow**: measure how long items take (cycle time) and how many finish each week (throughput), and remove the bottlenecks. +- **Make policies explicit**: what "done" means for each column, how urgent work is handled. +- **Improve continuously** using the data. + +```mermaid +flowchart LR + B[Backlog] --> R[Ready, limit 5] + R --> D[In progress, limit 3] + D --> V[Review, limit 2] + V --> T[Test, limit 2] + T --> X[Done] +``` + +[[CAPTION:Figure]] A Kanban board with work-in-progress limits. + +## Comparing them + +| | Scrum | Kanban | +| --- | --- | --- | +| Cadence | Fixed sprints | Continuous flow; releases whenever ready | +| Roles | Product owner, Scrum Master, developers | None required | +| Planning | At the start of each sprint | Whenever capacity frees up | +| Change during a cycle | Avoided within a sprint | Allowed at any time | +| Main limit | Sprint capacity | WIP limits per stage | +| Main metrics | Sprint goal achieved; velocity (for the team's own planning) | Cycle time, throughput, flow | +| Best for | Product development with a stable team and a roadmap | Support, operations, maintenance, or work that arrives unpredictably | + +[[CAPTION:Table]] Scrum and Kanban side by side. + +## Mixing them + +Many teams use Scrum's events with a Kanban board and WIP limits, sometimes called Scrumban. That is fine: the frameworks are tools. What matters is short feedback loops, finishing work before starting more, and a regular, honest look at how things are going. + +## Where next + +- [Agile development](agile-development.md) covers the values both frameworks share. +- The [Templates](../templates/index.md) section has definition of ready and definition of done checklists. +- [scrumguides.org](https://scrumguides.org/) has the official Scrum Guide, and the [Kanban Guide](https://kanban.university/kanban-guide/) describes the Kanban method. diff --git a/content/primers/sdlc-models.md b/content/primers/sdlc-models.md new file mode 100644 index 0000000..b250840 --- /dev/null +++ b/content/primers/sdlc-models.md @@ -0,0 +1,63 @@ +--- +title: Software development life cycles +--- + +# Software development life cycles + +A software development life cycle (SDLC) model describes the order in which a project does its work: understanding requirements, designing, building, testing, releasing and maintaining. Every project goes through these activities; the models differ in how they sequence them and how often they repeat them. + +## Waterfall + +Each phase is finished and signed off before the next begins: requirements, design, implementation, verification, maintenance. Winston Royce described it in 1970, mostly as a warning: his paper argued that the pure sequence was risky and that feedback loops were needed. + +- **Strengths**: clear milestones and documents; suits fixed-price contracts and work where requirements really are stable. +- **Weaknesses**: problems surface late, in testing, when they are most expensive to fix; working software appears only at the end; change is hard. + +## The V-model + +The V-model is a waterfall with the testing made explicit. Each phase on the way down has a matching test phase on the way up, planned at the same time. + +```mermaid +flowchart LR + R[Requirements] --> SD[System design] --> AD[Architecture design] --> MD[Module design] --> C[Code] + C --> UT[Unit tests] --> IT[Integration tests] --> ST[System tests] --> AT[Acceptance tests] + R -. verified by .-> AT + SD -. verified by .-> ST + AD -. verified by .-> IT + MD -. verified by .-> UT +``` + +[[CAPTION:Figure]] Each design level is verified by its own test level. + +It is common in safety-related and regulated work (automotive ISO 26262, medical IEC 62304, aerospace), where traceability from requirement to test is mandatory. In systems engineering, the V describes the whole product, hardware included. + +## Spiral + +Barry Boehm's spiral model (1986) repeats a cycle of planning, risk analysis, building and evaluating, with each loop producing a more complete prototype or product. Its key idea is that the biggest risks are tackled first. It suits large, novel or high-risk projects, and needs real skill in risk assessment. + +## Iterative and incremental + +The product is built in small slices, each one designed, built, tested and ideally delivered. Feedback from each slice shapes the next. The Rational Unified Process (RUP) formalised this, and [agile methods](agile-development.md) such as Scrum are its most popular modern form. + +- **Strengths**: early working software, early feedback, room to change direction, risk reduced in small steps. +- **Weaknesses**: needs discipline to keep the architecture coherent across increments; harder to fix a price and scope up front. + +## Choosing + +| Situation | Good fit | +| --- | --- | +| Requirements uncertain or likely to change | Iterative or agile | +| Safety-critical, regulated, heavy verification | V-model, often with iterative development inside each phase | +| Large, novel, high technical risk | Spiral, or iterative with early risk spikes | +| Small, well-understood, fixed scope | Waterfall can work | +| Hardware and software developed together | A V-model for the product, agile for the firmware and software | + +[[CAPTION:Table]] Matching the life cycle to the situation. + +Real projects mix models. A medical device team might run two-week sprints for firmware inside a V-model that governs requirements, verification and release. + +## Where next + +- [Agile development](agile-development.md) and [Scrum and Kanban](scrum-vs-kanban.md) cover the iterative approaches used most today. +- [Testing strategies](testing-strategies.md) covers the right-hand side of the V. +- The [Guides](../guides/index.md) section describes the development process used here. diff --git a/content/primers/secure-coding-practices.md b/content/primers/secure-coding-practices.md new file mode 100644 index 0000000..c48585e --- /dev/null +++ b/content/primers/secure-coding-practices.md @@ -0,0 +1,55 @@ +--- +title: Secure coding practices +--- + +# Secure coding practices + +Most security vulnerabilities are ordinary bugs in places where an attacker can reach them. Secure coding is a set of habits that prevent whole classes of those bugs, plus the discipline to treat every input from outside as potentially hostile. It applies equally to a web service and to device firmware. + +## Principles + +- **Validate all input** at the boundary: check type, length, range and format against what is allowed (an allow-list), and reject everything else. This includes data from devices, files, environment variables and other internal services, not just users. +- **Encode output for its context**: HTML-escape for web pages, use parameterised queries for SQL, pass argument lists rather than shell strings to processes. Most injection bugs come from mixing data and code in one string. +- **Least privilege**: every process, service account, token and device credential gets only the access it needs. +- **Defence in depth**: assume any one control can fail, and layer them. +- **Fail securely**: when something goes wrong, deny access and keep the system in a safe state. Error messages to users should not reveal internals; logs should have the detail. +- **Secure defaults**: the product must be secure as shipped, without relying on the user to change settings. +- **Keep it simple**: complex code hides bugs. The less code that handles untrusted data, the better. + +## Common weaknesses and their fixes + +| Weakness | Prevention | +| --- | --- | +| Injection (SQL, command, LDAP) | Parameterised queries, safe APIs, no string concatenation of inputs into commands | +| Cross-site scripting | Templating that escapes by default, Content Security Policy | +| Buffer overflow (C and C++) | Bounds-checked functions, length checks before every copy, compiler hardening, or a memory-safe language such as [Rust](rust-for-systems.md) | +| Integer overflow | Check arithmetic on sizes and lengths before using the result | +| Hard-coded secrets | A secrets manager or secure element; secret scanning in CI | +| Insecure deserialisation | Never deserialise untrusted data into arbitrary objects; use simple formats and validate them | +| Path traversal | Normalise paths and check they stay inside the allowed directory | +| Race conditions | Atomic operations, proper locking, check-and-use in one step | +| Weak cryptography | Standard libraries and algorithms only; see [cryptography basics](cryptography-basics.md) | + +[[CAPTION:Table]] Common weaknesses, as catalogued in the CWE list, and how to prevent them. + +## For embedded C + +- Follow a coding standard: MISRA C or the SEI CERT C standard. +- Enable compiler warnings and treat them as errors; enable stack protection and, where the hardware supports it, the memory protection unit. +- Never trust lengths in received packets. Check them against both the buffer size and the bytes actually received. +- Clear keys and secrets from memory after use, in a way the compiler cannot optimise away. +- Lock debug ports (JTAG, SWD) on production devices. + +## In the development process + +- **Threat model** new features: what are the assets, who might attack them, and how? STRIDE is a simple checklist. +- **Review** code with security in mind; see [code review](code-review.md). +- **Automate**: [static analysis](static-code-analysis.md), dependency scanning, secret scanning and fuzzing in the [CI/CD pipeline](cicd-pipelines.md). +- **Manage dependencies**: keep a software bill of materials, update regularly, and remove what you do not use. +- **Learn from incidents**: every vulnerability found is a chance to remove a whole class, not just one instance. + +## Where next + +- [OWASP Top 10](owasp-top-10.md) lists the most common web risks. +- The [Guidelines](../guidelines/index.md) section sets the secure coding rules used here. +- The [OWASP Secure Coding Practices quick reference](https://owasp.org/www-project-secure-coding-practices-quick-reference-guide/) and the [CWE Top 25](https://cwe.mitre.org/top25/) are good checklists. diff --git a/content/primers/security.md b/content/primers/security.md new file mode 100644 index 0000000..535a389 --- /dev/null +++ b/content/primers/security.md @@ -0,0 +1,28 @@ +--- +title: Security primers +--- + +# Security primers + +Security is a property of the whole system, from the silicon in a device to the people who run the cloud platform. These primers introduce the cryptography, protocols, practices and standards that come up most often. + +## Foundations + +- [Cryptography basics](cryptography-basics.md): hashes, symmetric and public-key cryptography, MACs, key derivation and random numbers. +- [PKI and certificates](pki-and-certificates.md): how identities are issued, checked and revoked. +- [TLS and DTLS](tls-and-dtls.md): secure channels for streams and datagrams. +- [Post-quantum cryptography](post-quantum-cryptography.md): preparing for quantum computers. + +## Building secure software + +- [Secure coding practices](secure-coding-practices.md): habits that prevent whole classes of bug. +- [OWASP Top 10](owasp-top-10.md): the most common web application risks. +- [OAuth 2.0 and OpenID Connect](oauth-and-oidc.md): delegated authorisation and sign-in. +- [Zero trust architecture](zero-trust-architecture.md): no trust based on network location. + +## Standards for connected products + +- [ETSI EN 303 645](etsi-en-303-645.md): the baseline for consumer IoT security. +- [ISO/IEC 30141](iso-30141.md): a reference architecture for IoT systems. + +The [Standards](../standards/index.md) and [Guidelines](../guidelines/index.md) sections set the security rules used here. The [Glossary](../glossary/index.md) defines the terms. diff --git a/content/primers/serverless-computing.md b/content/primers/serverless-computing.md new file mode 100644 index 0000000..46c8703 --- /dev/null +++ b/content/primers/serverless-computing.md @@ -0,0 +1,68 @@ +--- +title: Serverless computing +--- + +# Serverless computing + +Serverless computing means running code or using services without managing the servers underneath. There are still servers; the provider runs them, scales them and charges only for actual use. A serverless function that receives no requests costs nothing. + +The term covers two things: + +- **Functions as a Service (FaaS)**: you upload a function, and the platform runs it in response to events. AWS Lambda, Google Cloud Run functions and Azure Functions are the main examples. +- **Serverless back-end services**: managed databases, queues, storage and API gateways that scale automatically and bill per request (DynamoDB, Firestore, S3, Pub/Sub). + +## How a function runs + +```mermaid +sequenceDiagram + participant E as Event source + participant P as Platform + participant F as Function instance + E->>P: event (HTTP request, queue message, file upload) + alt no warm instance + P->>F: start instance (cold start) + end + P->>F: invoke handler(event) + F-->>P: result + Note over F: kept warm for a while, then stopped +``` + +[[CAPTION:Figure]] A function invoked by an event. + +```python +def handler(event, context): + device_id = event["deviceId"] + reading = float(event["temperature"]) + if reading > 70.0: + send_alert(device_id, reading) + return {"status": "ok"} +``` + +[[CAPTION:Figure]] A minimal function handler. + +## Strengths + +- No servers to patch or scale; the platform handles bursts automatically. +- Cost follows use, which suits spiky or low traffic. +- Natural fit for glue code: reacting to uploads, queue messages, schedules and webhooks. + +## Limits + +- **Cold starts**: the first request after a quiet period waits while an instance starts, from tens of milliseconds to a few seconds depending on the language and package size. Provisioned or minimum instances reduce this, at a cost. +- **Time and resource limits**: functions have maximum run times (15 minutes on Lambda) and memory limits. Long jobs need a different home. +- **Statelessness**: anything that must survive between calls goes in an external store. +- **Cost at steady high load**: per-request billing can cost more than a container that is busy all the time. +- **Lock-in and testing**: triggers and permissions are provider-specific, and a system made of many small functions can be hard to test and trace end to end. + +## Good habits + +- Keep functions small and focused, and keep start-up work light. +- Make handlers [idempotent](idempotency.md): event sources retry. +- Grant each function only the permissions it needs. +- Use [infrastructure as code](infrastructure-as-code.md) (AWS SAM, Serverless Framework, Terraform) rather than deploying from a console. +- Consider serverless containers (Cloud Run, AWS Fargate, Azure Container Apps) when you need longer runs or more control but still want scaling to zero. + +## Where next + +- [Event-driven architecture](event-driven-architecture.md) is the usual design around functions. +- The [Handbook](../handbook/index.md) covers choosing between functions, containers and virtual machines. diff --git a/content/primers/static-code-analysis.md b/content/primers/static-code-analysis.md new file mode 100644 index 0000000..b5958ca --- /dev/null +++ b/content/primers/static-code-analysis.md @@ -0,0 +1,48 @@ +--- +title: Static code analysis +--- + +# Static code analysis + +Static code analysis examines source code without running it, looking for bugs, security weaknesses, style problems and risky constructs. Because it needs no tests or test environment, it can run on every change in seconds to minutes. It also finds problems that tests rarely cover, such as an error path that is never exercised. + +## What it finds + +- **Defects**: null pointer dereferences, buffer overruns, use after free, resource leaks, uninitialised variables, unreachable code, integer overflow. +- **Security weaknesses**: injection, hard-coded secrets, unsafe functions, tainted data reaching a dangerous sink. +- **Standards violations**: MISRA C, SEI CERT, or your own coding rules. +- **Maintainability problems**: high complexity, duplication, overly long functions, dependency cycles. +- **Style and formatting**, usually better handled by a formatter that fixes it automatically. + +## Kinds of tool + +| Kind | What it does | Examples | +| --- | --- | --- | +| Formatter | Rewrites code into one consistent style | clang-format, Black, Ruff format, rustfmt, Prettier | +| Linter | Fast checks for common mistakes and style | Ruff, ESLint, clang-tidy, Clippy, ShellCheck | +| Type checker | Checks types in dynamically typed languages | mypy, Pyright, TypeScript | +| Deep analyser | Data-flow and path analysis across functions and files | Cppcheck, the Clang Static Analyzer, Coverity, Polyspace, PVS-Studio | +| Security scanner (SAST) | Looks for vulnerability patterns and tainted data flows | Semgrep, CodeQL, Bandit, SonarQube | +| Quality platform | Tracks findings and metrics over time | SonarQube, SonarCloud | + +[[CAPTION:Table]] Families of static analysis tools. + +Related checks often run alongside: dependency scanning (known vulnerabilities in libraries), secret scanning, and infrastructure-as-code scanning. + +## Making it useful + +- **Start in the editor**: the cheapest place to fix a finding is before it is committed. Most tools have editor plug-ins and pre-commit hooks. +- **Run it in the [CI pipeline](cicd-pipelines.md)** and fail the build on new findings of agreed severity. +- **Deal with legacy findings gradually**: a baseline that hides existing findings, with a rule that new code must be clean, works better than a thousand warnings on day one. +- **Tune the rules**: switch off checks that do not fit the codebase, and document why. Too many false positives and people stop reading the results. +- **Suppress narrowly**: when a finding is wrong, suppress that one instance with a comment explaining why, not the whole rule. +- **Combine it with other checks**: static analysis complements [testing](testing-strategies.md), [code review](code-review.md) and run-time tools such as sanitisers (AddressSanitizer, UndefinedBehaviorSanitizer) and fuzzing; it does not replace them. + +## For embedded C and C++ + +Compiler warnings are the first and cheapest analyser: build with `-Wall -Wextra` (and more) and treat warnings as errors. Add clang-tidy or Cppcheck for every change, and a deeper commercial tool or MISRA checker where the product's safety or certification needs it. + +## Where next + +- [Secure coding practices](secure-coding-practices.md) describes the weaknesses these tools look for. +- The [Tech Radar](../radar/index.md) lists the recommended tools, and the [Guidelines](../guidelines/index.md) set the rules they enforce. diff --git a/content/primers/technical-debt.md b/content/primers/technical-debt.md new file mode 100644 index 0000000..45e8a38 --- /dev/null +++ b/content/primers/technical-debt.md @@ -0,0 +1,48 @@ +--- +title: Technical debt +--- + +# Technical debt + +Technical debt is the extra cost of future change caused by shortcuts or outdated decisions in the code and design today. Ward Cunningham coined the metaphor in 1992: like financial debt, a little taken on deliberately can help you move faster, but it accrues interest. Every change to indebted code takes longer and is riskier, until the interest consumes most of the team's time. + +## Kinds of debt + +Martin Fowler's technical debt quadrant separates debt by whether it was deliberate and whether it was prudent: + +| | Reckless | Prudent | +| --- | --- | --- | +| **Deliberate** | "We don't have time for design." | "We must ship now and will deal with the consequences." | +| **Inadvertent** | "What's layering?" | "Now we know how we should have done it." | + +[[CAPTION:Table]] The technical debt quadrant. + +Prudent, deliberate debt is a legitimate business decision. Inadvertent, prudent debt is unavoidable: teams learn as they build. Reckless debt is the kind to prevent. + +Debt also takes many forms beyond messy code: missing tests, outdated dependencies, manual deployment steps, missing documentation, an architecture that no longer fits the product, and slow builds. + +## Signs of debt + +- Simple changes take surprisingly long, or keep breaking unrelated things. +- People avoid certain areas of the code. +- Estimates for work in some areas are consistently wrong. +- Bugs cluster in the same modules. +- New team members take a long time to become productive. + +## Managing it + +- **Make it visible**: record significant debt in the backlog with what it costs ("every change to billing needs two days of manual testing"), not just what is wrong. A debt register or labelled tickets work. +- **Prioritise by interest, not size**: fix the debt in code that changes often. Ugly code that nobody touches costs little. +- **Pay down continuously**: leave code better than you found it (the "boy scout rule"), and reserve a steady share of capacity, such as 15 to 20 %, rather than waiting for a "refactoring sprint" that never comes. +- **Prevent the reckless kind**: [code review](code-review.md), [static analysis](static-code-analysis.md), automated [tests](testing-strategies.md) and a clear definition of done. +- **Record deliberate debt** when you take it on, ideally in an [ADR](architecture-decision-records.md) with the condition for repaying it. +- **Measure what you can**: complexity, duplication, dependency age and change failure rate show trends, though no single number captures debt. + +## Talking about it + +Explain debt to non-technical stakeholders in terms of speed and risk. For example: "feature X will take three weeks instead of one because of Y. A week spent on Y first makes this and the next four features faster." That makes it a business decision rather than an engineering preference. + +## Where next + +- [Anti-patterns](anti-patterns.md) describes common forms of design debt. +- The [Guides](../guides/index.md) section covers refactoring and maintenance in more detail. diff --git a/content/primers/terminal-modernisation.md b/content/primers/terminal-modernisation.md new file mode 100644 index 0000000..7d64887 --- /dev/null +++ b/content/primers/terminal-modernisation.md @@ -0,0 +1,67 @@ +--- +title: A modern terminal +--- + +# A modern terminal + +Engineers spend hours a day at a command line. A few tools make that time faster and less error-prone. A good prompt shows the Git branch and whether the last command failed; suggestions come from history as you type; syntax highlighting shows a mistyped command before you run it. Oh My Zsh and Oh My Posh are the two most popular ways to get there. + +## Oh My Zsh + +Oh My Zsh is a community framework for configuring Zsh, the default shell on macOS and widely used on Linux. It bundles hundreds of plugins and themes behind a single `~/.zshrc`. + +```bash +sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" +``` + +Read any script before piping it into a shell; the installer is short and worth a look. + +Useful plugins, enabled in `~/.zshrc` with `plugins=(git z docker zsh-autosuggestions zsh-syntax-highlighting)`: + +- **git**: dozens of short aliases (`gst` for `git status`, `gco` for `git checkout`). +- **z** or **zoxide**: jump to frequently used directories by part of their name. +- **zsh-autosuggestions** and **zsh-syntax-highlighting**: separate projects, installed into Oh My Zsh's custom plug-ins folder, that suggest commands from history and colour commands as you type. +- **history-substring-search**: search history by any part of a command. + +## Oh My Posh + +Oh My Posh is a prompt engine, not a shell framework. It works with PowerShell, Bash, Zsh, Fish, Nushell and cmd (through Clink), on Windows, macOS and Linux, so one prompt configuration can follow you everywhere. Themes are JSON, YAML or TOML files. + +```powershell +winget install JanDeDobbeleer.OhMyPosh +oh-my-posh init pwsh --config "$env:POSH_THEMES_PATH/paradox.omp.json" | Invoke-Expression +``` + +Put the second line in your PowerShell profile to load it every time. On macOS and Linux, install it with Homebrew or the install script and add the matching `init` line for your shell. + +## Prompt themes + +A good prompt shows the current directory, Git branch and status, the exit status of the last command and the duration of slow commands. It also shows anything that changes how commands behave, such as a Python virtual environment, a Kubernetes context or an SSH session. + +- **Powerlevel10k** is a fast, very configurable Zsh theme with a setup wizard (`p10k configure`). Its author has said it will receive only limited maintenance from now on, so consider an alternative for new set-ups. +- **Starship** is a fast, cross-shell prompt written in Rust and configured in TOML, similar in scope to Oh My Posh. +- Oh My Posh ships with many themes, including a Powerlevel10k look-alike. + +Most themes need a **Nerd Font** (a programming font patched with icons, such as MesloLGS NF or FiraCode Nerd Font) set in the terminal. + +## Which to use + +| | Oh My Zsh | Oh My Posh | +| --- | --- | --- | +| What it is | Zsh configuration framework: plugins, aliases, themes | Prompt engine only | +| Shells | Zsh | PowerShell, Bash, Zsh, Fish, Nushell, cmd | +| Platforms | macOS, Linux, WSL | Windows, macOS, Linux | +| Best for | Zsh users who want plugins as well as a prompt | Windows users, or one prompt across several shells | + +[[CAPTION:Table]] Oh My Zsh and Oh My Posh compared. + +They work together: Oh My Zsh for plugins, with Oh My Posh or Starship drawing the prompt. + +## Keep it fast + +Each plugin adds start-up time. If a new terminal takes more than half a second to appear, remove plugins you do not use. On Windows, a modern terminal application (Windows Terminal, WezTerm or Alacritty) makes a bigger difference than any prompt. + +## Where next + +- [Git practices](git-best-practices.md) pairs well with a Git-aware prompt. +- The project sites are [ohmyz.sh](https://ohmyz.sh/), [ohmyposh.dev](https://ohmyposh.dev/) and [starship.rs](https://starship.rs/). diff --git a/content/primers/testing-strategies.md b/content/primers/testing-strategies.md new file mode 100644 index 0000000..902f511 --- /dev/null +++ b/content/primers/testing-strategies.md @@ -0,0 +1,60 @@ +--- +title: Testing strategies +--- + +# Testing strategies + +A testing strategy decides what to test, at which level, and how much, so that the team has confidence to release without spending all its time writing and maintaining tests. No strategy finds every bug. A good one finds the important ones early, when they are cheap to fix, and runs fast enough to use on every change. + +## The test pyramid + +```mermaid +flowchart TB + E2E[End-to-end and system tests: few, slow, realistic] + I[Integration and contract tests: some] + U[Unit tests: many, fast, precise] + E2E --- I --- U +``` + +[[CAPTION:Figure]] The test pyramid: most tests at the bottom. + +- **Unit tests** check one function or class in isolation, in milliseconds. When one fails, it points straight at the problem. +- **Integration tests** check that components work together: code with a real database, a service with its message broker, a driver with real hardware. +- **Contract tests** check that a service and its consumers agree on an interface, without running both together. Pact is a common tool. +- **End-to-end tests** exercise the whole system as a user would. They catch the problems the other levels miss, but are slow and fragile, so keep them to the critical paths. + +The shape is a guide. Some systems need more integration tests (a "testing trophy"), because most of their risk is in how parts fit together. + +## Other kinds of testing + +| Kind | Checks | +| --- | --- | +| Acceptance | The system does what the user or customer needs; often written with them (behaviour-driven development) | +| Performance and load | Response times and throughput under expected and peak load | +| Security | Vulnerabilities: static analysis, dependency scanning, penetration tests, fuzzing | +| Fuzzing | The code survives random and malformed input; especially valuable for parsers and protocol handlers | +| Property-based | General properties hold for many generated inputs (Hypothesis in Python, proptest in Rust) | +| Regression | Bugs once fixed stay fixed | +| Exploratory | A skilled person probing for what nobody thought to script | +| [Hardware-in-the-loop](hardware-in-the-loop-testing.md) | Firmware on real hardware, with simulated inputs and faults | + +[[CAPTION:Table]] Kinds of testing beyond the pyramid. + +## Good habits + +- **Test behaviour, not implementation**, so refactoring does not break tests. +- **Make tests deterministic**. A flaky test is worse than none: people learn to ignore failures. Fix it or delete it. +- **Run them in the [CI pipeline](cicd-pipelines.md)** on every change, fastest first. +- **Write a failing test before fixing a bug**, so it stays fixed. +- **Use test-driven development (TDD)** where it helps: writing the test first improves design and keeps code testable. +- **Design for testability**: [dependency inversion](design-principles.md) and [hexagonal architecture](hexagonal-architecture.md) let you swap real infrastructure for fakes. +- **Treat coverage as a guide, not a target**. Low coverage shows untested code; high coverage does not prove the tests check anything. Mutation testing measures that better. + +## In embedded work + +Run as much as possible on the host: business logic and protocol code compiled for a PC with fakes for the hardware layer (Unity, CppUTest and GoogleTest are common frameworks). Use emulators (QEMU, Renode) for code that needs the processor, and real boards for timing, drivers and power. + +## Where next + +- [Code review](code-review.md) catches what tests cannot. +- The [Guides](../guides/index.md) section covers testing practice in more detail, and the [Templates](../templates/index.md) section has a definition of done. diff --git a/content/primers/tls-and-dtls.md b/content/primers/tls-and-dtls.md new file mode 100644 index 0000000..9df1334 --- /dev/null +++ b/content/primers/tls-and-dtls.md @@ -0,0 +1,59 @@ +--- +title: TLS and DTLS +--- + +# TLS and DTLS + +TLS (Transport Layer Security) creates an encrypted, authenticated channel over a reliable byte stream such as TCP. It secures HTTPS, [MQTT](mqtt.md) on port 8883, email transfer and most other internet protocols. DTLS (Datagram TLS) provides the same protection over UDP, where packets can be lost or arrive out of order, and secures [CoAP](coap.md) and [LwM2M](lwm2m.md). + +TLS replaced SSL, which is long obsolete. TLS 1.0 and 1.1 are deprecated (RFC 8996). Use TLS 1.3 (RFC 8446) where you can, and TLS 1.2 with modern cipher suites where you must. + +## What it provides + +- **Confidentiality**: nobody on the path can read the data. +- **Integrity**: any change in transit is detected. +- **Server authentication**: the client knows it reached the right server, usually through an X.509 certificate (see [PKI and certificates](pki-and-certificates.md)). +- **Optional client authentication**: the server knows which client connected. Mutual TLS (mTLS) with a per-device certificate is the strongest common way to identify IoT devices. + +## The TLS 1.3 handshake + +```mermaid +sequenceDiagram + participant C as Client + participant S as Server + C->>S: ClientHello: supported ciphers, key share + S->>C: ServerHello: chosen cipher, key share + Note over C,S: both derive keys, and the rest is encrypted + S->>C: Certificate, CertificateVerify, Finished + C->>S: (Certificate, CertificateVerify if mutual) Finished + C->>S: application data +``` + +[[CAPTION:Figure]] A full TLS 1.3 handshake takes one round trip. + +TLS 1.3 removed weak options (static RSA key exchange, CBC-mode ciphers, SHA-1, RC4). It leaves five cipher suites, all with authenticated encryption and forward secrecy, so a stolen server key cannot decrypt past sessions. It also supports session resumption with pre-shared keys, and 0-RTT data, which saves a round trip but can be replayed, so use it only for idempotent requests. + +## DTLS + +DTLS adapts TLS to datagrams with sequence numbers, retransmission of handshake messages, and a cookie exchange to resist spoofed-source denial of service. DTLS 1.2 (RFC 6347) is the widely deployed version; DTLS 1.3 (RFC 9147) was published in 2022 and support is growing. + +**Connection ID** (RFC 9146 for DTLS 1.2) matters for cellular IoT: when a sleeping device wakes up, its NAT mapping and IP address may have changed. Without a connection ID the server cannot recognise the session and a new handshake is needed, which can cost more energy than the data being sent. + +## Credentials for constrained devices + +- **Certificates**: the most flexible and the largest. Chains can be several kilobytes. +- **Raw public keys** (RFC 7250): just the public key, without certificate overhead, provisioned in advance. +- **Pre-shared keys (PSK)**: a symmetric key shared in advance. Very light, but the server must hold every device's key securely, and a shared key across devices is a serious weakness. + +## Common mistakes + +- Turning off certificate verification "temporarily" and shipping it. Always verify the chain and the host name. +- No way to update the trust store on devices, so they fail when a root CA changes. +- Clocks that are wrong at boot, so certificate validity checks fail or are skipped. Plan how a device learns the time. +- Old protocol versions or cipher suites left enabled for compatibility. + +## Where next + +- [Cryptography basics](cryptography-basics.md) covers the algorithms underneath. +- The [Standards](../standards/index.md) section sets the TLS versions and cipher suites used here. +- Mozilla's [server-side TLS recommendations](https://wiki.mozilla.org/Security/Server_Side_TLS) are a good reference for server configuration. diff --git a/content/primers/watchdog-timers.md b/content/primers/watchdog-timers.md new file mode 100644 index 0000000..f1276eb --- /dev/null +++ b/content/primers/watchdog-timers.md @@ -0,0 +1,48 @@ +--- +title: Watchdog timers +--- + +# Watchdog timers + +A watchdog timer is a hardware counter that resets the microcontroller if software does not "kick" (or "feed") it within a set time. If the firmware hangs, deadlocks or runs away, the kicks stop and the watchdog restarts the device. For a device in a field or on a roof, that is often the difference between a short gap in data and a site visit. + +## How it works + +1. At start-up, the firmware enables the watchdog with a timeout, say two seconds. +2. During normal operation, the firmware writes to the watchdog register before the timeout expires. +3. If a kick is missed, the watchdog resets the chip. +4. After reset, the firmware reads the reset-cause register, sees a watchdog reset, logs it and carries on. + +Most microcontrollers have an independent watchdog with its own clock, so it keeps working even if the main clock fails. Many also have a windowed watchdog, which resets the chip if the kick comes too early as well as too late. That catches code stuck in a tight loop that happens to include the kick. + +## Kicking it properly + +The common mistake is to kick the watchdog from a timer interrupt or from a low-priority idle loop. The interrupt keeps firing even when every task is deadlocked, so the watchdog never fires and the device sits hung. + +A better design checks that every important task is alive: + +```mermaid +flowchart LR + T1[Sensor task] -- check-in --> S[Supervisor] + T2[Comms task] -- check-in --> S + T3[Control task] -- check-in --> S + S -- kick only if all checked in --> W[Hardware watchdog] +``` + +[[CAPTION:Figure]] A supervisor kicks the watchdog only when every task has reported. + +Each task sets a bit or increments a counter once per cycle. The supervisor kicks the hardware watchdog only when every task has checked in since the last kick, then clears the record. + +## Good habits + +- Enable the watchdog as early in start-up as possible, including in the [bootloader](bootloaders.md). +- Choose a timeout longer than the slowest legitimate operation (a flash erase, a modem attach) and short enough to recover promptly. If an operation is very long, break it up and check in between steps rather than disabling the watchdog. +- Record the reset cause and a little context (the last state, a task ID) in memory that survives reset, and report it. A rising count of watchdog resets in a fleet is an early warning of a bug. +- Avoid reset loops: if the device resets repeatedly at start-up, fall back to a safe mode or the previous firmware. +- Remember that sleep modes may or may not stop the watchdog; check the reference manual. +- Test it: deliberately hang a task on a test build and confirm the device recovers. + +## Where next + +- [Embedded systems design](embedded-systems-design.md) covers the wider reliability picture. +- [Hardware-in-the-loop testing](hardware-in-the-loop-testing.md) shows how to inject faults automatically. diff --git a/content/primers/zero-trust-architecture.md b/content/primers/zero-trust-architecture.md new file mode 100644 index 0000000..5506b99 --- /dev/null +++ b/content/primers/zero-trust-architecture.md @@ -0,0 +1,60 @@ +--- +title: Zero trust architecture +--- + +# Zero trust architecture + +Traditional network security draws a perimeter: firewalls keep attackers out, and anything inside is trusted. That model fails once an attacker gets inside, through phishing, a compromised laptop or a vulnerable device. It also fits poorly with cloud services, remote working and devices spread across the internet. Zero trust drops the idea of a trusted inside. Every request is authenticated, authorised and encrypted, wherever it comes from. + +The phrase is often summed up as "never trust, always verify". NIST SP 800-207 (2020) is the standard reference. + +## Principles + +- **Verify explicitly**: decide every access on identity (of user, device and service), device health, and context such as location and time, not on network location. +- **Least privilege**: grant only the access needed, for as short a time as needed. +- **Assume breach**: design so that one compromised component cannot reach everything. Segment, encrypt internal traffic, and monitor. + +## The parts + +```mermaid +flowchart LR + S[Subject: user, device or service] --> PEP[Policy enforcement point: gateway, proxy or service mesh] + PEP <--> PDP[Policy decision point] + PDP --> ID[Identity provider] + PDP --> DH[Device health and inventory] + PDP --> TI[Threat intelligence and logs] + PEP --> R[Resource] +``` + +[[CAPTION:Figure]] The logical components of NIST SP 800-207. + +- **Policy decision point**: decides whether a request is allowed, using identity, device state and policy. +- **Policy enforcement point**: sits in front of the resource and enforces the decision. It might be an identity-aware proxy, an API gateway or a service mesh sidecar. +- **Signals**: identity from an identity provider, device posture from management tools, behaviour from monitoring. + +## In practice + +- **Strong identity for people**: single sign-on with multi-factor authentication, ideally phishing-resistant (FIDO2 passkeys). See [OAuth 2.0 and OpenID Connect](oauth-and-oidc.md). +- **Strong identity for machines**: every service and device has its own credential, such as a certificate for mutual TLS. See [PKI and certificates](pki-and-certificates.md). +- **Micro-segmentation**: small network zones, with traffic between them allowed only where needed. +- **Encrypt everything**, including traffic inside the data centre or cluster. +- **Replace VPNs** with per-application access through identity-aware proxies (zero trust network access). +- **Continuous monitoring**: log every access decision and look for anomalies. + +## For IoT + +Devices cannot run an agent or answer a multi-factor prompt, so zero trust for IoT relies on: + +- A unique, hardware-protected identity per device, never a shared key. +- Mutual authentication with the platform, and per-device authorisation (a device can publish only to its own topics). +- Network segmentation that puts devices in their own zones with the minimum allowed traffic. +- Monitoring for behaviour that does not fit the device's profile. + +## Getting there + +Zero trust is a direction, not a product. Start with the most valuable resources: put strong identity in front of them, remove broad network trust around them, and add monitoring. Then extend step by step. The US CISA Zero Trust Maturity Model describes the stages. + +## Where next + +- [Secure coding practices](secure-coding-practices.md) and [TLS and DTLS](tls-and-dtls.md) cover parts of the picture. +- The [Handbook](../handbook/index.md) covers security architecture in more depth. diff --git a/py-doc.yml b/py-doc.yml index 6b6f4cf..004a775 100644 --- a/py-doc.yml +++ b/py-doc.yml @@ -38,9 +38,80 @@ documents: outputs: {docx: {}, pdf: {}} primers: - # Primers. The section's own pull request lists the rest of its files here. + # Primers (the old Tech 101s): the overview, then one page per area, each followed by its + # primers. The primers sit one heading level below their area in the Word and PDF files. sources: - content/primers/index.md + - {file: content/primers/architecture.md, page_break: true} + - {file: content/primers/architecture-decision-records.md, heading_offset: 1} + - {file: content/primers/c4-model.md, heading_offset: 1} + - {file: content/primers/design-principles.md, heading_offset: 1} + - {file: content/primers/design-patterns.md, heading_offset: 1} + - {file: content/primers/anti-patterns.md, heading_offset: 1} + - {file: content/primers/hexagonal-architecture.md, heading_offset: 1} + - {file: content/primers/domain-driven-design.md, heading_offset: 1} + - {file: content/primers/event-driven-architecture.md, heading_offset: 1} + - {file: content/primers/modular-monolith-vs-microservices.md, heading_offset: 1} + - {file: content/primers/digital-twins-and-device-shadows.md, heading_offset: 1} + - {file: content/primers/distributed-systems.md, page_break: true} + - {file: content/primers/cap-and-pacelc.md, heading_offset: 1} + - {file: content/primers/consistency-models.md, heading_offset: 1} + - {file: content/primers/idempotency.md, heading_offset: 1} + - {file: content/primers/saga-pattern.md, heading_offset: 1} + - {file: content/primers/apis-and-data.md, page_break: true} + - {file: content/primers/rest-apis.md, heading_offset: 1} + - {file: content/primers/openapi-and-asyncapi.md, heading_offset: 1} + - {file: content/primers/data-formats.md, heading_offset: 1} + - {file: content/primers/cbor.md, heading_offset: 1} + - {file: content/primers/cloud.md, page_break: true} + - {file: content/primers/cicd-pipelines.md, heading_offset: 1} + - {file: content/primers/infrastructure-as-code.md, heading_offset: 1} + - {file: content/primers/serverless-computing.md, heading_offset: 1} + - {file: content/primers/kafka-event-streaming.md, heading_offset: 1} + - {file: content/primers/iot-on-google-cloud.md, heading_offset: 1} + - {file: content/primers/iot-and-embedded.md, page_break: true} + - {file: content/primers/embedded-systems-design.md, heading_offset: 1} + - {file: content/primers/rtos-and-freertos.md, heading_offset: 1} + - {file: content/primers/finite-state-machines.md, heading_offset: 1} + - {file: content/primers/interrupts-and-dma.md, heading_offset: 1} + - {file: content/primers/watchdog-timers.md, heading_offset: 1} + - {file: content/primers/bootloaders.md, heading_offset: 1} + - {file: content/primers/mqtt.md, heading_offset: 1} + - {file: content/primers/coap.md, heading_offset: 1} + - {file: content/primers/lwm2m.md, heading_offset: 1} + - {file: content/primers/cellular-lpwan.md, heading_offset: 1} + - {file: content/primers/firmware-over-the-air.md, heading_offset: 1} + - {file: content/primers/edge-computing-for-iot.md, heading_offset: 1} + - {file: content/primers/hardware-in-the-loop-testing.md, heading_offset: 1} + - {file: content/primers/security.md, page_break: true} + - {file: content/primers/cryptography-basics.md, heading_offset: 1} + - {file: content/primers/pki-and-certificates.md, heading_offset: 1} + - {file: content/primers/tls-and-dtls.md, heading_offset: 1} + - {file: content/primers/post-quantum-cryptography.md, heading_offset: 1} + - {file: content/primers/secure-coding-practices.md, heading_offset: 1} + - {file: content/primers/owasp-top-10.md, heading_offset: 1} + - {file: content/primers/oauth-and-oidc.md, heading_offset: 1} + - {file: content/primers/zero-trust-architecture.md, heading_offset: 1} + - {file: content/primers/etsi-en-303-645.md, heading_offset: 1} + - {file: content/primers/iso-30141.md, heading_offset: 1} + - {file: content/primers/methods-and-delivery.md, page_break: true} + - {file: content/primers/sdlc-models.md, heading_offset: 1} + - {file: content/primers/agile-development.md, heading_offset: 1} + - {file: content/primers/scrum-vs-kanban.md, heading_offset: 1} + - {file: content/primers/event-storming.md, heading_offset: 1} + - {file: content/primers/lightweight-architecture-evaluations.md, heading_offset: 1} + - {file: content/primers/testing-strategies.md, heading_offset: 1} + - {file: content/primers/code-review.md, heading_offset: 1} + - {file: content/primers/technical-debt.md, heading_offset: 1} + - {file: content/primers/docs-as-code.md, heading_offset: 1} + - {file: content/primers/root-cause-analysis.md, heading_offset: 1} + - {file: content/primers/programming-and-tools.md, page_break: true} + - {file: content/primers/git-best-practices.md, heading_offset: 1} + - {file: content/primers/static-code-analysis.md, heading_offset: 1} + - {file: content/primers/plantuml-and-mermaid.md, heading_offset: 1} + - {file: content/primers/python-for-automation.md, heading_offset: 1} + - {file: content/primers/rust-for-systems.md, heading_offset: 1} + - {file: content/primers/terminal-modernisation.md, heading_offset: 1} outputs: {docx: {}, pdf: {}} reference: