diff --git a/.dockerignore b/.dockerignore index d25ccb29..00fec1f3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,17 +5,11 @@ # # License: BUSL-1.1 # Copyright: (c) 2026 HYPERI PTY LIMITED -# -# The dfe-docker image copies its whole context into the folder `init` sets up, -# so this names what that folder holds and nothing more. An allowlist, because -# a file nobody listed then stays out: .env, env/, certs/ and access-summary.md -# hold secrets and can never reach an image layer. -# -# scripts/build_dev_images.py stages each component's source into its own temp -# context, so this file does not reach those builds. +# An allowlist, as init's folder holds only what it names * +# Secrets stay out: .env, env/, certs/, access-summary.md !.env.example !Makefile !VERSION @@ -31,7 +25,9 @@ !scripts/ !service_profiles.yaml -# Inside the kept directories: tests and bytecode never run in a deployment +# Inside kept directories, tests and bytecode never run scripts/tests/ **/__pycache__/ **/*.pyc + +# build_dev_images.py stages its own contexts, so this skips it diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f75cacd1..69e250eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,9 +5,7 @@ # License: BUSL-1.1 - HYPERI PTY LIMITED # Copyright: (c) 2026 HYPERI PTY LIMITED # -# Hand-written: hyperi-ci has no reusable workflow for a compose-packaging repo. -# ubuntu-26.04 is named, not ubuntu-latest, so a label move cannot swap the tools and the arm64 image build, which has no -latest label, runs on the same OS. -# Renovate automerges patch and digest updates once this workflow passes. +# Hand-written (no hyperi-ci compose workflow), runner pinned name: CI @@ -16,26 +14,26 @@ on: push: branches: - main - # The inputs `hyperi-ci release` dispatches with. A bare dispatch only validates. + # `hyperi-ci release` sets these, a bare dispatch only validates workflow_dispatch: inputs: - tag: - type: string + bump: + default: "auto" + description: "Version for from-head: auto | patch | minor (forced) | X.Y.Z (exact)." required: false - default: "" - description: "Existing tag to publish a GitHub release for when it has none." - from-head: type: string - required: false + from-head: default: "" description: "'true' to release the current HEAD. The CI creates the tag." - bump: + required: false type: string + tag: + default: "" + description: "Existing tag to publish a GitHub release for when it has none." required: false - default: "auto" - description: "Version for from-head: auto | patch | minor (forced) | X.Y.Z (exact)." + type: string -# A release run has its own group and is never cancelled, because a cancel after the tag push leaves a tag with no GitHub release. +# Release runs are never cancelled, so no tag loses its release concurrency: group: "${{ github.workflow }}-${{ (inputs.tag != '' || inputs.from-head == 'true' || github.event_name == 'push' && (contains(github.event.head_commit.message, 'Release: true') || contains(github.event.head_commit.message, 'Release:true') || contains(github.event.head_commit.message, 'Publish: true') || contains(github.event.head_commit.message, 'Publish:true'))) && 'release-' || '' }}${{ github.ref }}" cancel-in-progress: "${{ !(inputs.tag != '' || inputs.from-head == 'true' || github.event_name == 'push' && (contains(github.event.head_commit.message, 'Release: true') || contains(github.event.head_commit.message, 'Release:true') || contains(github.event.head_commit.message, 'Publish: true') || contains(github.event.head_commit.message, 'Publish:true'))) }}" @@ -47,7 +45,7 @@ permissions: # The house Python floor. The runner's own python3 is older. env: PYTHON_VERSION: "3.14.7" - # The hyperi-ci CLI the Release job runs. The HYPERCI_INSTALL_OVERRIDE variable replaces it, as in hyperi-ci's own workflows. + # The Release job's CLI, replaced by HYPERCI_INSTALL_OVERRIDE HYPERCI_INSTALL: ${{ vars.HYPERCI_INSTALL_OVERRIDE || 'uvx --python 3.14 --no-cache --refresh hyperi-ci' }} jobs: @@ -99,7 +97,7 @@ jobs: pipx install --python "${{ steps.python.outputs.python-path }}" "pytest==$(make -s print-pytest-version)" pipx inject pytest "pyyaml==$(make -s print-pyyaml-version)" - # The churn tile test reads the stack's ClickHouse pin through scripts/stack.py, which needs oras, and runs the tiles on that image + # The zero_ok tile test reads the ClickHouse pin through oras - name: Install oras uses: oras-project/setup-oras@5e2f2482b30b0cb3e14e67f89a80c88c659a7c63 # v2.0.3 @@ -146,8 +144,7 @@ jobs: - name: Proxy configs run: make check-proxy -# Only Docker on PATH: each profile is set up, and the default one starts, serves -# the console and runs the demo. The Windows job runs the default install. + # Only Docker on PATH: every profile set up, the default run image: name: Image runs-on: ubuntu-26.04 @@ -165,11 +162,11 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} - # The stack deployment.example.yaml pins, as image.yml bakes into the : image + # The stack deployment.example.yaml pins, as image.yml bakes in - name: Build the image run: docker build --file docker/dfe-docker.Dockerfile --build-arg "IMAGE=$IMAGE" --build-arg "STACK=$(python3 scripts/quickstart_json.py --print-stack)" --tag "$IMAGE" . - # docker, curl for the console check and the few tools the checks below use. No make, no python + # docker, curl and a few check tools. No make, no python - name: Put only Docker on PATH run: | bin="$RUNNER_TEMP/docker-only" @@ -208,7 +205,7 @@ jobs: docker compose config -q docker compose config --services > "$WORK/default.services" - # A second init, a bad profile and an upgrade, each against what it must leave alone + # Re-init, a bad profile and an upgrade must leave state alone - name: Refuse and upgrade run: | export PATH="$DOCKER_ONLY" @@ -238,17 +235,17 @@ jobs: exit 1 fi - # A throwaway install whose stack is torn down below, so its minted secrets protect nothing + # A throwaway install, so its minted secrets protect nothing - name: Hand the default install to the Windows job uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: + if-no-files-found: error + include-hidden-files: true name: default-install path: | ${{ env.WORK }}/default ${{ env.WORK }}/default.services !${{ env.WORK }}/default/access-summary.md - include-hidden-files: true - if-no-files-found: error retention-days: 1 - name: Start the default profile and reach the console @@ -274,7 +271,7 @@ jobs: run: | docker compose ps --all docker compose logs --no-color --tail 100 - # Low runner disk stops Redpanda and ClickHouse taking writes, so a demo that lands nothing shows it here + # A full runner disk stops writes, so a rowless demo shows it df -h "$(docker info --format '{{.DockerRootDir}}')" || true docker compose exec -T kafka-redpanda rpk cluster health || true @@ -286,7 +283,7 @@ jobs: docker compose --profile '*' down --volumes --remove-orphans fi - # One Published compose leg per demo pack: resources/demo, then each resources/demo- directory in byte order, as scripts/_common.py demo_pack_names lists them + # One Published leg per pack, in demo_pack_names order demo-packs: name: Demo Packs runs-on: ubuntu-26.04 @@ -305,7 +302,7 @@ jobs: dirs="$({ echo demo; find resources -mindepth 1 -maxdepth 1 -type d -name 'demo-*' -printf '%f\n' | LC_ALL=C sort; } | jq -Rnc '[inputs]')" echo "dirs=$dirs" | tee -a "$GITHUB_OUTPUT" - # The files a release publishes as OCI artifacts, generated from this commit and run from a folder holding nothing else, as `-f oci://` runs them. Each demo file holds the whole stack plus its pack, so it starts and has to land the data its pack replays. The demo leg also writes the stack file, which is read and checked + # Release OCI files from this commit, run from an empty folder published: name: Published compose (${{ matrix.pack }}) needs: demo-packs @@ -337,7 +334,7 @@ jobs: - name: Build the image run: docker build --file docker/dfe-docker.Dockerfile --build-arg "IMAGE=$IMAGE" --tag "$IMAGE" . - # The image is local here, so its digest is the one finding allowed + # The image is local, so its digest is the one finding allowed - name: Generate and check the published files env: PACK: ${{ matrix.pack }} @@ -350,6 +347,7 @@ jobs: python3 scripts/publish_compose.py --image "$IMAGE" --out-dir "$RUNNER_TEMP/compose" "${pack[@]}" echo "COMPOSE_DIR=$RUNNER_TEMP/compose" >> "$GITHUB_ENV" + # The demo file holds the whole stack plus the demo - name: Start the demo and reach the console timeout-minutes: 25 working-directory: ${{ env.COMPOSE_DIR }} @@ -357,7 +355,7 @@ jobs: docker compose -f "$FILE" up -d --wait --wait-timeout 1200 curl -fsSL --retry 5 --retry-all-errors -o /dev/null -w 'console answered %{http_code} at %{url_effective}\n' http://localhost:3000/ - # The secrets are this runner's own: no service holds one in its environment, ClickHouse refuses its empty default password, and the login the file prints is held for a change + # Secrets minted here: none in an env, no defaults accepted - name: Run on secrets minted here working-directory: ${{ env.COMPOSE_DIR }} run: | @@ -393,7 +391,7 @@ jobs: # Runs after the secrets step because the console login check replaces the login's issued password, which that step expects held. One checker proves the pack worked: its replayed sources land parsed rows, the intake and default tables stay empty, every source is deployed, each hunt fires under its ceiling, each dashboard and saved search returns rows, the console login can search, replayed fetcher polls land and the dead-letter queue stays empty. What a pack expects is data in resources//verify.json - name: Verify the pack worked - # Above the checker's summed waits (land, hunts, console and fetchers: 38 min), so a slow failure still prints its results + # Over the checker's 38 min of summed waits, so failures print timeout-minutes: 40 working-directory: ${{ env.COMPOSE_DIR }} env: @@ -413,7 +411,7 @@ jobs: esac done docker compose -f "$FILE" logs --no-color --tail 100 "${services[@]}" - # A broker under its free-space floor takes no writes, so a pack that lands nothing shows it here + # A broker under its disk floor takes no writes, shown here docker compose -f "$FILE" exec -T kafka-redpanda rpk cluster health || true docker compose -f "$FILE" exec -T kafka-redpanda df -h /var/lib/redpanda/data || true df -h "$(docker info --format '{{.DockerRootDir}}')" || true @@ -425,7 +423,7 @@ jobs: docker compose -f "$COMPOSE_DIR/$FILE" down --volumes --remove-orphans fi - # Compose on Windows splits COMPOSE_FILE on `;` unless COMPOSE_PATH_SEPARATOR in .env says otherwise. The one runner exception: only a Windows runner runs that path. + # Windows splits COMPOSE_FILE unless .env sets a separator image-windows: name: Image (Windows compose) needs: image @@ -453,35 +451,36 @@ jobs: } Write-Output "Windows compose starts the same services: $got" -# Tags and publishes a release on `Release: true` or a `hyperi-ci release` dispatch, -# after the image jobs. The App's token publishes it so image.yml starts on it. + # Tags and publishes on `Release: true` or a dispatch release: name: Release + # An image that fails CI never gets a tag needs: [compose, dockerfile, image, image-windows, published] if: github.event_name != 'pull_request' runs-on: ubuntu-26.04 timeout-minutes: 20 - # semantic-release's dry run checks push access, which needs write + # A dry run checks push access, which needs write permissions: contents: write + # Shared actions: hyperi-ci's release takes four languages steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ inputs.tag || github.sha }} # semantic-release reads every commit since the last tag fetch-depth: 0 persist-credentials: false + ref: ${{ inputs.tag || github.sha }} - # The trailer gate, the classification check and the version, as every hyperi-ci pipeline decides them + # Trailer gate, classification and version, as hyperi-ci does - name: Plan the release id: plan uses: hyperi-io/hyperi-ci/.github/actions/predict-version@main with: + bump: ${{ inputs.bump || 'auto' }} + from-head: ${{ inputs.from-head }} github-token: ${{ secrets.GITHUB_TOKEN }} tag: ${{ inputs.tag }} - from-head: ${{ inputs.from-head }} - bump: ${{ inputs.bump || 'auto' }} - name: Install uv if: steps.plan.outputs.will-release == 'true' @@ -489,18 +488,19 @@ jobs: with: enable-cache: false + # A GITHUB_TOKEN release starts no image.yml run - name: Mint the release token id: bot if: steps.plan.outputs.will-release == 'true' uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 with: client-id: ${{ vars.GH_APP_CLIENT_ID }} - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} - repositories: ${{ github.event.repository.name }} permission-contents: write + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + repositories: ${{ github.event.repository.name }} - # Plan installed semantic-release and the central config in this job + # Plan installed semantic-release and its config in this job - name: Tag (semantic-release) if: steps.plan.outputs.will-release == 'true' && (github.event_name == 'push' || inputs.from-head == 'true' && inputs.bump == 'auto') env: @@ -514,7 +514,7 @@ jobs: RELEASE_VERSION: ${{ steps.plan.outputs.version }} run: ${{ env.HYPERCI_INSTALL }} tag-head --bump "$RELEASE_VERSION" - # semantic-release tags nothing when this checkout is behind main, and gh release create would then tag main's tip + # Behind main nothing is tagged, so gh would tag main's tip - name: Check the tag is on this commit if: steps.plan.outputs.will-release == 'true' env: @@ -554,7 +554,7 @@ jobs: fi gh release create "$tag" --repo "$GITHUB_REPOSITORY" --verify-tag --title "$tag" --generate-notes "${kind[@]}" - # Non-fatal: the release is already published. A tag dispatch checks out an old tag, so it commits nothing back + # Non-fatal: the release is out, and an old tag commits nothing - name: Commit VERSION and CHANGELOG.md back if: steps.plan.outputs.will-release == 'true' && inputs.tag == '' continue-on-error: true diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 9a38811d..ae4b6af3 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -4,24 +4,13 @@ # # License: BUSL-1.1 - HYPERI PTY LIMITED # Copyright: (c) 2026 HYPERI PTY LIMITED -# -# Two images per release, built from one Dockerfile on each architecture's own -# runner. `:` installs the stack deployment.example.yaml pins at the -# release tag, so it installs the same stack every time. `:latest` installs the -# newest published stack when init runs, and moves only on a release that is not -# a pre-release. A pull request that touches the image builds both on both -# architectures and pushes nothing. + +# Two images per release, both built from one Dockerfile name: Image on: - release: - types: [published] - workflow_dispatch: - inputs: - tag: - description: The release tag to build and attach quickstart.json to - required: true + # A pull request builds both images and pushes nothing pull_request: paths: - .dockerignore @@ -29,6 +18,13 @@ on: - Makefile - docker/dfe-docker.Dockerfile - scripts/** + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: The release tag to build and attach quickstart.json to + required: true permissions: contents: read @@ -59,8 +55,8 @@ jobs: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ env.TAG || github.sha }} persist-credentials: false + ref: ${{ env.TAG || github.sha }} - name: Set up Python uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 @@ -74,7 +70,7 @@ jobs: id: release run: | version="${TAG#v}" - # Assigned on its own line so a prerelease pin fails this step, not the publish. + # Own line, so a prerelease pin fails here, not at publish stack="$(python3 scripts/quickstart_json.py --print-stack)" { echo "version=${version:-0.0.0-pr}" @@ -88,7 +84,7 @@ jobs: GH_TOKEN: ${{ github.token }} run: printf '%s' "$GH_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin - # Pushed by digest alone, so the publish job tags both architectures together + # Pushed by digest only, the publish job tags both architectures - name: Build both images env: PUSH: ${{ steps.release.outputs.push }} @@ -96,6 +92,7 @@ jobs: VERSION: ${{ steps.release.outputs.version }} run: | mkdir -p "$RUNNER_TEMP/digests" + # : bakes in the pinned stack, :latest the newest for variant in latest pinned; do if [ "$variant" = latest ]; then stack=latest @@ -120,9 +117,9 @@ jobs: if: steps.release.outputs.push == 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: + if-no-files-found: error name: digests-${{ matrix.arch }} path: ${{ runner.temp }}/digests/ - if-no-files-found: error retention-days: 1 publish: @@ -138,8 +135,8 @@ jobs: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ env.TAG }} persist-credentials: false + ref: ${{ env.TAG }} - name: Set up Python uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 @@ -152,15 +149,16 @@ jobs: - name: Fetch the digests uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - pattern: digests-* - path: ${{ runner.temp }}/digests merge-multiple: true + path: ${{ runner.temp }}/digests + pattern: digests-* - name: GHCR login env: GH_TOKEN: ${{ github.token }} run: printf '%s' "$GH_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + # :latest moves only on a release that is not a pre-release - name: Tag both architectures env: GH_TOKEN: ${{ github.token }} @@ -179,13 +177,14 @@ jobs: - name: Install oras uses: oras-project/setup-oras@5e2f2482b30b0cb3e14e67f89a80c88c659a7c63 # v2.0.3 - # The compose files go into the same package under their own tags, so the image tags above stay as they are: the stack and demo, then each resources/demo- as demo-. The demo's helpers run the image just tagged, by digest + # Own tags per compose file, demo- for each demo pack - name: Publish the compose files env: GH_TOKEN: ${{ github.token }} run: | version="${TAG#v}" docker compose version + # The demo's helpers run the image just tagged, by digest digest="$(docker buildx imagetools inspect "$REPOSITORY:$version" --format '{{.Manifest.Digest}}')" moving=() if [ "$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json isPrerelease --jq .isPrerelease)" = false ]; then diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f578ca30..d9e378c5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,12 +106,7 @@ green instead of bouncing: hyperi-ci check ``` -`hyperi-ci` is the HyperI CI CLI (public on PyPI). If you do not have it, install -it once with `uv tool install hyperi-ci` (or `pipx install hyperi-ci`), or run it -ad hoc with `uvx hyperi-ci check`. It runs the project's full local validation -- -the same suite CI runs -- and reports what to fix. Run it before every push and before opening a -pull request. It is the single best way to give your change the best chance of -surviving CI. +`hyperi-ci` is the HyperI CI CLI (public on PyPI). If you do not have it, install it once with `uv tool install hyperi-ci` (or `pipx install hyperi-ci`) or run it ad hoc with `uvx hyperi-ci check`. It runs the project's full local validation - the same suite CI runs - and reports what to fix. Run it before every push and before opening a pull request. It is the single best way to give your change the best chance of surviving CI. ## CI/CD Workflow @@ -133,7 +128,7 @@ If you have questions about contributing, please open an issue or contact us. ### What this is -dfe-docker stands the whole DFE suite up as one Docker Compose stack on a single host, and it is the definition-of-done target for the suite's docker path: a change is proven when it runs on the deployment target, not because it ran on someone's laptop. It packages the stack: its release builds the dfe-docker image that sets a folder up from this tree, publishes it with the Compose artefacts and `quickstart.json`, and builds no component image (`make dev` builds those locally, for a test rig). It provisions no ClickHouse table and no Kafka topic. dfe-engine is the only schema and topic controller, and `scripts/tests/test_engine_only_schema_control.py` fails the build on a DDL file or a topic-creating step appearing here. When the data path is wrong the fix is nearly always in a component repo. It is not the Kubernetes path either: Kubernetes stays primary and dfe-infra owns it, which is why there is deliberately no `scale` profile - Compose cannot run an HA broker or a ClickHouse cluster. `main` is protected, so every change arrives as a pull request, with Conventional Commits ([CONTRIBUTING.md](CONTRIBUTING.md)). +dfe-docker stands the whole DFE suite up as one Docker Compose stack on a single host and it is the definition-of-done target for the suite's docker path: a change is proven when it runs on the deployment target, not because it ran on someone's laptop. It packages the stack: its release builds the dfe-docker image that sets a folder up from this tree, publishes it with the Compose artefacts and `quickstart.json` and builds no component image (`make dev` builds those locally, for a test rig). It provisions no ClickHouse table and no Kafka topic. dfe-engine is the only schema and topic controller and `scripts/tests/test_engine_only_schema_control.py` fails the build on a DDL file or a topic-creating step appearing here. When the data path is wrong the fix is nearly always in a component repo. It is not the Kubernetes path either: Kubernetes stays primary and dfe-infra owns it, which is why there is deliberately no `scale` profile - Compose cannot run an HA broker or a ClickHouse cluster. `main` is protected, so every change arrives as a pull request, with Conventional Commits ([CONTRIBUTING.md](CONTRIBUTING.md)). ### Where things live @@ -167,11 +162,11 @@ make test-e2e # the declarative suite, one stack per test make down # stop and remove containers across EVERY profile ``` -`make check` is `check-compose check-hardfail check-dockerfile check-docs check-proxy check-python check-tests check-locks`, and CI runs each one through the same make target so the two cannot drift. +`make check` is `check-compose check-hardfail check-dockerfile check-docs check-proxy check-python check-tests check-locks` and CI runs each one through the same make target so the two cannot drift. Two ways a green run says less than it looks: -- `make check` does not start the stack. It resolves compose, lints the helpers and unit-tests them, and `check-proxy` runs temporary Envoy containers to validate the proxy config. CI's image and published-compose jobs start the stack from the dfe-docker image and check each demo source lands rows, but CI runs neither `make post` nor `make test-e2e`, so run those for a change to the data path. +- `make check` does not start the stack. It resolves compose, lints the helpers and unit-tests them and `check-proxy` runs temporary Envoy containers to validate the proxy config. CI's image and published-compose jobs start the stack from the dfe-docker image and check each demo source lands rows, but CI runs neither `make post` nor `make test-e2e`, so run those for a change to the data path. - `check-compose` validates compose STRUCTURE, not digests. It substitutes placeholders for the mandatory keys so it reads neither the stack SSoT nor the registry, which means no particular pin is proved to resolve. `check-hardfail` resolves compose with a scrubbed environment and an empty `--env-file` so a developer's pinned `.env` cannot mask it, requires the run to fail on a missing pin, then reads every `image:` line and requires each to resolve to an `@sha256:` digest. @@ -185,12 +180,12 @@ Two ways a green run says less than it looks: | Assume events go through Kafka | Read the profile's `transport` first | A `-grpc` profile is `transport: grpc`: the receiver dials `dfe-loader:6000` directly and no broker starts. The `-kafka` profiles, the shipped `single-kafka` among them, start one. The core data path is otherwise identical. `post` and `test-e2e` assert the same landing row either way | | Add a DDL file or a topic-creating step here | Change the schema in dfe-engine | dfe-engine ships the schemas in its own image and reports healthy only once it has applied them. `scripts/tests/test_engine_only_schema_control.py` checks the deployment trees for the schema and topic bootstrap patterns it knows, with exceptions for the self-test and the e2e cleanup (#118) | | Give an external ClickHouse's port through `CLICKHOUSE_HTTP_PORT` or `CLICKHOUSE_NATIVE_PORT` | Set `CLICKHOUSE_EXTERNAL_HTTP_PORT` and `CLICKHOUSE_EXTERNAL_NATIVE_PORT` | The first pair is the bundled container's host publish and nothing else, so it is safe to move for a second stack. Services on the network dial 8123/9000 unless the EXTERNAL pair says otherwise (#75) | -| Move the loader's `grpc.listen` off 6000, or confuse it with the receiver's 6000 | Keep the loader on container port 6000 | dfe-engine compiles every sender's loader endpoint as `dfe-loader:6000`, so a loader listening anywhere else drops every record behind an HTTP 202 with nothing logged. The receiver's 6000 is the external Vector protocol, off until `config.grpc.enabled` is set through the engine, because under `--config` the receiver takes only its flat env keys, and unpublished until `DFE_RECEIVER_GRPC_PORT` is set. The loader's is the internal `scalo.transport.v1.Transport/Push` and publishes no host port. `scripts/tests/test_loader_grpc_port.py` fails the build on a mismatch | +| Move the loader's `grpc.listen` off 6000 or confuse it with the receiver's 6000 | Keep the loader on container port 6000 | dfe-engine compiles every sender's loader endpoint as `dfe-loader:6000`, so a loader listening anywhere else drops every record behind an HTTP 202 with nothing logged. The receiver's 6000 is the external Vector protocol, off until `config.grpc.enabled` is set through the engine, because under `--config` the receiver takes only its flat env keys and unpublished until `DFE_RECEIVER_GRPC_PORT` is set. The loader's is the internal `scalo.transport.v1.Transport/Push` and publishes no host port. `scripts/tests/test_loader_grpc_port.py` fails the build on a mismatch | | Publish the UIs beyond loopback and leave `DFE_EXTERNAL_ORIGIN` alone | Set it to the address browsers actually use | `DFE_BIND_SCOPE=all` with a loopback origin builds Search's frame-ancestors policy and every next-auth redirect from the wrong host, so the console comes up with its observability views blocked and sends logins to the wrong machine. The Makefile now refuses the combination (#108) | | Raise `REDPANDA_MEMORY` on its own | Raise `DFE_BROKER_MEMORY` above it | The broker gets at most the container limit less the host's `vm.min_free_kbytes`: `redpanda/start.sh` lowers `--memory` to that and logs a WARN naming the `DFE_BROKER_MEMORY` that restores it (`.env.example:229`, `.env.example:519`) | | Publish the broker's in-network listener `9092` to the host | Use the host listener on `19092` (`KAFKA_PLAINTEXT_HOST_PORT`) | `9092` advertises `kafka:9092`, a name a host client can't resolve. It also collides with an always-on dev Kafka (#36) | | Set `DFE_UPDATE_WIPE_STATE=1` on a box that also sets `DFE_DATA_ROOT` | Leave `DFE_DATA_ROOT` unset if you rely on the wipe | `make clean` does not chain the storage overlay, so it removes the volume objects while the bind directories keep their contents and the wipe silently becomes a no-op (`ops/daemon-update/README.md`) | -| Take a local, gitignored note as current | Read the tracked file you are asking about | A local note is nobody's to maintain, and the ones seen so far named repos and directories that no longer exist | +| Take a local, gitignored note as current | Read the tracked file you are asking about | A local note is nobody's to maintain and the ones seen so far named repos and directories that no longer exist | ### Profile resolution @@ -292,7 +287,7 @@ The suite graph in `dfe-infra/suite.yaml` records exactly one edge on this repo | Repo | Direction | How they interact | |---|---|---| -| dfe-infra | inbound, `derived-pins` | dfe-docker holds no copy of the pins. `scripts/stack.py` renders them at `make stack` time, from a local dfe-infra checkout when `DFE_INFRA_DIR` names one, else from the signed OCI stack manifest pulled with `oras`. `ops/daemon-update/self_update.py` is the second consumer of that manifest. The edge's declared check is "nothing" - there is no second copy that can drift, and the graph records the edge so the suite tooling knows to walk past it. Do not go hunting for a pin file here to update | -| dfe-receiver, dfe-fetcher, dfe-loader, dfe-archiver, dfe-transform-elastic, dfe-transform-vector, dfe-transform-vrl, dfe-engine, dfe-ui, dfe-hyperdx | inbound | This stack runs their published GHCR images at the digests `make stack` pins, and `make dev` builds the same components from source into local images. Nothing flows the other way | +| dfe-infra | inbound, `derived-pins` | dfe-docker holds no copy of the pins. `scripts/stack.py` renders them at `make stack` time, from a local dfe-infra checkout when `DFE_INFRA_DIR` names one, else from the signed OCI stack manifest pulled with `oras`. `ops/daemon-update/self_update.py` is the second consumer of that manifest. The edge's declared check is "nothing" - there is no second copy that can drift and the graph records the edge so the suite tooling knows to walk past it. Do not go hunting for a pin file here to update | +| dfe-receiver, dfe-fetcher, dfe-loader, dfe-archiver, dfe-transform-elastic, dfe-transform-vector, dfe-transform-vrl, dfe-engine, dfe-ui, dfe-hyperdx | inbound | This stack runs their published GHCR images at the digests `make stack` pins and `make dev` builds the same components from source into local images. Nothing flows the other way | Nothing in the suite graph depends on dfe-docker, so a change here breaks a deployment rather than another repo's build. `dfe-transform-splack` and `dfe-transform-wasm` have repos but no service in this stack. diff --git a/Makefile b/Makefile index 37ff199b..da7ebd87 100644 --- a/Makefile +++ b/Makefile @@ -383,7 +383,7 @@ else ifneq ($(filter dev dev-build,$(MAKECMDGOALS)),) else ifeq ($(filter ci,$(MAKECMDGOALS)),) RESOLVE_LOCAL_ARGS = --local-overlay $(LOCAL_OVERLAY) endif -# The goals that start or recreate services, or hand a plain `docker compose` what they would, the only ones that ask docker compose for the CPU limits the collector reports as cores deployed +# Goals that start or feed compose, which need its CPU limits USAGE_GOALS := dev ci up infra apply apply-services print-compose-env .tmp/profile.mk: FORCE @python3 scripts/resolve_profile.py $(if $(KAFBAT_GATED),--unpublished kafbat) $(RESOLVE_LOCAL_ARGS) $(if $(filter $(USAGE_GOALS),$(MAKECMDGOALS)),--usage) @@ -621,10 +621,11 @@ check-locks: ## Assert each inline-metadata script's lock matches its pins (relo print-compose-file: @echo "COMPOSE_FILE=$(COMPOSE_FILE)" -# What a plain `docker compose` reads from .env in a folder the dfe-docker image set up (scripts/installer.py writes it there): this run's chain and compose profiles, and each value make works out rather than reads from .env. The live UI chain is a `make dev` matter and stays out. An ORIGIN_GOALS member, because a plain `docker compose up` starts from it +# The .env lines installer.py writes for a plain compose up empty := space := $(empty) $(empty) comma := , +# The live UI chain is a `make dev` matter and stays out .PHONY: print-compose-env print-compose-env: $(info COMPOSE_FILE=docker-compose.yml$(STORAGE_CHAIN)$(UI_CHAIN)) @@ -635,18 +636,19 @@ print-compose-env: $(if $(filter true,$(strip $(DFE_PROXY_TLS))),$(foreach key,DFE_NETWORK_IP_RANGE DFE_NETWORK_SUBNET DFE_PROXY_CERT_DIR DFE_PROXY_IP,$(info $(key)=$($(key))))) @: -# The profile `make up` and the dfe-docker image start when none is chosen +# The profile `make up` and the image start when none is chosen .PHONY: print-default-profile print-default-profile: @echo $(DEFAULT_PROFILE) -# The replay script, protocol and rate `make demo` streams with, which the published demo streams with too +# The replay script, protocol and rate both demos stream with .PHONY: print-demo-stream print-demo-stream: @echo $(DEMO_GENERATOR) $(DEMO_PROTOCOL) $(DEMO_RATE) -# The two compose files a release publishes as OCI artifacts, for the stack VERSION names (deployment.example.yaml's pin when unset). PUBLISH_IMAGE is the dfe-docker image the demo's helpers run +# Release compose files, for VERSION's stack or the pinned one PUBLISH_DIR ?= .tmp/publish +# The dfe-docker image the demo's helpers run PUBLISH_IMAGE ?= ghcr.io/hyperi-io/dfe-docker .PHONY: publish-compose publish-compose: ## Write the stack and demo compose files a release publishes into PUBLISH_DIR and check them (VERSION=X.Y.Z, PUBLISH_IMAGE=) diff --git a/README.md b/README.md index 7be5a3fb..66487706 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ The whole DFE stack on one host with Docker Compose, from pinned release images - Runs DFE on one box, an edge site, a partner deployment or a developer's test rig. Compose is a supported production target for small environments. Kubernetes stays the primary path, so where both would work, choose Kubernetes. - Pins every image by digest from a certified DFE stack release with `make stack`. With a pin unset the stack will not start rather than pull `latest`. -- Starts from the registry images with `make up`, or builds chosen components from source with `make dev`. +- Starts from the registry images with `make up` or builds chosen components from source with `make dev`. - Runs Kafka or Redpanda as the broker, with opt-in overlays for TLS, storage and an oauth2-proxy `auth` profile. - Checks that the running stack moves data from the ingest edge to a ClickHouse row with `make post`, which fails if an enabled run asserts nothing. `DFE_POST_ENABLED=false` skips it. - Loads seed sources, rules and hunts into a running engine with `make sources`, `make rules` and `make hunts`. @@ -34,7 +34,7 @@ docker compose -f oci://ghcr.io/hyperi-io/dfe-docker:demo up -d docker compose -f oci://ghcr.io/hyperi-io/dfe-docker:demo run --rm secrets-init login ``` -`down` with the same `-f` stops either one, and `down -v` deletes its data. +`down` with the same `-f` stops either one and `down -v` deletes its data. From a clone, on a host with Docker Compose v2, make, git and Python 3: @@ -51,23 +51,23 @@ make down # stop everything, keep the data ## Configuration -A profile is `-`: `slim-grpc`, `slim-kafka`, `single-grpc` or `single-kafka`. `slim` is the core data path alone. `single-kafka` adds the per-source apps and the archiver, and `single-grpc` runs the same services as `slim-grpc`, because the engine runs per-source apps on Kafka only. The engine still runs as `docker-single` there and as `docker-slim` on a `slim` profile. There is no `scale` tier, because Compose cannot scale out. `KAFKA_BACKEND=apache` swaps Redpanda, which is BSL, for Apache Kafka on the kafka profiles. +A profile is `-`: `slim-grpc`, `slim-kafka`, `single-grpc` or `single-kafka`. `slim` is the core data path alone. `single-kafka` adds the per-source apps and the archiver and `single-grpc` runs the same services as `slim-grpc`, because the engine runs per-source apps on Kafka only. The engine still runs as `docker-single` there and as `docker-slim` on a `slim` profile. There is no `scale` tier, because Compose cannot scale out. `KAFKA_BACKEND=apache` swaps Redpanda, which is BSL, for Apache Kafka on the kafka profiles. -Each service reads its own `env/.env`, created from `env.example/` by `make init`, which never overwrites a file and reports the keys an old `.env` is missing. Only the console and Search authenticate. The ingest ports, Kafbat and the metrics ports ask for no login, so put the stack behind a VPN, a tunnel or a firewall before anyone you have not met can reach it, or arm the `auth` profile. Every variable, the port map and the image list are in [docs/configuration.md](docs/configuration.md). +Each service reads its own `env/.env`, created from `env.example/` by `make init`, which never overwrites a file and reports the keys an old `.env` is missing. Only the console and Search authenticate people, while the receiver's HTTP listener takes only bearer tokens once an ingest credential names it. The other ingest ports, Kafbat and the metrics ports ask for no login, so put the stack behind a VPN, a tunnel or a firewall before anyone you have not met can reach it or arm the `auth` profile. Every variable, the port map and the image list are in [docs/configuration.md](docs/configuration.md). ## Documentation Pick the page for what you are doing: - [docs/evaluating.md](docs/evaluating.md): a first look, a demo, a proof of concept. -- [docs/operating.md](docs/operating.md): running it where other people depend on it, and exactly what is exposed. -- [docs/deploying.md](docs/deploying.md): standing a deployment up, or moving one to a newer certified stack. +- [docs/operating.md](docs/operating.md): running it where other people depend on it and exactly what is exposed. +- [docs/deploying.md](docs/deploying.md): standing a deployment up or moving one to a newer certified stack. - [docs/developing.md](docs/developing.md): using the stack as a test rig for a component you are changing. - [docs/troubleshooting.md](docs/troubleshooting.md), then [known issues](docs/known-issues.md) and [common failures](docs/common-failures.md). - [docs/observability.md](docs/observability.md): health endpoints, self-telemetry and what a passing self test proves. - [docs/profiles.md](docs/profiles.md) and [docs/architecture.md](docs/architecture.md): how the profiles and the pieces fit together. -The suite's published docs are at , and this repository's pages at . +The suite's published docs are at and this repository's pages at . ## Help and contributing diff --git a/compose/one-shots.yml b/compose/one-shots.yml index cc999237..64363e9f 100644 --- a/compose/one-shots.yml +++ b/compose/one-shots.yml @@ -153,14 +153,16 @@ services: extends: {file: compose/limits.yml, service: limits-sidecar} entrypoint: ["/bin/bash", "/usr/local/bin/dfe-redpanda-configure.sh"] environment: + REDPANDA_LICENSE: ${REDPANDA_LICENSE:-} REDPANDA_MIN_FREE_BYTES: ${REDPANDA_MIN_FREE_BYTES:-5368709120} REDPANDA_RETENTION_BYTES: ${REDPANDA_RETENTION_BYTES:-10737418240} - REDPANDA_LICENSE: ${REDPANDA_LICENSE:-} depends_on: kafka-redpanda: condition: service_healthy volumes: - ./redpanda/configure.sh:/usr/local/bin/dfe-redpanda-configure.sh:ro + # Stays empty: covers the image's VOLUME for make clean + - kafka-redpanda-config-data:/var/lib/redpanda/data restart: "no" hyperdx-documentdb-tls: @@ -175,7 +177,7 @@ services: set -eu umask 077 cd /tls - # Running containers mount ca and server by subpath, so the directories stay and each file is renamed into place whole + # Mounted by subpath, so files are renamed in and the dirs stay mkdir -p ca server if ! { [ -s ca/ca.crt ] && [ -s server/tls.key ] && [ -s server/tls.crt ] \ && openssl x509 -checkend 2592000 -noout -in server/tls.crt >/dev/null; }; then diff --git a/config/clickhouse/disk-reserve.xml b/config/clickhouse/disk-reserve.xml index 9092efb1..a29c1210 100644 --- a/config/clickhouse/disk-reserve.xml +++ b/config/clickhouse/disk-reserve.xml @@ -8,7 +8,7 @@ Copyright: (c) 2026 HYPERI PTY LIMITED --> - + diff --git a/config/gnmic/gnmic.yaml b/config/gnmic/gnmic.yaml index a0e31d21..d47ff231 100644 --- a/config/gnmic/gnmic.yaml +++ b/config/gnmic/gnmic.yaml @@ -6,7 +6,7 @@ # License: BUSL-1.1 # Copyright: (c) 2026 HYPERI PTY LIMITED -# gnmic exits until a `targets:` map names a device. docs/gnmi.md has the keys. +# gnmic exits until `targets:` names a device (docs/gnmi.md) api-server: address: 127.0.0.1:7890 diff --git a/config/proxy/envoy.tls.yaml b/config/proxy/envoy.tls.yaml index 446510ea..1c2903d7 100644 --- a/config/proxy/envoy.tls.yaml +++ b/config/proxy/envoy.tls.yaml @@ -1,6 +1,6 @@ # Project: dfe-docker # File: config/proxy/envoy.tls.yaml -# Purpose: envoy.yaml with TLS on both listeners +# Purpose: envoy.yaml with TLS on every listener # Language: Envoy static config # # License: BUSL-1.1 @@ -8,7 +8,7 @@ # # Mounted instead of envoy.yaml when DFE_PROXY_TLS=true # Must equal envoy.yaml plus the marked TLS blocks (test_proxy_tls_config.py) -# The receiver-ingest listener exists only here: TLS is its whole purpose +# The receiver-ingest listener is TLS only, so lives here only admin: address: @@ -163,7 +163,7 @@ static_resources: transport_socket: *console_tls # END TLS # BEGIN TLS - # External pushers send to the receiver here, over the console's certificate + # External pushers reach the receiver here, on the console cert - name: receiver-ingest address: socket_address: { address: 0.0.0.0, port_value: 8443 } @@ -182,7 +182,7 @@ static_resources: - name: receiver domains: ["*"] routes: - # The timeout bounds the wait for the receiver's reply. The receiver enforces its own body size and request timeout + # Waits on the receiver's reply, which sets its own limits - match: { prefix: / } route: { cluster: dfe-receiver, timeout: 120s } stat_prefix: receiver_ingest diff --git a/docker-compose.yml b/docker-compose.yml index 27c3dc0e..f275f8a1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ # License: BUSL-1.1 # Copyright: (c) 2026 HYPERI PTY LIMITED # -# Compose profiles and the services each starts: docs/architecture.md. +# Profiles and the services each starts: docs/architecture.md include: - path: compose/one-shots.yml @@ -505,7 +505,7 @@ services: OTEL_EXPORTER_OTLP_ENDPOINT: ${DFE_OTEL_EXPORTER_ENDPOINT:-} OTEL_SERVICE_NAME: dfe-hunt-runner volumes: - # Only the alert destinations from the engine's file secrets backend, never the whole root. + # Alert destinations only, never the whole secrets root - type: volume source: dfe-engine-config target: /app/config/.secrets/alert-destinations @@ -798,5 +798,6 @@ volumes: kafka-apache-config: kafka-apache-data: kafka-apache-secrets: + kafka-redpanda-config-data: kafka-redpanda-data: transform-elastic-geoip: diff --git a/docker/dfe-docker.Dockerfile b/docker/dfe-docker.Dockerfile index 32d543f6..1c9ebb6c 100644 --- a/docker/dfe-docker.Dockerfile +++ b/docker/dfe-docker.Dockerfile @@ -6,11 +6,11 @@ # License: BUSL-1.1 # Copyright: (c) 2026 HYPERI PTY LIMITED -# Built with the repo root as context, so .dockerignore decides what the image carries +# Repo root is the context, so .dockerignore sets the contents ARG BASE_IMAGE=python:3.14.10-slim-trixie@sha256:9aaa474097d094040649f2162a627c0049b408f52138d980916788d80f04802f -# The Docker CLI and its compose plugin are static binaries, so the Alpine userland of this image never ships +# Docker CLI and compose are static, so no Alpine userland ships ARG DOCKER_CLI_IMAGE=docker:29.9.3-cli@sha256:2357639ad1ea7d0194134bf3292622fd4e562483966ed674e5d7c945b5f3b28d -# The oras and cosign scripts/stack.py and scripts/_registry.py already run, at the same digests +# Same oras and cosign digests as stack.py and _registry.py ARG ORAS_IMAGE=ghcr.io/oras-project/oras:v1.3.6@sha256:9cfb578be975b762fd898b9d004243d8003aeb807a3e15b2ed212cdd1326c9eb ARG COSIGN_IMAGE=gcr.io/projectsigstore/cosign:v3.1.4@sha256:0f06154836d2de34f887a5c273506d52ca43edaa1d9e53e6510fa78a48f69c2d @@ -20,9 +20,9 @@ FROM ${COSIGN_IMAGE} AS cosign FROM ${BASE_IMAGE} -# make runs the Makefile's own compose chain, git fetches the demo's generator, openssl mints the console certificate -# Unpinned (hadolint DL3008): the trixie archive drops a superseded package version, so a pinned one stops installing +# git for the generator, make for the chain, openssl for certs RUN apt-get update \ + # Unpinned: trixie drops superseded versions (hadolint DL3008) && apt-get install -y --no-install-recommends git make openssl \ && rm -rf /var/lib/apt/lists/* @@ -34,7 +34,7 @@ COPY --from=cosign /ko-app/cosign /usr/local/bin/cosign ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 -# The published demo's helpers read its source files with PyYAML, installed from wheels whose hashes the file pins +# PyYAML for the demo's helpers, from wheels with pinned hashes COPY docker/dfe-docker.requirements.txt /tmp/requirements.txt RUN pip install --no-cache-dir --root-user-action=ignore --only-binary=:all: --require-hashes -r /tmp/requirements.txt \ && rm /tmp/requirements.txt @@ -42,10 +42,10 @@ RUN pip install --no-cache-dir --root-user-action=ignore --only-binary=:all: --r WORKDIR /opt/dfe-docker COPY . . -# latest pins the newest published stack when init runs. A release build passes the stack it was cut against, so its version tag always installs that one +# `latest` resolves at init, a release build passes its stack ARG STACK=latest ARG VERSION=dev -# The reference init's printed commands name, so an install from a version tag keeps using that tag +# Image init's printed commands use, so tag installs keep it ARG IMAGE=ghcr.io/hyperi-io/dfe-docker ENV DFE_DOCKER_IMAGE=${IMAGE} \ DFE_DOCKER_STACK=${STACK} \ @@ -57,6 +57,6 @@ LABEL org.opencontainers.image.description="Sets up a folder that runs the DFE s org.opencontainers.image.title="dfe-docker" \ org.opencontainers.image.version="${VERSION}" -# No USER: the entrypoint starts as root only to take on the owner of the mounted folder, so what it writes there is that user's +# No USER: root first, to write files as the folder's owner ENTRYPOINT ["python3", "/opt/dfe-docker/scripts/installer.py"] CMD ["--help"] diff --git a/docs/architecture.md b/docs/architecture.md index bbbaa979..34afbe45 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -131,18 +131,18 @@ flowchart TB style ctlgrp fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 ``` -*Figure 1. L2 Services -- DFE Compose stack. Dotted paths are optional.* Up: top of dfe-docker. Down: [L4 Detail -- Loader topic choice](known-issues.md#a-_load-topic-changes-what-the-loader-reads), [L4 Detail -- Loader and schema checks](troubleshooting.md#events-are-accepted-but-nothing-lands), [L4 Detail -- One console origin](operating.md#console-origin-and-the-port-6000-listeners), [L4 Detail -- Two listeners on port 6000](operating.md#console-origin-and-the-port-6000-listeners). +*Figure 1. L2 Services - DFE Compose stack. Dotted paths are optional.* Up: top of dfe-docker. Down: [L4 Detail - Loader topic choice](known-issues.md#a-_load-topic-changes-what-the-loader-reads), [L4 Detail - Loader and schema checks](troubleshooting.md#events-are-accepted-but-nothing-lands), [L4 Detail - One console origin](operating.md#console-origin-and-the-port-6000-listeners), [L4 Detail - Two listeners on port 6000](operating.md#console-origin-and-the-port-6000-listeners). ## How data flows -1. Push clients send to dfe-receiver over HTTP, Vector, Beats, OTLP, HEC, syslog or flow. dfe-fetcher pulls from REST APIs, databases and files, and idles until a fetcher source exists. +1. Push clients send to dfe-receiver over HTTP, Vector, Beats, OTLP, HEC, syslog or flow. dfe-fetcher pulls from REST APIs, databases and files. It idles until a fetcher source exists. 2. On a `*-kafka` profile both produce to a `*_land` topic named after the event's `_source`. The name carries through character for character, so `filebeat-vector` gives `filebeat-vector_land` and a table that needs backticks in SQL. -3. A source with a transform gets its own instance. It reads `_land` and writes `_load`, and from then on the loader reads `_load` instead. Transforms run on Kafka profiles alone. +3. A source with a transform gets its own instance. It reads `_land` and writes `_load`. From then on the loader reads `_load` instead. Transforms run on Kafka profiles alone. 4. dfe-loader auto-discovers every topic ending in `_land` or `_load` and inserts into ClickHouse. dfe-archiver writes each source's landing topic to its archive, so archiving needs a Kafka profile. -5. On a `*-grpc` profile there is no broker, and the receiver dials `dfe-loader:6000` directly. -6. The DFE console sends every change through dfe-engine. The engine creates each table and topic, and renders each app's config. +5. On a `*-grpc` profile there is no broker and the receiver dials `dfe-loader:6000` directly. +6. The DFE console sends every change through dfe-engine. The engine creates each table and topic and renders each app's config. Create a source before you send it data. A loader that meets rows for a table it has no schema for buffers them, then dead-letters them. @@ -155,7 +155,7 @@ Create a source before you send it data. A loader that meets rows for a table it | Source topic | The `*_land` topic each source writes | [Broker](broker.md) | | Transforms | VRL, Vector or compiled Elastic pipelines, one instance per source | [vrl](https://github.com/hyperi-io/dfe-transform-vrl/blob/main/docs/architecture.md), [vector](https://github.com/hyperi-io/dfe-transform-vector/blob/main/docs/architecture.md), [elastic](https://github.com/hyperi-io/dfe-transform-elastic/blob/main/docs/architecture.md) | | Transformed topic | The `*_load` topic a transform writes | [Known issues](known-issues.md#a-_load-topic-changes-what-the-loader-reads) | -| dfe-loader | Writes ClickHouse, and hosts the gRPC Push listener | [dfe-loader](https://github.com/hyperi-io/dfe-loader/blob/main/docs/architecture.md) | +| dfe-loader | Writes ClickHouse and hosts the gRPC Push listener | [dfe-loader](https://github.com/hyperi-io/dfe-loader/blob/main/docs/architecture.md) | | ClickHouse | Every table, plus the stack's self-telemetry | [Secrets and storage](secrets-and-storage.md) | | dfe-archiver | Writes raw events to a filesystem, S3, MinIO, GCS or Azure Blob | [dfe-archiver](https://github.com/hyperi-io/dfe-archiver/blob/main/docs/architecture.md) | | Archive | The archiver's files | [Secrets and storage](secrets-and-storage.md) | @@ -170,7 +170,7 @@ Create a source before you send it data. A loader that meets rows for a table it |---|---|---| | Docker with Compose | always | one Linux host | | Registry | always | the registry the stack pins, at the digests the Kubernetes path runs | -| Broker | on `*-kafka` profiles | Redpanda, or Apache Kafka with `KAFKA_BACKEND=apache` ([broker](broker.md)) | +| Broker | on `*-kafka` profiles | Redpanda or Apache Kafka with `KAFKA_BACKEND=apache` ([broker](broker.md)) | | Object storage | for an archive past local disk | S3, MinIO, GCS, Azure Blob | | OIDC provider | with the `auth` profile | any, through oauth2-proxy ([auth profile](auth-profile.md)) | @@ -194,15 +194,15 @@ A profile declares its footprint through the `clickhouse`, `core`, `kafbat`, `hy Four limit tiers set CPU and memory ceilings ([operating](operating.md#resource-limits-four-tiers-ceilings-not-reservations)). -Kubernetes is the primary DFE path, and Compose is for small environments. Three things differ on purpose: +Kubernetes is the primary DFE path and Compose is for small environments. Three things differ on purpose: - A new source's apps start when you run `make apply`. Argo does that on Kubernetes. - Envoy's OIDC filters stay unconfigured, because Docker mode can't assume an issuer exists. The engine API still needs a login per route. -- Container logs reach the collector through Docker's `fluentd` log driver, and the host's CPUs are the one host metric collected ([observability](observability.md)). +- Container logs reach the collector through Docker's `fluentd` log driver and the host's CPUs are the one host metric collected ([observability](observability.md)). ## Design considerations -**Security.** The console and its Search authenticate. The ingest ports, Kafbat and the metrics ports ask for no login, so put the stack behind a VPN, a tunnel or a firewall, or arm the `auth` profile ([operating](operating.md)). +**Security.** The console and its Search authenticate. The receiver's HTTP listener takes only bearer tokens once an ingest credential names it. The other ingest ports, Kafbat and the metrics ports ask for no login, so arm the `auth` profile or put the stack behind a VPN, a tunnel or a firewall ([operating](operating.md)). **Reliability.** Every app waits for dfe-engine to report healthy, which means its tables and topics exist. Before an app starts, a one-shot runs its `config-check` against the exact config it will use, so a broken config stops `make up` with the app's own error and not a restart loop. @@ -214,7 +214,7 @@ Kubernetes is the primary DFE path, and Compose is for small environments. Three **Same images, same registry, same pins.** Compose is safe to deploy because it runs the exact digests Kubernetes runs, pinned from the same stack definition. -**dfe-engine renders each app's config.** Compose runs no chart, so the engine merges each instance's overlay over the committed `config//.yaml` and writes the result to the `dfe-app-config` volume. Deployment facts stay in the committed file, and nothing deployment-specific goes into the deploy repo. +**dfe-engine renders each app's config.** Compose runs no chart, so the engine merges each instance's overlay over the committed `config//.yaml` and writes the result to the `dfe-app-config` volume. Deployment facts stay in the committed file and nothing deployment-specific goes into the deploy repo. **dfe-engine alone owns the schema and topics.** The schemas ride inside the engine image, so a schema change is an engine release and there's no schema pin here. `scripts/tests/test_engine_only_schema_control.py` fails the build on any DDL or topic step in this repo. @@ -222,14 +222,14 @@ Kubernetes is the primary DFE path, and Compose is for small environments. Three ## Diagram map -- Figure 1. L2 Services -- DFE Compose stack, at the top of this page - - [L4 Detail -- Loader topic choice](known-issues.md#a-_load-topic-changes-what-the-loader-reads) - - [L4 Detail -- Loader and schema checks](troubleshooting.md#events-are-accepted-but-nothing-lands) - - [L4 Detail -- Topic and dead-letter checks](troubleshooting.md#events-are-accepted-but-nothing-lands) - - [L4 Detail -- One console origin](operating.md#console-origin-and-the-port-6000-listeners) - - [L4 Detail -- Two listeners on port 6000](operating.md#console-origin-and-the-port-6000-listeners) - - [L4 Detail -- Self-telemetry](observability.md) - - [L4 Detail -- Profile resolution](../CONTRIBUTING.md#profile-resolution), for contributors +- Figure 1. L2 Services - DFE Compose stack, at the top of this page + - [L4 Detail - Loader topic choice](known-issues.md#a-_load-topic-changes-what-the-loader-reads) + - [L4 Detail - Loader and schema checks](troubleshooting.md#events-are-accepted-but-nothing-lands) + - [L4 Detail - Topic and dead-letter checks](troubleshooting.md#events-are-accepted-but-nothing-lands) + - [L4 Detail - One console origin](operating.md#console-origin-and-the-port-6000-listeners) + - [L4 Detail - Two listeners on port 6000](operating.md#console-origin-and-the-port-6000-listeners) + - [L4 Detail - Self-telemetry](observability.md) + - [L4 Detail - Profile resolution](../CONTRIBUTING.md#profile-resolution), for contributors ## What's next diff --git a/docs/auth-profile.md b/docs/auth-profile.md index 419873e4..6b726f28 100644 --- a/docs/auth-profile.md +++ b/docs/auth-profile.md @@ -10,22 +10,13 @@ Copyright: (c) 2026 HYPERI PTY LIMITED # Gating the infra UIs with OIDC -Status: opt-in and off by default (`DFE_AUTH_ENABLED=false`). `make check-compose` checks its wiring, and no e2e test signs in through it. +Status: opt-in and off by default (`DFE_AUTH_ENABLED=false`). `make check-compose` checks its wiring and no e2e test signs in through it. -`DFE_AUTH_ENABLED=true` arms the `auth` profile, which puts an oauth2-proxy in -front of each infra UI. Reaching Kafbat, or dfe-hyperdx on `:8090` and `:8000`, then -needs an OIDC sign-in **and** membership of one of `DFE_OIDC_ALLOWED_GROUPS` -(`dfe-infra`, `dfe-admin` by default). The group check is the gate -- -authentication alone is not, which is the same rule the Kubernetes tier applies -at the Envoy edge. +`DFE_AUTH_ENABLED=true` arms the `auth` profile, which puts an oauth2-proxy in front of each infra UI. Reaching Kafbat or dfe-hyperdx on `:8090` and `:8000`, then needs an OIDC sign-in **and** membership of one of `DFE_OIDC_ALLOWED_GROUPS` (`dfe-infra`, `dfe-admin` by default). The group check is the gate - authentication alone is not, which is the same rule the Kubernetes tier applies at the Envoy edge. -The embed on `:8091` stays outside that gate, as the Kubernetes tier's dfe-hyperdx -route does. An OIDC redirect cannot complete inside the console's frame, and -dfe-hyperdx authenticates each caller itself from the console's `dfe_token`. So any -console user reaches dfe-hyperdx through the embed, group or not. +The embed on `:8091` stays outside that gate, as the Kubernetes tier's dfe-hyperdx route does. An OIDC redirect cannot complete inside the console's frame and dfe-hyperdx authenticates each caller itself from the console's `dfe_token`. So any console user reaches dfe-hyperdx through the embed, group or not. -Three proxies, one per origin, because oauth2-proxy serves a single listener and -dfe-hyperdx is one UI across two origins: +Three proxies, one per origin, because oauth2-proxy serves a single listener and dfe-hyperdx is one UI across two origins: | Host port | Proxy | Upstream | |---|---|---| @@ -33,25 +24,18 @@ dfe-hyperdx is one UI across two origins: | 8090 | oauth2-proxy-hyperdx | dfe-hyperdx app. Gates page access only, as dfe-hyperdx reads the engine token itself | | 8000 | oauth2-proxy-hyperdx-api | dfe-hyperdx API, an operator surface. `OAUTH2_PROXY_PASS_HOST_HEADER` forwards the request's Host header to dfe-hyperdx | -They take those ports over and the UIs stop publishing their own, so nothing -moves for anyone using them. All three share one cookie secret, and cookies are -not port-scoped, so a single sign-in covers the set. Register all three redirect -URIs (`:/oauth2/callback`) with the IdP, and make sure it emits a -groups claim or every sign-in is refused. +They take those ports over and the UIs stop publishing their own, so nothing moves for anyone using them. All three share one cookie secret and cookies are not port-scoped, so a single sign-in covers the set. Register all three redirect URIs (`:/oauth2/callback`) with the IdP and make sure it emits a groups claim or every sign-in is refused. -The infra kill switch still wins: `DFE_INFRA_UIS_EXTERNAL=false` unpublishes the -proxies as well as the UIs. +The infra kill switch still wins: `DFE_INFRA_UIS_EXTERNAL=false` unpublishes the proxies as well as the UIs. ## Three limits -It gates page access only. Neither UI learns from the proxy who signed in, because dfe-hyperdx identifies callers by the engine token, and neither gets roles or an audit trail from it. +It gates page access only. Neither UI learns from the proxy who signed in, because dfe-hyperdx identifies callers by the engine token and neither gets roles or an audit trail from it. An expired session on the API origin fails opaquely. The browser blocks a cross-origin XHR that gets a redirect to the IdP, so the app shows failed requests rather than a login prompt. Reload the app page to sign back in. Nothing else moves behind it. Ingest, ClickHouse, Kafka, the metrics ports and the product UIs are untouched. -It must stay possible to run every other profile without an -issuer. Arming it with a setting missing stops `make` and names the key rather -than starting a proxy that redirects nowhere. +It must stay possible to run every other profile without an issuer. Arming it with a setting missing stops `make` and names the key rather than starting a proxy that redirects nowhere. Every variable the profile reads is in [configuration.md](configuration.md#infra-ui-authentication-opt-in). The access model it sits in is in [operating.md](operating.md). diff --git a/docs/broker.md b/docs/broker.md index ddaf1456..b53d478e 100644 --- a/docs/broker.md +++ b/docs/broker.md @@ -12,13 +12,13 @@ Copyright: (c) 2026 HYPERI PTY LIMITED The `*-kafka` profiles run one broker, Redpanda by default or Apache Kafka with `KAFKA_BACKEND=apache`. The `*-grpc` profiles run none. Every broker variable is in [configuration.md](configuration.md#kafka). -The two backends are mutually exclusive and share the `kafka` network alias and the same host port, so nothing downstream knows which one runs. Every config addresses `kafka:9092`, and dfe-engine creates the bootstrap topic set over the Kafka protocol on both. The host listener is `19092` (`KAFKA_PLAINTEXT_HOST_PORT`) on `DFE_BIND_HOST`. +The two backends are mutually exclusive and share the `kafka` network alias and the same host port, so nothing downstream knows which one runs. Every config addresses `kafka:9092` and dfe-engine creates the bootstrap topic set over the Kafka protocol on both. The host listener is `19092` (`KAFKA_PLAINTEXT_HOST_PORT`) on `DFE_BIND_HOST`. Compose interpolates every service before profiles filter anything, so `REDPANDA_VERSION` must still be pinned for the file to resolve on the Apache path. No Redpanda image is pulled or run. A Redpanda pin in `.env` is not a Redpanda deployment, but if your licence position needs zero reference to the artefact, that pin is what is left. -Redpanda's core is source-available under the BSL. It is not OSS. Local development and CI sit inside its Additional Use Grant. A partner or edge deployment needs a licence check of its own, and that check is a question for a human, not an assumption. +Redpanda's core is source-available under the BSL. It is not OSS. Local development and CI sit inside its Additional Use Grant. A partner or edge deployment needs a licence check of its own and that check is a question for a human, not an assumption. -If it does not come back clean, `KAFKA_BACKEND=apache` selects Apache Kafka (Apache-2.0), and the Apache path pulls and runs no BSL-licensed artefact. +If it does not come back clean, `KAFKA_BACKEND=apache` selects Apache Kafka (Apache-2.0) and the Apache path pulls and runs no BSL-licensed artefact. ## What the config one-shot sets @@ -34,20 +34,21 @@ docker compose exec kafka-redpanda rpk cluster config set core_balancing_continu docker compose exec kafka-redpanda rpk cluster license info ``` -The broker and ClickHouse share the docker data disk, and every source and route adds a topic, so a cap on each partition cannot bound the total. The same one-shot sets two floors instead: +The broker and ClickHouse share the docker data disk and every source and route adds a topic, so a cap on each partition cannot bound the total. The same one-shot sets two floors instead: | Setting | Default | What it does | |---|---|---| -| `REDPANDA_MIN_FREE_BYTES` | 5 GiB | The broker refuses writes once free disk falls below it, so the disk never fills. Producers back off, and the receiver spools | +| `REDPANDA_MIN_FREE_BYTES` | 5 GiB | The broker refuses writes once free disk falls below it, so the disk never fills. Producers back off and the receiver spools | | `REDPANDA_RETENTION_BYTES` | 10 GiB | The most one partition keeps, beside the seven-day time retention | Both apply at the next `make up`, to a running stack and a fresh volume alike. The 5 GiB floor fits a laptop. A host with a large data disk raises it in `.env`, for example `REDPANDA_MIN_FREE_BYTES=21474836480` for 20 GiB. With less free disk than the floor, the broker reports itself unhealthy (`rpk cluster health` lists it under high disk usage) and dfe-engine does not start. -With a Redpanda Enterprise licence (beta, and untested against a paid licence), set `REDPANDA_LICENSE` to the licence string in `.env`. The one-shot then applies it, leaves Redpanda's default features on, and exits 1 if `rpk cluster license info` reports a violation. POST skips the trial-feature check on a licensed broker. + +With a Redpanda Enterprise licence (beta and untested against a paid licence), set `REDPANDA_LICENSE` to the licence string in `.env`. The one-shot then applies it, leaves Redpanda's default features on and exits 1 if `rpk cluster license info` reports a violation. POST skips the trial-feature check on a licensed broker. ## Redpanda ships in developer mode The default broker runs `--mode=dev-container --smp=1` with `REDPANDA_MEMORY=1G`: Redpanda's own developer mode, with relaxed fsync on one core. It is not a production configuration. -For a small-environment deploy, set `REDPANDA_MODE=production` with real `REDPANDA_SMP` and `REDPANDA_MEMORY` values, and raise `DFE_BROKER_MEMORY` **above** `REDPANDA_MEMORY`. Seastar reserves its `--memory` up front, so a container limit that does not exceed it gets the broker OOM-killed rather than backpressured. +For a small-environment deploy, set `REDPANDA_MODE=production` with real `REDPANDA_SMP` and `REDPANDA_MEMORY` values and raise `DFE_BROKER_MEMORY` **above** `REDPANDA_MEMORY`. Seastar reserves its `--memory` up front, so a container limit that does not exceed it gets the broker OOM-killed rather than backpressured. Redpanda subtracts the host's `vm.min_free_kbytes` from the container limit before it checks `--memory`, so the usable figure is lower than the limit you set and varies by host. A 92 GiB host reserving 1.84 GiB left the 2560M default 675M against a 1G request. `redpanda/start.sh` reads both at start and passes the smaller of `REDPANDA_MEMORY` and what the limit leaves, with a WARN naming the `DFE_BROKER_MEMORY` that restores the full amount. Under 512M it will not start and names the value that would fit. Check a host with `sysctl vm.min_free_kbytes`. diff --git a/docs/checks.md b/docs/checks.md index b9becad3..b475eaf1 100644 --- a/docs/checks.md +++ b/docs/checks.md @@ -10,7 +10,7 @@ Copyright: (c) 2026 HYPERI PTY LIMITED # Outage, source and static checks -The outage tests stop a service under load, the source test drives a new source through the console, and the static checks are what CI runs. The e2e suite they build on is in [testing.md](testing.md). +The outage tests stop a service under load, the source test drives a new source through the console and the static checks are what CI runs. The e2e suite they build on is in [testing.md](testing.md). ## Outage tests stop a service under load @@ -19,18 +19,11 @@ make test-resilience make test-resilience E2E_TESTS="clickhouse-outage loader-outage-grpc" ``` -Opt-in: `make test-e2e` never runs these, and rejects one listed in `E2E_TESTS`. -An `outage:` test sends steady load and, once its first records land, stops the -named service for `seconds`, starts it again in a `finally`, and keeps sending -for 30 s. It then asserts every other DFE service ran straight through (same -pid, no restart, no die or OOM event), the receiver answered every request (a -refusal such as 503 passes, silence fails), every accepted record landed, and on Kafka -every consumer group is back at zero lag. They stop services, so they never run -in `make post` either. +Opt-in: `make test-e2e` never runs these and rejects one listed in `E2E_TESTS`. An `outage:` test sends steady load and, once its first records land, stops the named service for `seconds`, starts it again in a `finally` and keeps sending for 30 s. It then asserts every other DFE service ran straight through (same pid, no restart, no die or OOM event), the receiver answered every request (a refusal such as 503 passes, silence fails), every accepted record landed and on Kafka every consumer group is back at zero lag. They stop services, so they never run in `make post` either. -An outage's `method` is `stop` by default, a graceful stop that lets the service drain what it holds. `method: kill` sends SIGKILL and a crash gets no drain, so a record the service held in memory survives only if its acknowledgement was held until the next hop had it. The `*-kill-*` tests kill the loader, the receiver and the VRL transform on Kafka, and the loader and receiver on direct gRPC. They pass only when every record answered 2xx lands, duplicates counted but allowed. Where the killed service is the receiver itself, a request in flight while it is down may go unanswered but silence outside that window still fails. +An outage's `method` is `stop` by default, a graceful stop that lets the service drain what it holds. `method: kill` sends SIGKILL and a crash gets no drain, so a record the service held in memory survives only if its acknowledgement was held until the next hop had it. The `*-kill-*` tests kill the loader, the receiver and the VRL transform on Kafka and the loader and receiver on direct gRPC. They pass only when every record answered 2xx lands, duplicates counted but allowed. Where the killed service is the receiver itself, a request in flight while it is down may go unanswered but silence outside that window still fails. -A `pause` freezes the next hop first, so the hop taken down dies holding what it took meanwhile. The `-sigterm-` tests stop the same hops gracefully. Only three kills have an `-acks-off` control: the loader on gRPC, the receiver on gRPC and the receiver on Kafka. The VRL transform, the archiver and the loader on Kafka have none. A control runs its kill with acknowledgements off on every hop to ClickHouse and sets `loss: required`, so it fails when it loses nothing: a control that cannot lose proves nothing. `loss: unreliable` is for a hop that holds too little to lose every run, and reports a clean run as non-discriminating instead of failing it. Other `outage:` keys read the receiver's disk spool (`spool`), refused records and where they go (`poison`, `dead_letters`) and the archive (`archive`). The keys are listed in `tests/e2e/e2e-tests.yaml`. +A `pause` freezes the next hop first, so the hop taken down dies holding what it took meanwhile. The `-sigterm-` tests stop the same hops gracefully. Only three kills have an `-acks-off` control: the loader on gRPC, the receiver on gRPC and the receiver on Kafka. The VRL transform, the archiver and the loader on Kafka have none. A control runs its kill with acknowledgements off on every hop to ClickHouse and sets `loss: required`, so it fails when it loses nothing: a control that cannot lose proves nothing. `loss: unreliable` is for a hop that holds too little to lose every run and reports a clean run as non-discriminating instead of failing it. Other `outage:` keys read the receiver's disk spool (`spool`), refused records and where they go (`poison`, `dead_letters`) and the archive (`archive`). The keys are listed in `tests/e2e/e2e-tests.yaml`. A kill that catches nothing in flight proves nothing about a crash, so a kill test fails on it. For the receiver or a gRPC hop that means no request out at the instant the container exited. For a Kafka consumer it means nothing its group had left uncommitted just before the kill. The kill tests raise the load with `workers` and `interval` (the default is 4 workers each waiting 0.5 s) so that a kill lands with records in flight. @@ -49,15 +42,13 @@ Above, sources come through the API. This one creates one in the console, attach Beyond a running stack it needs `DFE_INFRA_DIR` (the runner), `DFE_ENGINE_REPO` (the synthetic corpus and its reader), `DFE_TRANSFORM_VRL_REPO` for the filebeat case (the bundled pipeline), a dfe-transform-vector checkout for the vector case (`DFE_TRANSFORM_VECTOR_REPO`, else the one beside the engine repo) and an interpreter carrying Playwright (`DFE_ACCEPTANCE_PYTHON` where the system one is externally managed). The cloudwatch case polls a real AWS upstream and writes nothing to it: put the two `AWS_*` credentials in `env/fetcher.env` and `DFE_AWS_REGION` in `.env`. -Every step is a report row and a screenshot under `--shots-dir` (`.tmp/source`). -A step the console cannot do falls back to the engine API and says so, because -that is a finding about the console rather than about the pipeline. +Every step is a report row and a screenshot under `--shots-dir` (`.tmp/source`). A step the console cannot do falls back to the engine API and says so, because that is a finding about the console rather than about the pipeline. -`make proof` runs the filebeat and vector cases on every stack version. A failed run keeps its screenshots under `.tmp/proof-shots//` in the checkout, names that path in its `RESULT` line and keeps the newest three runs. Each transform checkout is cloned at an explicit ref override when one is set, else at the tag the stack pins, else at its default branch with a WARN, and only the pinned tag proves that pin. +`make proof` runs the filebeat and vector cases on every stack version. A failed run keeps its screenshots under `.tmp/proof-shots//` in the checkout, names that path in its `RESULT` line and keeps the newest three runs. Each transform checkout is cloned at an explicit ref override when one is set, else at the tag the stack pins, else at its default branch with a WARN and only the pinned tag proves that pin. ## Static checks run what CI runs -`make check` runs every static check CI runs, and CI calls each through the same make target. +`make check` runs every static check CI runs and CI calls each through the same make target. | Target | What it does | |---|---| @@ -68,21 +59,14 @@ that is a finding about the console rather than about the pipeline. | `make check-docs` | asserts every relative link across the README and every page under `docs/` resolves | | `make check-proxy` | `envoy --mode validate` on both proxy configs and their console TLS variants, on the Envoy the stack pins | | `make check-python` | `ruff check` and `ruff format --check` on `scripts/`, `ops/` and the demo packs | -| `make check-tests` | `pytest scripts/tests` -- unit tests over the helper scripts, including the theatre churn tiles' SQL run on the stack's ClickHouse image over seeded tracks | +| `make check-tests` | `pytest scripts/tests` - unit tests over the helper scripts, including the SQL of the theatre tiles `verify.json` lists in `zero_ok`, run on the stack's ClickHouse image over seeded tracks | | `make check-locks` | `uv lock --check` on each script that declares packages inline, so a pin moved without `uv lock --script