From 08825ffb6b8afb94dfc0fdccbf65f88c42862bd0 Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Thu, 15 Oct 2026 17:19:22 +1100 Subject: [PATCH 1/7] chore(dev): run local dev, integration and e2e on DocumentDB instead of MongoDB Local development and CI now use open-source DocumentDB (Postgres plus a Mongo wire-protocol gateway, MIT) in place of MongoDB 5.0 (SSPL, EOL). The API still talks to it through Mongoose, so this is the first step towards dropping MongoDB from what we ship, tested daily by every developer and CI run before users see it. HDX-5709. - One image, built from source in docker/documentdb and shared by the dev, integration and e2e stacks; it handles root-owned volumes, interrupted first boots and clean shutdown, and pins the upstream commit. - e2e database helpers use the Node driver instead of mongosh, which the image doesn't ship; CI builds the image with the GHA cache and waits on its healthcheck. - Test teardown retries DocumentDB's intermittent dropDatabase failure. - Dev data moves to .volumes/docdb_dev_; old MongoDB volumes are not migrated. --- .github/actions/e2e-setup/action.yml | 36 +++--- .github/workflows/main.yml | 4 +- AGENTS.md | 7 +- Makefile | 8 +- agent_docs/development.md | 19 ++- docker-compose.ci.yml | 9 +- docker-compose.dev.yml | 12 +- docker/documentdb/Dockerfile | 48 ++++++++ docker/documentdb/entrypoint.sh | 49 ++++++++ packages/api/.env.development | 2 +- packages/api/.env.e2e | 4 +- packages/api/.env.test | 2 +- packages/api/scripts/seed-alerts.ts | 4 +- packages/api/src/fixtures.ts | 9 +- packages/app/package.json | 1 + packages/app/tests/e2e/docker-compose.yml | 8 +- .../app/tests/e2e/features/dashboard.spec.ts | 78 +++++++----- .../app/tests/e2e/features/sources.spec.ts | 8 +- .../app/tests/e2e/global-setup-fullstack.ts | 116 +++++++++--------- packages/app/tests/e2e/utils/db-helpers.ts | 91 +++++++------- scripts/dev-env.sh | 2 +- scripts/dev-portal/index.html | 2 +- scripts/dev-portal/server.js | 6 +- scripts/test-e2e.sh | 71 ++--------- yarn.lock | 1 + 25 files changed, 340 insertions(+), 257 deletions(-) create mode 100644 docker/documentdb/Dockerfile create mode 100755 docker/documentdb/entrypoint.sh diff --git a/.github/actions/e2e-setup/action.yml b/.github/actions/e2e-setup/action.yml index f4eaaf1ac9..3bcbfe1626 100644 --- a/.github/actions/e2e-setup/action.yml +++ b/.github/actions/e2e-setup/action.yml @@ -3,7 +3,7 @@ description: | Shared setup for E2E test workflows: sets the HDX_E2E_* port env vars and common Playwright env vars, then runs Node.js setup, dependency install, build, Playwright browser install, and starts the E2E Docker Compose stack - (MongoDB + ClickHouse) with health checks. The caller is responsible for + (DocumentDB + ClickHouse) with health checks. The caller is responsible for `actions/checkout`, running Playwright, uploading artifacts, and tearing down the Docker stack. runs: @@ -22,7 +22,7 @@ runs: echo "HDX_E2E_MONGO_PORT=$MONGO_PORT" >> $GITHUB_ENV echo "HDX_E2E_APP_LOCAL_PORT=21200" >> $GITHUB_ENV echo "HDX_E2E_APP_PORT=21300" >> $GITHUB_ENV - echo "MONGO_URI=mongodb://localhost:$MONGO_PORT/hyperdx-e2e" >> $GITHUB_ENV + echo "MONGO_URI=mongodb://hyperdx:hyperdx@localhost:$MONGO_PORT/hyperdx-e2e?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256" >> $GITHUB_ENV echo "E2E_FULLSTACK=true" >> $GITHUB_ENV echo "E2E_UNIQUE_USER=true" >> $GITHUB_ENV echo "E2E_API_HEALTH_CHECK_MAX_RETRIES=60" >> $GITHUB_ENV @@ -66,12 +66,22 @@ runs: shell: bash run: cd packages/app && npx playwright install --with-deps chromium + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v4 + + # DocumentDB is built from source; the GHA cache keeps that to the first run. + - name: Build DocumentDB image + shell: bash + run: | + docker buildx create --use --driver=docker-container + docker buildx bake -f packages/app/tests/e2e/docker-compose.yml db --set *.cache-to="type=gha,mode=max,scope=documentdb" --set *.cache-from="type=gha,scope=documentdb" --load + - name: Start E2E Docker Compose shell: bash run: | # Pre-pull images with retries to handle transient Docker Hub timeouts for attempt in 1 2 3; do - if docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml pull; then + if docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml pull --ignore-buildable; then echo "Docker images pulled successfully on attempt $attempt" break fi @@ -82,20 +92,12 @@ runs: echo "Docker pull failed (attempt $attempt/3), retrying in 10s..." sleep 10 done - docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml up -d - echo "Waiting for MongoDB..." - for i in $(seq 1 30); do - if docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml exec -T db mongosh --quiet --eval "db.adminCommand({ping:1})" >/dev/null 2>&1; then - echo "MongoDB is ready" - break - fi - if [ "$i" -eq 30 ]; then - echo "MongoDB failed to become ready after 30 seconds" - exit 1 - fi - echo "Waiting for MongoDB... ($i/30)" - sleep 1 - done + # --wait blocks on the DocumentDB healthcheck (gateway accepting connections). + if ! docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml up -d --wait --wait-timeout 300; then + echo "E2E services failed to become healthy" + docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml logs db | tail -30 + exit 1 + fi echo "Waiting for ClickHouse..." for i in $(seq 1 60); do if curl -sf http://localhost:${HDX_E2E_CH_PORT}/ping >/dev/null 2>&1; then diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5b4c3c94a5..912d7fe028 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -125,7 +125,7 @@ jobs: - name: Build dependencies run: make ci-build # Integration tests boot a MockServer on fixed ports and share a single - # MongoDB/ClickHouse database, so they must stay serial (--runInBand) + # DocumentDB/ClickHouse database, so they must stay serial (--runInBand) # within a runner. We shard ACROSS runners instead: each shard is its own # VM with its own Docker stack, and jest's --shard splits the test files # so the shards run in parallel. @@ -143,7 +143,7 @@ jobs: # Actions streams stdout. - name: Run integration tests run: | - docker compose -f ./docker-compose.ci.yml up -d --quiet-pull + docker compose -f ./docker-compose.ci.yml up -d --quiet-pull --wait --wait-timeout 300 status=0 yarn workspace @hyperdx/common-utils ci:int --shard=${{ matrix.shard }}/${{ strategy.job-total }} || status=1 yarn workspace @hyperdx/api ci:int --shard=${{ matrix.shard }}/${{ strategy.job-total }} || status=1 diff --git a/AGENTS.md b/AGENTS.md index 9e5653ec04..3590fde95d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,7 +44,8 @@ yarn dev # Start full stack with worktree-isolated ports ``` The project uses **Yarn 4.13.0** workspaces. Docker Compose manages ClickHouse, -MongoDB, and the OTel Collector. +DocumentDB (the local metadata store, which the API reaches over the Mongo wire +protocol), and the OTel Collector. **This repo is multi-agent friendly.** `yarn dev`, `make dev-int`, and `make dev-e2e` all use slot-based port isolation so multiple worktrees can run @@ -461,7 +462,7 @@ default `/workspace` directory, the slot is **76**, so services are at: - **App**: http://localhost:30276 - **API**: http://localhost:30176 - **ClickHouse**: http://localhost:30576 -- **MongoDB**: localhost:30476 +- **DocumentDB**: localhost:30476 ### Key commands reference @@ -475,7 +476,7 @@ reference. Quick summary: ### First-time registration -When the dev stack starts fresh (empty MongoDB), the app shows a registration +When the dev stack starts fresh (empty DocumentDB), the app shows a registration page. Create any account to get started — no external auth provider is needed. --- diff --git a/Makefile b/Makefile index 91896878ea..02caba3ab8 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ install-tools: # API server : 30100 + slot # App (Next.js) : 30200 + slot # OpAMP : 30300 + slot -# MongoDB : 30400 + slot +# DocumentDB : 30400 + slot # ClickHouse HTTP : 30500 + slot # ClickHouse Native : 30600 + slot # OTel health : 30700 + slot @@ -158,7 +158,7 @@ dev-int: @echo "Using CI slot $(HDX_CI_SLOT) (project=$(HDX_CI_PROJECT) ch=$(HDX_CI_CH_PORT) mongo=$(HDX_CI_MONGO_PORT) api=$(HDX_CI_API_PORT))" @mkdir -p $(HDX_CI_LOGS_DIR) @bash scripts/ensure-dev-portal.sh - docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --build --wait --wait-timeout 300 bash -c 'set -o pipefail; npx nx run @hyperdx/api:dev:int $(FILE) 2>&1 | tee $(HDX_CI_LOGS_DIR)/api-int.log'; ret=$$?; \ docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down -v; \ $(call archive-int-logs); \ @@ -169,7 +169,7 @@ dev-int-common-utils: @echo "Using CI slot $(HDX_CI_SLOT) (project=$(HDX_CI_PROJECT) ch=$(HDX_CI_CH_PORT) mongo=$(HDX_CI_MONGO_PORT))" @mkdir -p $(HDX_CI_LOGS_DIR) @bash scripts/ensure-dev-portal.sh - docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --build --wait --wait-timeout 300 bash -c 'set -o pipefail; npx nx run @hyperdx/common-utils:dev:int $(FILE) 2>&1 | tee $(HDX_CI_LOGS_DIR)/common-utils-int.log'; ret=$$?; \ docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down -v; \ $(call archive-int-logs); \ @@ -178,7 +178,7 @@ dev-int-common-utils: .PHONY: ci-int ci-int: @mkdir -p $(HDX_CI_LOGS_DIR) - docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --quiet-pull + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --quiet-pull --wait --wait-timeout 300 bash -c 'set -o pipefail; npx nx run-many -t ci:int --parallel=false --output-style=stream 2>&1 | tee $(HDX_CI_LOGS_DIR)/ci-int.log'; ret=$$?; \ docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down -v; \ $(call archive-int-logs); \ diff --git a/agent_docs/development.md b/agent_docs/development.md index 305c0d9380..f3c281c4d2 100644 --- a/agent_docs/development.md +++ b/agent_docs/development.md @@ -30,7 +30,16 @@ yarn dev # or equivalently: make dev ## Environment Configuration - `.env.development`: Development environment variables -- Docker Compose manages ClickHouse, MongoDB, OTel Collector +- Docker Compose manages ClickHouse, DocumentDB, OTel Collector +- The metadata store is [DocumentDB](https://github.com/documentdb/documentdb) + (Postgres plus a Mongo wire-protocol gateway), built from + `docker/documentdb/Dockerfile`. The API still talks to it through Mongoose. + To browse it with MongoDB Compass or `mongosh`, connect to + `mongodb://hyperdx:hyperdx@localhost:/?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256`. + Everything works except Compass's Performance tab (the gateway doesn't + implement `serverStatus` or `top`). + Dev data lives in `.volumes/docdb_dev_`; data from the old MongoDB + volumes (`.volumes/db_dev_`) is not migrated, so the dev stack starts empty. - Hot reload enabled for all services in development ## Worktree Isolation (Multi-Agent / Multi-Developer) @@ -59,7 +68,7 @@ integration tests (14320-40098) and E2E tests (20320-21399). | API server | 30100 | 30100 - 30199 | `HYPERDX_API_PORT` | | App (Next.js) | 30200 | 30200 - 30299 | `HYPERDX_APP_PORT` | | OpAMP | 30300 | 30300 - 30399 | `HYPERDX_OPAMP_PORT` | -| MongoDB | 30400 | 30400 - 30499 | `HDX_DEV_MONGO_PORT` | +| DocumentDB | 30400 | 30400 - 30499 | `HDX_DEV_MONGO_PORT` | | ClickHouse HTTP | 30500 | 30500 - 30599 | `HDX_DEV_CH_HTTP_PORT` | | ClickHouse Native | 30600 | 30600 - 30699 | `HDX_DEV_CH_NATIVE_PORT` | | OTel health | 30700 | 30700 - 30799 | `HDX_DEV_OTEL_HEALTH_PORT` | @@ -173,7 +182,7 @@ Port mapping (base + slot): | Service | Default port (slot 0) | Variable | | --------------- | --------------------- | ----------------- | | ClickHouse HTTP | 18123 | HDX_CI_CH_PORT | -| MongoDB | 39999 | HDX_CI_MONGO_PORT | +| DocumentDB | 39999 | HDX_CI_MONGO_PORT | | API test server | 19000 | HDX_CI_API_PORT | | OpAMP | 14320 | HDX_CI_OPAMP_PORT | @@ -181,7 +190,7 @@ Port mapping (base + slot): - Uses separate Docker Compose configuration (`docker-compose.ci.yml`) - Isolated test environment with unique `-p int-` project name -- Includes all necessary services (ClickHouse, MongoDB, OTel Collector) +- Includes all necessary services (ClickHouse, DocumentDB, OTel Collector) - Tests run against real database instances for accurate integration testing ### E2E Testing @@ -198,7 +207,7 @@ E2E port mapping (base + slot): | ClickHouse HTTP | 20500 | 20500 - 20599 | `HDX_E2E_CH_PORT` | | ClickHouse Native | 20600 | 20600 - 20699 | `HDX_E2E_CH_NATIVE_PORT` | | API server | 21000 | 21000 - 21099 | `HDX_E2E_API_PORT` | -| MongoDB | 21100 | 21100 - 21199 | `HDX_E2E_MONGO_PORT` | +| DocumentDB | 21100 | 21100 - 21199 | `HDX_E2E_MONGO_PORT` | | App (local) | 21200 | 21200 - 21299 | `HDX_E2E_APP_LOCAL_PORT` | | App (fullstack) | 21300 | 21300 - 21399 | `HDX_E2E_APP_PORT` | diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index bbc07b9799..f425f5d98a 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -35,10 +35,13 @@ services: networks: - internal db: - image: mongo:5.0.32-focal - command: --port 29999 + image: hyperdx/documentdb:local + build: ./docker/documentdb + environment: + DOCDB_USER: hyperdx + DOCDB_PASSWORD: hyperdx ports: - - '${HDX_CI_MONGO_PORT:-39999}:29999' + - '${HDX_CI_MONGO_PORT:-39999}:10260' networks: - internal networks: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 9c3c0491ec..c93cc67a0b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -7,13 +7,17 @@ services: db: labels: <<: *hdx-labels - hdx.dev.service: mongodb + hdx.dev.service: documentdb hdx.dev.port: '${HDX_DEV_MONGO_PORT:-27017}' - image: mongo:5.0.32-focal + image: hyperdx/documentdb:local + build: ./docker/documentdb + environment: + DOCDB_USER: hyperdx + DOCDB_PASSWORD: hyperdx volumes: - - .volumes/db_dev_${HDX_DEV_SLOT:-0}:/data/db + - .volumes/docdb_dev_${HDX_DEV_SLOT:-0}:/data/docdb ports: - - '${HDX_DEV_MONGO_PORT:-27017}:27017' + - '${HDX_DEV_MONGO_PORT:-27017}:10260' networks: - internal otel-collector: diff --git a/docker/documentdb/Dockerfile b/docker/documentdb/Dockerfile new file mode 100644 index 0000000000..a17c9c9b0b --- /dev/null +++ b/docker/documentdb/Dockerfile @@ -0,0 +1,48 @@ +# DocumentDB (MIT): Postgres extensions plus a Mongo wire-protocol gateway, used +# in place of MongoDB (SSPL). Built from source because upstream ships glibc +# packages only; Alpine 3.24 matches the all-in-one base and its PostGIS targets PG18. +ARG DOCUMENTDB_VERSION=v1.0-RC2 +# Tags can move; the build fails unless the tag still resolves to this commit. +ARG DOCUMENTDB_COMMIT=b0cb0dae85e8746f4e0b79c847894179b2f80d64 + +FROM alpine:3.24 AS builder +ARG DOCUMENTDB_VERSION +ARG DOCUMENTDB_COMMIT +RUN apk add --no-cache build-base cmake git curl bash tar pkgconf perl linux-headers python3 \ + icu-dev krb5-dev openssl-dev rust cargo \ + postgresql18 postgresql18-dev postgresql18-contrib postgresql-pg_cron postgresql-pgvector postgis +ENV PG_CONFIG=/usr/libexec/postgresql18/pg_config INSTALL_DEPENDENCIES_ROOT=/deps +RUN git clone --depth 1 --branch ${DOCUMENTDB_VERSION} https://github.com/documentdb/documentdb.git /src \ + && test "$(git -C /src rev-parse HEAD)" = "${DOCUMENTDB_COMMIT}" \ + && mkdir -p /deps +# musl exports _init/_fini from every .so, which the duplicate-export check counts. +RUN sed -i "s#grep -vxE '_PG_init|Pg_magic_func'#grep -vxE '_PG_init|Pg_magic_func|_init|_fini'#g" \ + /src/pg_documentdb_extended_rum/validate_rum_core_exports.sh +RUN cd /src/scripts && MAKE_PROGRAM=cmake bash install_setup_libbson.sh \ + && bash install_setup_pcre2.sh && bash install_setup_intel_decimal_math_lib.sh +# Serial make: with -j the RUM export check can run before the library is linked. +RUN cd /src && make install-documentdb with_llvm=no PG_CONFIG=$PG_CONFIG +RUN cd /src/pg_documentdb_gw && cargo build --release -p documentdb_gateway \ + && install -s target/release/documentdb_gateway /usr/bin/documentdb-gateway +# Keep only what Postgres and DocumentDB link against: no GDAL, JIT or PostGIS raster/topology. +RUN set -e; PG=/usr/libexec/postgresql18; LIB=/usr/lib/postgresql18; SH=/usr/share/postgresql18; \ + rm -f $LIB/llvmjit*.so $LIB/postgis_raster-3.so $LIB/postgis_sfcgal-3.so $LIB/postgis_topology-3.so $LIB/address_standardizer*.so; rm -rf $LIB/bitcode; \ + rm -f $SH/extension/postgis_raster* $SH/extension/postgis_sfcgal* $SH/extension/postgis_topology* $SH/extension/postgis_tiger* $SH/extension/address_standardizer*; rm -rf $SH/contrib/postgis*; \ + for f in $PG/postgres $PG/pg_ctl $PG/initdb $PG/psql /usr/bin/documentdb-gateway $LIB/*.so; do echo $f; ldd $f 2>/dev/null | awk '/=> \//{print $3}'; done | sort -u > /tmp/files; \ + mkdir /out; tar -ch -T /tmp/files -f - | tar -x -C /out; \ + tar -ch $PG/postgres $PG/pg_ctl $PG/initdb $PG/psql $LIB $SH /usr/share/proj /usr/share/icu -f - | tar -x -C /out; \ + install -D -m 644 /src/pg_documentdb_gw/SetupConfiguration.json /out/etc/documentdb/gateway/SetupConfiguration.json + +FROM alpine:3.24 AS runtime +RUN apk add --no-cache bash openssl su-exec \ + && addgroup -S postgres && adduser -S -G postgres -h /var/lib/postgresql postgres \ + && install -d -o postgres -g postgres /data/docdb +COPY --from=builder /out/ / +COPY --chmod=755 entrypoint.sh /usr/local/bin/documentdb-entrypoint.sh +ENV PATH=/usr/libexec/postgresql18:$PATH PGDATA=/data/docdb/pgdata +EXPOSE 10260 +# The gateway only listens once Postgres is initialised and the user exists; the +# psql probe catches Postgres dying under a gateway that keeps its port open. +HEALTHCHECK --interval=2s --timeout=3s --start-period=60s --retries=30 \ + CMD nc -z 127.0.0.1 10260 && psql -h /tmp -U postgres -d postgres -qtAX -c 'SELECT 1' >/dev/null +ENTRYPOINT ["bash", "/usr/local/bin/documentdb-entrypoint.sh"] diff --git a/docker/documentdb/entrypoint.sh b/docker/documentdb/entrypoint.sh new file mode 100755 index 0000000000..388c264294 --- /dev/null +++ b/docker/documentdb/entrypoint.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Starts Postgres with the DocumentDB extensions, then the gateway on :10260. +# Run as root: fixes data dir ownership, then re-runs itself as postgres. +set -euo pipefail +: "${DOCDB_USER:?DOCDB_USER is required}" "${DOCDB_PASSWORD:?DOCDB_PASSWORD is required}" + +if [ "$(id -u)" = 0 ]; then + # Volumes may be root-owned or hold lost+found, and initdb needs an empty dir it owns. + install -d -o postgres -g postgres -m 700 "$PGDATA" + exec su-exec postgres "$0" "$@" +fi + +# The marker is written last, so a first boot killed part-way is redone, not left broken. +if [ ! -f "$PGDATA/.documentdb-initialised" ]; then + find "$PGDATA" -mindepth 1 -delete + initdb -D "$PGDATA" -U postgres --auth=trust >/dev/null + cat >> "$PGDATA/postgresql.conf" </dev/null + psql -h /tmp -U postgres -d postgres -qX -v ON_ERROR_STOP=1 -c "CREATE EXTENSION documentdb CASCADE; CREATE EXTENSION documentdb_extended_rum;" + doc=$(printf '{"createUser":"%s","pwd":"%s","roles":[{"role":"readWriteAnyDatabase","db":"admin"},{"role":"clusterAdmin","db":"admin"}]}' "$DOCDB_USER" "$DOCDB_PASSWORD") + psql -h /tmp -U postgres -d postgres -qX -v ON_ERROR_STOP=1 -v doc="$doc" <<< "SELECT documentdb_api.create_user(:'doc');" >/dev/null + touch "$PGDATA/.documentdb-initialised" +else + pg_ctl -D "$PGDATA" -l "$PGDATA/postgresql.log" -w start >/dev/null +fi + +echo "postgresql://postgres@localhost:5432/postgres" > /tmp/pg-url +mkdir -p /tmp/tls +export DOCUMENTDB_PG_URL_FILE=/tmp/pg-url DOCUMENTDB_TLS_STATE_DIR=/tmp/tls + +documentdb-gateway run --config /etc/documentdb/gateway/SetupConfiguration.json & +gateway=$! +# Stop Postgres cleanly on docker stop so persistent volumes skip crash recovery. +trap 'kill "$gateway" 2>/dev/null; pg_ctl -D "$PGDATA" -m fast stop >/dev/null; exit 0' TERM INT +# Keep the gateway's exit code so a crash shows as a failed container, not a clean exit. +rc=0 +wait "$gateway" || rc=$? +pg_ctl -D "$PGDATA" -m fast stop >/dev/null || true +exit "$rc" diff --git a/packages/api/.env.development b/packages/api/.env.development index d4995195fd..9da78e573a 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -6,7 +6,7 @@ FRONTEND_URL="http://localhost:${HYPERDX_APP_PORT}" HDX_NODE_ADVANCED_NETWORK_CAPTURE=1 HDX_NODE_BETA_MODE=1 HDX_NODE_CONSOLE_CAPTURE=1 -MONGO_URI="mongodb://localhost:${HDX_DEV_MONGO_PORT}/hyperdx" +MONGO_URI="mongodb://hyperdx:hyperdx@localhost:${HDX_DEV_MONGO_PORT}/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256" NODE_ENV=development OTEL_SERVICE_NAME="hdx-oss-dev-api" OTEL_RESOURCE_ATTRIBUTES="service.version=dev" diff --git a/packages/api/.env.e2e b/packages/api/.env.e2e index 9db633c4d6..8b93a929c7 100644 --- a/packages/api/.env.e2e +++ b/packages/api/.env.e2e @@ -10,8 +10,8 @@ CLICKHOUSE_PASSWORD= CLICKHOUSE_USER=default RUN_SCHEDULED_TASKS_EXTERNALLY=true FRONTEND_URL=http://localhost:${HDX_E2E_APP_PORT:-21300} -# MongoDB connection string -MONGO_URI=mongodb://localhost:${HDX_E2E_MONGO_PORT:-21100}/hyperdx-e2e +# DocumentDB (Mongo wire protocol) connection string +MONGO_URI=mongodb://hyperdx:hyperdx@localhost:${HDX_E2E_MONGO_PORT:-21100}/hyperdx-e2e?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256 NODE_ENV=test PORT=${HDX_E2E_API_PORT:-21000} OPAMP_PORT=${HDX_E2E_OPAMP_PORT:-20320} diff --git a/packages/api/.env.test b/packages/api/.env.test index 45ee1e9b43..ddba188320 100644 --- a/packages/api/.env.test +++ b/packages/api/.env.test @@ -7,7 +7,7 @@ CLICKHOUSE_PASSWORD=api CLICKHOUSE_USER=api RUN_SCHEDULED_TASKS_EXTERNALLY=true FRONTEND_URL=http://app:8080 -MONGO_URI=mongodb://localhost:${HDX_CI_MONGO_PORT:-39999}/hyperdx-test +MONGO_URI=mongodb://hyperdx:hyperdx@localhost:${HDX_CI_MONGO_PORT:-39999}/hyperdx-test?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256 NODE_ENV=test PORT=${HDX_CI_API_PORT:-19000} OPAMP_PORT=${HDX_CI_OPAMP_PORT:-14320} diff --git a/packages/api/scripts/seed-alerts.ts b/packages/api/scripts/seed-alerts.ts index 28220ab4ea..fa68ca4e18 100644 --- a/packages/api/scripts/seed-alerts.ts +++ b/packages/api/scripts/seed-alerts.ts @@ -120,7 +120,7 @@ function parseArgs(argv: string[]): Args { } /** - * Mongo URI of this worktree's dev stack. `yarn seed:alerts` runs through + * URI of this worktree's dev DocumentDB. `yarn seed:alerts` runs through * scripts/with-slots.sh, which sets HDX_DEV_MONGO_PORT. */ function devSlotMongoUri(): string { @@ -130,7 +130,7 @@ function devSlotMongoUri(): string { 'HDX_DEV_MONGO_PORT is not set. Run this with `yarn seed:alerts`, or pass --mongo-uri.', ); } - return `mongodb://localhost:${port}/hyperdx`; + return `mongodb://hyperdx:hyperdx@localhost:${port}/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256`; } function randomInt(minInclusive: number, maxInclusive: number): number { diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts index 47d2edafd8..58a8678d53 100644 --- a/packages/api/src/fixtures.ts +++ b/packages/api/src/fixtures.ts @@ -22,6 +22,7 @@ import { AlertInterval, AlertSource } from '@/models/alert'; import Server from '@/server'; import logger from '@/utils/logger'; import { MetricModel } from '@/utils/logParser'; +import { withRetry } from '@/utils/retry'; import { ExternalDashboardTile } from './utils/zod'; @@ -141,7 +142,13 @@ export const closeDB = async () => { if (!config.IS_CI) { throw new Error('ONLY execute this in CI env 😈 !!!'); } - await mongooseConnection.dropDatabase(); + // DocumentDB intermittently fails dropDatabase while background index builds + // run (WriteConflict 112, or InternalError 1); a retry clears it. + await withRetry(() => mongooseConnection.dropDatabase(), { + maxRetries: 5, + initialDelayMs: 200, + retryOnlyOnStatus: [1, 112], + }); await mongoose.disconnect(); }; diff --git a/packages/app/package.json b/packages/app/package.json index c6fc18f131..334ab03e62 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -139,6 +139,7 @@ "identity-obj-proxy": "^3.0.0", "jest": "^30.2.0", "jest-environment-jsdom": "^30.2.0", + "mongodb": "^6.15.0", "msw": "^2.3.0", "msw-storybook-addon": "^2.0.2", "postcss": "^8.5.23", diff --git a/packages/app/tests/e2e/docker-compose.yml b/packages/app/tests/e2e/docker-compose.yml index b8152ccc6c..a7a2c3a2b1 100644 --- a/packages/app/tests/e2e/docker-compose.yml +++ b/packages/app/tests/e2e/docker-compose.yml @@ -1,9 +1,13 @@ name: hdx-e2e services: db: - image: mongo:5.0.32-focal + image: hyperdx/documentdb:local + build: ../../../../docker/documentdb + environment: + DOCDB_USER: hyperdx + DOCDB_PASSWORD: hyperdx ports: - - ${HDX_E2E_MONGO_PORT:-21100}:27017 + - ${HDX_E2E_MONGO_PORT:-21100}:10260 networks: - internal deploy: diff --git a/packages/app/tests/e2e/features/dashboard.spec.ts b/packages/app/tests/e2e/features/dashboard.spec.ts index 924006f1a3..59f89e6c73 100644 --- a/packages/app/tests/e2e/features/dashboard.spec.ts +++ b/packages/app/tests/e2e/features/dashboard.spec.ts @@ -12,7 +12,7 @@ import { DEFAULT_METRICS_SOURCE_NAME, DEFAULT_TRACES_SOURCE_NAME, } from '../utils/constants'; -import { runMongoshScript } from '../utils/db-helpers'; +import { ObjectId, withE2eDb } from '../utils/db-helpers'; import { getSqlEditor } from '../utils/locators'; test.describe('Dashboard', { tag: ['@dashboard'] }, () => { @@ -1158,17 +1158,16 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { // Written straight to Mongo: the API and the form both refuse this // state now, so the only way to hold it is to predate that rule. const dashboardId = dashboardPage.getCurrentDashboardId(); - const out = runMongoshScript( - [ - "use('hyperdx-e2e');", - 'print(JSON.stringify(db.dashboards.updateOne(', - ` { _id: ObjectId(${JSON.stringify(dashboardId)}) },`, - ' { $set: { "filters.0.isVariableEnabled": false } }', - ')));', - ].join('\n'), + const out = await withE2eDb(db => + db + .collection('dashboards') + .updateOne( + { _id: new ObjectId(dashboardId) }, + { $set: { 'filters.0.isVariableEnabled': false } }, + ), ); - expect(out).toContain('"matchedCount":1'); - expect(out).toContain('"modifiedCount":1'); + expect(out.matchedCount).toBe(1); + expect(out.modifiedCount).toBe(1); await dashboardPage.gotoDashboard(dashboardId); @@ -3351,19 +3350,18 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { await test.step('Mark it provisioned', async () => { // ProvisionDashboardsTask is the only thing that sets this in normal // operation, so write it directly rather than running that job. - const out = runMongoshScript( - [ - "use('hyperdx-e2e');", - 'print(JSON.stringify(db.dashboards.updateOne(', - ` { _id: ObjectId(${JSON.stringify(dashboardId)}) },`, - ' { $set: { provisioned: true } }', - ')));', - ].join('\n'), + const out = await withE2eDb(db => + db + .collection('dashboards') + .updateOne( + { _id: new ObjectId(dashboardId) }, + { $set: { provisioned: true } }, + ), ); // A zero-match write would make the assertion below pass for a reason // that has nothing to do with the gate. - expect(out).toContain('"matchedCount":1'); - expect(out).toContain('"modifiedCount":1'); + expect(out.matchedCount).toBe(1); + expect(out.modifiedCount).toBe(1); }); await test.step('Verify it is withheld after reload', async () => { @@ -3407,19 +3405,33 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { // Written directly rather than built in the chart editor: the editor // flow for a PromQL tile is long and this test is about the gate, not // the editor. - const out = runMongoshScript( - [ - "use('hyperdx-e2e');", - 'print(JSON.stringify(db.dashboards.updateOne(', - ` { _id: ObjectId(${JSON.stringify(dashboardId)}) },`, - " { $set: { tiles: [{ id: 'promql-1', x: 0, y: 0, w: 4, h: 2,", - " config: { configType: 'promql', promqlExpression: 'up',", - " connection: 'c1', source: 's1', displayType: 'line' } }] } }", - ')));', - ].join('\n'), + const out = await withE2eDb(db => + db.collection('dashboards').updateOne( + { _id: new ObjectId(dashboardId) }, + { + $set: { + tiles: [ + { + id: 'promql-1', + x: 0, + y: 0, + w: 4, + h: 2, + config: { + configType: 'promql', + promqlExpression: 'up', + connection: 'c1', + source: 's1', + displayType: 'line', + }, + }, + ], + }, + }, + ), ); - expect(out).toContain('"matchedCount":1'); - expect(out).toContain('"modifiedCount":1'); + expect(out.matchedCount).toBe(1); + expect(out.modifiedCount).toBe(1); }); await test.step('Verify it is withheld after reload', async () => { diff --git a/packages/app/tests/e2e/features/sources.spec.ts b/packages/app/tests/e2e/features/sources.spec.ts index 0ee087888a..79cac5b20d 100644 --- a/packages/app/tests/e2e/features/sources.spec.ts +++ b/packages/app/tests/e2e/features/sources.spec.ts @@ -341,7 +341,7 @@ test.describe('Sources Functionality', { tag: ['@sources'] }, () => { 'should show series Table field when isMetricsSeriesTableEnabled is on', { tag: ['@full-stack'] }, async () => { - setTeamFlag('isMetricsSeriesTableEnabled', true); + await setTeamFlag('isMetricsSeriesTableEnabled', true); try { await searchPage.goto(); await searchPage.sourceActionsMenu.click(); @@ -358,7 +358,7 @@ test.describe('Sources Functionality', { tag: ['@sources'] }, () => { await searchPage.page.keyboard.press('Escape'); } finally { - setTeamFlag('isMetricsSeriesTableEnabled', false); + await setTeamFlag('isMetricsSeriesTableEnabled', false); } }, ); @@ -367,7 +367,7 @@ test.describe('Sources Functionality', { tag: ['@sources'] }, () => { 'should warn when the configured series table does not match the series table schema', { tag: ['@full-stack'] }, async ({ page }) => { - setTeamFlag('isMetricsSeriesTableEnabled', true); + await setTeamFlag('isMetricsSeriesTableEnabled', true); const API_URL = getApiUrl(); const metricSources = await getSources(page, 'metric'); @@ -415,7 +415,7 @@ test.describe('Sources Functionality', { tag: ['@sources'] }, () => { ).toBeVisible({ timeout: 20000 }); } finally { await page.request.delete(`${API_URL}/sources/${createdSourceId}`); - setTeamFlag('isMetricsSeriesTableEnabled', false); + await setTeamFlag('isMetricsSeriesTableEnabled', false); } }, ); diff --git a/packages/app/tests/e2e/global-setup-fullstack.ts b/packages/app/tests/e2e/global-setup-fullstack.ts index 9c196b6900..819d097789 100644 --- a/packages/app/tests/e2e/global-setup-fullstack.ts +++ b/packages/app/tests/e2e/global-setup-fullstack.ts @@ -2,13 +2,13 @@ * Global setup for full-stack E2E tests * * This setup: - * 1. Clears MongoDB database to ensure clean state + * 1. Clears the database to ensure clean state * 2. Creates a test user and team * 3. Applies DEFAULT_SOURCES from .env.e2e * 4. Saves authentication state for tests * * Full-stack mode uses: - * - MongoDB (local) for authentication, teams, users, persistence + * - DocumentDB (local) for authentication, teams, users, persistence * - API server (local) for backend logic * - Demo ClickHouse (remote) for telemetry data (logs, traces, metrics, K8s) */ @@ -17,7 +17,7 @@ import fs from 'fs'; import path from 'path'; import { chromium, FullConfig } from '@playwright/test'; -import { runMongoshScript } from './utils/db-helpers'; +import { dropE2eDatabase, ObjectId, withE2eDb } from './utils/db-helpers'; import { seedClickHouse } from './seed-clickhouse'; // Configuration constants @@ -63,14 +63,14 @@ export const SEEDED_ERROR_ALERT = { }; /** - * Clears the MongoDB database to ensure a clean slate for tests + * Clears the database to ensure a clean slate for tests */ -function clearDatabase() { - console.log('Clearing MongoDB database for fresh test run...'); +async function clearDatabase() { + console.log('Clearing the e2e database for a fresh test run...'); try { - runMongoshScript("use('hyperdx-e2e'); db.dropDatabase();"); - console.log(' ✓ Database cleared successfully (via Docker)'); + await dropE2eDatabase(); + console.log(' ✓ Database cleared successfully'); } catch (error) { console.warn(' ⚠ Warning: Could not clear database'); console.warn(` ${error instanceof Error ? error.message : String(error)}`); @@ -85,7 +85,7 @@ function clearDatabase() { async function globalSetup(_config: FullConfig) { console.log('Setting up full-stack E2E environment'); - console.log(' MongoDB: local (auth, teams, persistence)'); + console.log(' DocumentDB: local (auth, teams, persistence)'); console.log(' ClickHouse: local instance (telemetry data)'); // Set timezone @@ -138,8 +138,8 @@ async function globalSetup(_config: FullConfig) { ); } - // Clear MongoDB database to ensure DEFAULT_SOURCES is applied - clearDatabase(); + // Clear the database to ensure DEFAULT_SOURCES is applied + await clearDatabase(); // Create test user and save auth state console.log('Creating test user and logging in'); @@ -302,7 +302,7 @@ async function globalSetup(_config: FullConfig) { * Seeds an alert with a recorded execution error. The alert is created via the * API (so all referenced documents — saved search, webhook — exist and the * alerts list endpoint populates correctly), then the `errors` array is - * patched in directly via mongosh since it's only ever set by the check-alerts + * patched in directly via the driver since it's only ever set by the check-alerts * background job in normal operation. */ async function seedAlertWithErrors( @@ -379,7 +379,7 @@ async function seedAlertWithErrors( const alert = (await alertRes.json()).data; const alertId: string = alert._id ?? alert.id; - // 4) Patch the `executionErrors` array directly via mongosh. The + // 4) Patch the `executionErrors` array directly in the database. The // check-alerts job is the only code that writes this field in normal // operation, so we write it here to avoid having to run that job during // setup. @@ -390,51 +390,53 @@ async function seedAlertWithErrors( const errorWindowStart = Math.floor(Date.now() / windowMs) * windowMs; const okWindowStart = errorWindowStart - windowMs; - const patchScript = ` -use('hyperdx-e2e'); -db.alerts.updateOne( - { _id: ObjectId(${JSON.stringify(alertId)}) }, - { - $set: { - executionErrors: [ - { - timestamp: new Date(), - type: ${JSON.stringify(SEEDED_ERROR_ALERT.errorType)}, - message: ${JSON.stringify(SEEDED_ERROR_ALERT.errorMessage)} - } - ], - state: 'OK' - } - } -); -db.alerthistories.deleteMany({ alert: ObjectId(${JSON.stringify(alertId)}) }); -db.alerthistories.insertMany([ - { - alert: ObjectId(${JSON.stringify(alertId)}), - createdAt: new Date(${okWindowStart}), - state: 'OK', - counts: 0, - lastValues: [{ startTime: new Date(${okWindowStart - windowMs}), count: 0 }] - }, - { - alert: ObjectId(${JSON.stringify(alertId)}), - createdAt: new Date(${errorWindowStart}), - state: 'ERROR', - counts: 0, - lastValues: [], - errors: [ - { - timestamp: new Date(), - type: ${JSON.stringify(SEEDED_ERROR_ALERT.historyErrorType)}, - message: ${JSON.stringify(SEEDED_ERROR_ALERT.historyErrorMessage)} - } - ] - } -]); -`; - try { - runMongoshScript(patchScript); + const alertObjectId = new ObjectId(alertId); + await withE2eDb(async db => { + await db.collection('alerts').updateOne( + { _id: alertObjectId }, + { + $set: { + executionErrors: [ + { + timestamp: new Date(), + type: SEEDED_ERROR_ALERT.errorType, + message: SEEDED_ERROR_ALERT.errorMessage, + }, + ], + state: 'OK', + }, + }, + ); + await db + .collection('alerthistories') + .deleteMany({ alert: alertObjectId }); + await db.collection('alerthistories').insertMany([ + { + alert: alertObjectId, + createdAt: new Date(okWindowStart), + state: 'OK', + counts: 0, + lastValues: [ + { startTime: new Date(okWindowStart - windowMs), count: 0 }, + ], + }, + { + alert: alertObjectId, + createdAt: new Date(errorWindowStart), + state: 'ERROR', + counts: 0, + lastValues: [], + errors: [ + { + timestamp: new Date(), + type: SEEDED_ERROR_ALERT.historyErrorType, + message: SEEDED_ERROR_ALERT.historyErrorMessage, + }, + ], + }, + ]); + }); console.log( ` ✓ Seeded alert "${alert.name}" (${alertId}) with a ${SEEDED_ERROR_ALERT.errorType}`, ); diff --git a/packages/app/tests/e2e/utils/db-helpers.ts b/packages/app/tests/e2e/utils/db-helpers.ts index 3d30f61307..947f4a4974 100644 --- a/packages/app/tests/e2e/utils/db-helpers.ts +++ b/packages/app/tests/e2e/utils/db-helpers.ts @@ -1,55 +1,44 @@ /** - * Direct MongoDB and ClickHouse access helpers for full-stack E2E tests. Only usable in - * full-stack mode (real Mongo via docker-compose) — there is no database in + * Direct metadata-store and ClickHouse access helpers for full-stack E2E tests. Only + * usable in full-stack mode (DocumentDB via docker-compose); there is no database in * local mode, so callers must gate on `{ tag: ['@full-stack'] }`. */ -import { execFileSync } from 'child_process'; -import fs from 'fs'; -import path from 'path'; +import { type Db, MongoClient } from 'mongodb'; + +export { ObjectId } from 'mongodb'; + +const E2E_DB_NAME = 'hyperdx-e2e'; + +// Built from the e2e slot port rather than MONGO_URI, so a MONGO_URI exported in a +// developer's shell can never point these destructive helpers at another database. +function e2eMongoUri(): string { + return `mongodb://hyperdx:hyperdx@localhost:${process.env.HDX_E2E_MONGO_PORT || '21100'}/${E2E_DB_NAME}?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256`; +} + +/** Runs `fn` against the e2e database over the Mongo wire protocol. */ +export async function withE2eDb(fn: (db: Db) => Promise): Promise { + const client = await MongoClient.connect(e2eMongoUri()); + try { + return await fn(client.db(E2E_DB_NAME)); + } finally { + await client.close(); + } +} /** - * Run a mongosh script against the e2e MongoDB container by piping the script - * through stdin. Using stdin (rather than `--eval "<...>"`) avoids having to - * escape quotes in the script body, so callers can pass multi-line JavaScript - * with string literals verbatim. - * - * `execFileSync` with an argument array rather than a shell string: the project - * slug comes from an env var, and with no shell nothing in it can be read as a - * metacharacter. - * - * Throws if the docker-compose file can't be found (meaning we're not running - * in the expected Docker-backed e2e environment). + * Drops the e2e database. DocumentDB intermittently fails this while the API's + * background index builds run (WriteConflict 112, or InternalError 1). */ -export function runMongoshScript(script: string): string { - const dockerComposeFile = path.join(__dirname, '..', 'docker-compose.yml'); - if (!fs.existsSync(dockerComposeFile)) { - throw new Error( - `docker-compose.yml not found at ${dockerComposeFile} — e2e Docker stack unavailable`, - ); +export async function dropE2eDatabase(): Promise { + for (let attempt = 1; ; attempt++) { + try { + await withE2eDb(db => db.dropDatabase()); + return; + } catch (e: any) { + if (attempt >= 5 || ![1, 112].includes(e?.code)) throw e; + await new Promise(resolve => setTimeout(resolve, 200 * attempt)); + } } - - const e2eSlot = process.env.HDX_E2E_SLOT || '0'; - - return execFileSync( - 'docker', - [ - 'compose', - '-p', - `e2e-${e2eSlot}`, - '-f', - dockerComposeFile, - 'exec', - '-T', - 'db', - 'mongosh', - '--quiet', - ], - { - encoding: 'utf-8', - stdio: ['pipe', 'pipe', 'pipe'], - input: script, - }, - ); } /** @@ -64,11 +53,13 @@ export function runMongoshScript(script: string): string { * racing each other — e.g. via `test.describe.serial(...)` — since * `fullyParallel: true` lets tests in the same file run concurrently. */ -export function setTeamFlag(flagName: string, value: boolean): void { - runMongoshScript(` -use('hyperdx-e2e'); -db.teams.updateOne({}, { $set: { [${JSON.stringify(flagName)}]: ${JSON.stringify(value)} } }); -`); +export async function setTeamFlag( + flagName: string, + value: boolean, +): Promise { + await withE2eDb(db => + db.collection('teams').updateOne({}, { $set: { [flagName]: value } }), + ); } const CLICKHOUSE_HOST = diff --git a/scripts/dev-env.sh b/scripts/dev-env.sh index 9ef34ae14b..4c01308c26 100755 --- a/scripts/dev-env.sh +++ b/scripts/dev-env.sh @@ -103,7 +103,7 @@ echo "╠═══════════════════════ echo "║ App (Next.js) http://localhost:${HYPERDX_APP_PORT}$(printf '%*s' $((22 - ${#HYPERDX_APP_PORT})) '')║" echo "║ API http://localhost:${HYPERDX_API_PORT}$(printf '%*s' $((22 - ${#HYPERDX_API_PORT})) '')║" echo "║ ClickHouse http://localhost:${HDX_DEV_CH_HTTP_PORT}$(printf '%*s' $((22 - ${#HDX_DEV_CH_HTTP_PORT})) '')║" -echo "║ MongoDB localhost:${HDX_DEV_MONGO_PORT}$(printf '%*s' $((29 - ${#HDX_DEV_MONGO_PORT})) '')║" +echo "║ DocumentDB localhost:${HDX_DEV_MONGO_PORT}$(printf '%*s' $((29 - ${#HDX_DEV_MONGO_PORT})) '')║" echo "║ OTel HTTP http://localhost:${HDX_DEV_OTEL_HTTP_PORT}$(printf '%*s' $((22 - ${#HDX_DEV_OTEL_HTTP_PORT})) '')║" echo "║ OTel gRPC localhost:${HDX_DEV_OTEL_GRPC_PORT}$(printf '%*s' $((29 - ${#HDX_DEV_OTEL_GRPC_PORT})) '')║" echo "║ OpAMP localhost:${HYPERDX_OPAMP_PORT}$(printf '%*s' $((29 - ${#HYPERDX_OPAMP_PORT})) '')║" diff --git a/scripts/dev-portal/index.html b/scripts/dev-portal/index.html index 8dc02c85eb..505fe84c97 100644 --- a/scripts/dev-portal/index.html +++ b/scripts/dev-portal/index.html @@ -329,7 +329,7 @@

app: 'App (Next.js)', api: 'API Server', clickhouse: 'ClickHouse', - mongodb: 'MongoDB', + documentdb: 'DocumentDB', 'otel-collector': 'OTel Collector', 'otel-collector-json': 'OTel JSON', alerts: 'Alerts Task', diff --git a/scripts/dev-portal/server.js b/scripts/dev-portal/server.js index 05524a4e0a..ab34e6b096 100755 --- a/scripts/dev-portal/server.js +++ b/scripts/dev-portal/server.js @@ -184,7 +184,7 @@ function parseProject(projectName) { // compose service name (ch-server, db, otel-collector, …). const COMPOSE_SERVICE_ALIASES = { 'ch-server': 'clickhouse', - db: 'mongodb', + db: 'documentdb', }; function canonicalServiceName(labels) { @@ -457,7 +457,7 @@ async function buildDashboardData() { 'alerts', 'common-utils', 'clickhouse', - 'mongodb', + 'documentdb', 'otel-collector', 'otel-collector-json', ]; @@ -517,7 +517,7 @@ const ENV_LOG_DIRS = { // Map canonical service names to Docker Compose service names const DOCKER_SERVICE_NAMES = { clickhouse: 'ch-server', - mongodb: 'db', + documentdb: 'db', 'otel-collector': 'otel-collector', 'otel-collector-json': 'otel-collector-json', }; diff --git a/scripts/test-e2e.sh b/scripts/test-e2e.sh index 7b6e060d66..76d3c55ac7 100755 --- a/scripts/test-e2e.sh +++ b/scripts/test-e2e.sh @@ -1,7 +1,7 @@ #!/bin/bash # Run E2E tests in full-stack or local mode -# Full-stack mode (default): MongoDB + API + local ClickHouse -# Local mode: Frontend + local ClickHouse (no MongoDB/API) +# Full-stack mode (default): DocumentDB + API + local ClickHouse +# Local mode: Frontend + local ClickHouse (no DocumentDB/API) # # Usage: # ./scripts/test-e2e.sh # Run all tests in fullstack mode @@ -57,11 +57,9 @@ exec > >(tee "$HDX_E2E_LOGS_DIR/e2e.log") 2>&1 # shellcheck source=./ensure-dev-portal.sh source "${REPO_ROOT}/scripts/ensure-dev-portal.sh" -echo "Using E2E slot ${HDX_E2E_SLOT} (project=${E2E_PROJECT} ch=${HDX_E2E_CH_PORT} ch-native=${HDX_E2E_CH_NATIVE_PORT} mongo=${HDX_E2E_MONGO_PORT} api=${HDX_E2E_API_PORT} app=${HDX_E2E_APP_PORT} app-local=${HDX_E2E_APP_LOCAL_PORT} opamp=${HDX_E2E_OPAMP_PORT})" +echo "Using E2E slot ${HDX_E2E_SLOT} (project=${E2E_PROJECT} ch=${HDX_E2E_CH_PORT} ch-native=${HDX_E2E_CH_NATIVE_PORT} db=${HDX_E2E_MONGO_PORT} api=${HDX_E2E_API_PORT} app=${HDX_E2E_APP_PORT} app-local=${HDX_E2E_APP_LOCAL_PORT} opamp=${HDX_E2E_OPAMP_PORT})" # Configuration constants -readonly MAX_MONGODB_WAIT_ATTEMPTS=15 -readonly MONGODB_WAIT_DELAY_SECONDS=1 readonly MAX_CLICKHOUSE_WAIT_ATTEMPTS=30 readonly CLICKHOUSE_WAIT_DELAY_SECONDS=1 @@ -107,22 +105,6 @@ METAEOF rm -rf "$HDX_E2E_LOGS_DIR" 2>/dev/null || true } -check_mongodb_health() { - # Health check script that tests ping, insert, and delete operations - # Note: MongoDB runs on port 27017 inside the container (default) - docker compose -p "$E2E_PROJECT" -f "$DOCKER_COMPOSE_FILE" exec -T db mongosh --quiet --eval " - try { - db.adminCommand('ping'); - db.getSiblingDB('test').test.insertOne({_id: 'healthcheck', ts: new Date()}); - db.getSiblingDB('test').test.deleteOne({_id: 'healthcheck'}); - print('ready'); - } catch(e) { - print('not ready: ' + e); - quit(1); - } - " 2>&1 -} - check_clickhouse_health() { # Health check from HOST perspective (not inside container) # This ensures the port is actually accessible to Playwright @@ -152,40 +134,6 @@ wait_for_clickhouse() { done } -wait_for_mongodb() { - echo "Waiting for MongoDB to be ready..." - local attempt=1 - - # Verify mongosh is available in the container - if ! docker compose -p "$E2E_PROJECT" -f "$DOCKER_COMPOSE_FILE" exec -T db which mongosh >/dev/null 2>&1; then - echo "ERROR: mongosh not found in MongoDB container" - echo "Container may not be running or using incompatible image" - echo "Try running: docker compose -p $E2E_PROJECT -f $DOCKER_COMPOSE_FILE logs db" - return 1 - fi - - while [ $attempt -le $MAX_MONGODB_WAIT_ATTEMPTS ]; do - local result - result=$(check_mongodb_health) - - if echo "$result" | grep -q "ready"; then - echo "MongoDB is ready and accepting writes" - return 0 - fi - - if [ $attempt -eq $MAX_MONGODB_WAIT_ATTEMPTS ]; then - local total_wait=$((MAX_MONGODB_WAIT_ATTEMPTS * MONGODB_WAIT_DELAY_SECONDS)) - echo "MongoDB failed to become ready after $total_wait seconds" - echo "Last error: $result" - return 1 - fi - - echo "Waiting for MongoDB... ($attempt/$MAX_MONGODB_WAIT_ATTEMPTS)" - attempt=$((attempt + 1)) - sleep $MONGODB_WAIT_DELAY_SECONDS - done -} - # Main execution setup_cleanup_trap() { if [ "$SKIP_CLEANUP" = false ]; then @@ -216,7 +164,7 @@ run_tests() { echo "Running tests in local mode (frontend + ClickHouse)..." yarn test:e2e --local "${PLAYWRIGHT_FLAGS[@]}" else - echo "Running tests in full-stack mode (MongoDB + API + ClickHouse)..." + echo "Running tests in full-stack mode (DocumentDB + API + ClickHouse)..." yarn test:e2e "${PLAYWRIGHT_FLAGS[@]}" fi } @@ -230,12 +178,13 @@ rm -rf "$REPO_ROOT/packages/app/.next-e2e" 2>/dev/null || true # Always start and seed ClickHouse (shared by both modes) setup_clickhouse -# Conditionally start MongoDB for full-stack mode +# Conditionally start DocumentDB for full-stack mode if [ "$LOCAL_MODE" = false ]; then - echo "Starting MongoDB for full-stack mode..." - docker compose -p "$E2E_PROJECT" -f "$DOCKER_COMPOSE_FILE" up -d db - - if ! wait_for_mongodb; then + echo "Starting DocumentDB for full-stack mode..." + # --wait blocks on the image healthcheck, which passes once the gateway accepts connections. + if ! docker compose -p "$E2E_PROJECT" -f "$DOCKER_COMPOSE_FILE" up -d --build --wait --wait-timeout 300 db; then + echo "DocumentDB failed to become healthy" + docker compose -p "$E2E_PROJECT" -f "$DOCKER_COMPOSE_FILE" logs db | tail -20 exit 1 fi fi diff --git a/yarn.lock b/yarn.lock index fc99d7bb95..9ffb586e14 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4880,6 +4880,7 @@ __metadata: ky: "npm:^0.30.0" ky-universal: "npm:^0.10.1" lodash: "npm:^4.18.1" + mongodb: "npm:^6.15.0" ms: "npm:^2.1.3" msw: "npm:^2.3.0" msw-storybook-addon: "npm:^2.0.2" From a7be35acf04bda6b1c5e892a6bd31f716247d0ab Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Thu, 15 Oct 2026 22:20:53 +1100 Subject: [PATCH 2/7] fix(dev): build the e2e DocumentDB image directly and surface Postgres start errors CI e2e setup failed on every shard: `buildx bake` on the e2e compose file resolves its out-of-tree build context as a filesystem entitlement and refuses to run. Build docker/documentdb directly with the same GHA cache. A failed Postgres start now prints the tail of postgresql.log to the container logs, so `docker compose logs db` shows the cause. --- .github/actions/e2e-setup/action.yml | 3 ++- docker/documentdb/entrypoint.sh | 12 ++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/actions/e2e-setup/action.yml b/.github/actions/e2e-setup/action.yml index 3bcbfe1626..bab0316186 100644 --- a/.github/actions/e2e-setup/action.yml +++ b/.github/actions/e2e-setup/action.yml @@ -70,11 +70,12 @@ runs: uses: crazy-max/ghaction-github-runtime@v4 # DocumentDB is built from source; the GHA cache keeps that to the first run. + # The tag must match the e2e compose `db` image so `up` reuses it instead of rebuilding. - name: Build DocumentDB image shell: bash run: | docker buildx create --use --driver=docker-container - docker buildx bake -f packages/app/tests/e2e/docker-compose.yml db --set *.cache-to="type=gha,mode=max,scope=documentdb" --set *.cache-from="type=gha,scope=documentdb" --load + docker buildx build docker/documentdb -t hyperdx/documentdb:local --cache-to "type=gha,mode=max,scope=documentdb" --cache-from "type=gha,scope=documentdb" --load - name: Start E2E Docker Compose shell: bash diff --git a/docker/documentdb/entrypoint.sh b/docker/documentdb/entrypoint.sh index 388c264294..02da89a67f 100755 --- a/docker/documentdb/entrypoint.sh +++ b/docker/documentdb/entrypoint.sh @@ -10,6 +10,14 @@ if [ "$(id -u)" = 0 ]; then exec su-exec postgres "$0" "$@" fi +# Postgres logs to a file; print it on a failed start so `docker logs` shows the cause. +start_postgres() { + pg_ctl -D "$PGDATA" -l "$PGDATA/postgresql.log" -w start >/dev/null || { + tail -n 50 "$PGDATA/postgresql.log" >&2 + exit 1 + } +} + # The marker is written last, so a first boot killed part-way is redone, not left broken. if [ ! -f "$PGDATA/.documentdb-initialised" ]; then find "$PGDATA" -mindepth 1 -delete @@ -25,13 +33,13 @@ documentdb.indexBuildsScheduledOnBgWorker = false documentdb.rum_library_load_option = 'require_documentdb_extended_rum' documentdb.alternate_index_handler_name = 'extended_rum' CONF - pg_ctl -D "$PGDATA" -l "$PGDATA/postgresql.log" -w start >/dev/null + start_postgres psql -h /tmp -U postgres -d postgres -qX -v ON_ERROR_STOP=1 -c "CREATE EXTENSION documentdb CASCADE; CREATE EXTENSION documentdb_extended_rum;" doc=$(printf '{"createUser":"%s","pwd":"%s","roles":[{"role":"readWriteAnyDatabase","db":"admin"},{"role":"clusterAdmin","db":"admin"}]}' "$DOCDB_USER" "$DOCDB_PASSWORD") psql -h /tmp -U postgres -d postgres -qX -v ON_ERROR_STOP=1 -v doc="$doc" <<< "SELECT documentdb_api.create_user(:'doc');" >/dev/null touch "$PGDATA/.documentdb-initialised" else - pg_ctl -D "$PGDATA" -l "$PGDATA/postgresql.log" -w start >/dev/null + start_postgres fi echo "postgresql://postgres@localhost:5432/postgres" > /tmp/pg-url From aecade3cbe3e47caa4f6beff5acf594925fddea6 Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Thu, 15 Oct 2026 22:38:56 +1100 Subject: [PATCH 3/7] fix(ci): wait only on DocumentDB before integration tests The integration job's `up --wait` failed before any test ran: the OTel collector's healthcheck needs an OpAMP server the job doesn't run, so it never reports healthy. Start every service as before, then wait on db. --- .github/workflows/main.yml | 4 +++- Makefile | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 912d7fe028..8f10361478 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -143,7 +143,9 @@ jobs: # Actions streams stdout. - name: Run integration tests run: | - docker compose -f ./docker-compose.ci.yml up -d --quiet-pull --wait --wait-timeout 300 + docker compose -f ./docker-compose.ci.yml up -d --quiet-pull + # Wait on DocumentDB only: the collector's healthcheck needs an OpAMP server this job doesn't run. + docker compose -f ./docker-compose.ci.yml up -d --wait --wait-timeout 300 db status=0 yarn workspace @hyperdx/common-utils ci:int --shard=${{ matrix.shard }}/${{ strategy.job-total }} || status=1 yarn workspace @hyperdx/api ci:int --shard=${{ matrix.shard }}/${{ strategy.job-total }} || status=1 diff --git a/Makefile b/Makefile index 02caba3ab8..87192ea1e4 100644 --- a/Makefile +++ b/Makefile @@ -158,7 +158,8 @@ dev-int: @echo "Using CI slot $(HDX_CI_SLOT) (project=$(HDX_CI_PROJECT) ch=$(HDX_CI_CH_PORT) mongo=$(HDX_CI_MONGO_PORT) api=$(HDX_CI_API_PORT))" @mkdir -p $(HDX_CI_LOGS_DIR) @bash scripts/ensure-dev-portal.sh - docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --build --wait --wait-timeout 300 + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --build + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --wait --wait-timeout 300 db bash -c 'set -o pipefail; npx nx run @hyperdx/api:dev:int $(FILE) 2>&1 | tee $(HDX_CI_LOGS_DIR)/api-int.log'; ret=$$?; \ docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down -v; \ $(call archive-int-logs); \ @@ -169,7 +170,8 @@ dev-int-common-utils: @echo "Using CI slot $(HDX_CI_SLOT) (project=$(HDX_CI_PROJECT) ch=$(HDX_CI_CH_PORT) mongo=$(HDX_CI_MONGO_PORT))" @mkdir -p $(HDX_CI_LOGS_DIR) @bash scripts/ensure-dev-portal.sh - docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --build --wait --wait-timeout 300 + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --build + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --wait --wait-timeout 300 db bash -c 'set -o pipefail; npx nx run @hyperdx/common-utils:dev:int $(FILE) 2>&1 | tee $(HDX_CI_LOGS_DIR)/common-utils-int.log'; ret=$$?; \ docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down -v; \ $(call archive-int-logs); \ @@ -178,7 +180,8 @@ dev-int-common-utils: .PHONY: ci-int ci-int: @mkdir -p $(HDX_CI_LOGS_DIR) - docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --quiet-pull --wait --wait-timeout 300 + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --quiet-pull + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --wait --wait-timeout 300 db bash -c 'set -o pipefail; npx nx run-many -t ci:int --parallel=false --output-style=stream 2>&1 | tee $(HDX_CI_LOGS_DIR)/ci-int.log'; ret=$$?; \ docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down -v; \ $(call archive-int-logs); \ From a4a8b4639098ba52495a327a97161c4429870473 Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Thu, 5 Nov 2026 16:27:09 +1100 Subject: [PATCH 4/7] fix(ci): give the DocumentDB image its own integration cache scope Integration shards were rebuilding DocumentDB from source and timing out. The bake shared one GHA cache scope between the collector and db images, and a scope holds a single manifest, so each export overwrote the other. db now uses the documentdb scope the e2e job already fills. --- .github/workflows/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8f10361478..346bc40286 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -121,7 +121,11 @@ jobs: - name: Spin up docker services run: | docker buildx create --use --driver=docker-container - docker buildx bake -f ./docker-compose.ci.yml --set *.cache-to="type=gha,mode=max,scope=ci-integration" --set *.cache-from="type=gha,scope=ci-integration" --load + # One cache scope per image (a scope holds one manifest); db reuses the e2e job's documentdb scope. + docker buildx bake -f ./docker-compose.ci.yml \ + --set otel-collector.cache-to="type=gha,mode=max,scope=ci-integration" --set otel-collector.cache-from="type=gha,scope=ci-integration" \ + --set db.cache-to="type=gha,mode=max,scope=documentdb" --set db.cache-from="type=gha,scope=documentdb" \ + --load - name: Build dependencies run: make ci-build # Integration tests boot a MockServer on fixed ports and share a single From a83c279521f4e5a6a3aec6e0a7a327a67159cc0d Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Thu, 19 Nov 2026 09:13:59 +1100 Subject: [PATCH 5/7] chore(dev): take main's self-telemetry env vars ahead of merging main --- packages/api/.env.development | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/api/.env.development b/packages/api/.env.development index 9da78e573a..0c856a56a5 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -3,9 +3,10 @@ HYPERDX_LOG_LEVEL=debug # Development only: keep sessions valid across nodemon restarts. EXPRESS_SESSION_SECRET="local-development-only" FRONTEND_URL="http://localhost:${HYPERDX_APP_PORT}" -HDX_NODE_ADVANCED_NETWORK_CAPTURE=1 -HDX_NODE_BETA_MODE=1 -HDX_NODE_CONSOLE_CAPTURE=1 +# Self-telemetry request and response capture. Booleans must be `true`. +CLICKSTACK_CAPTURE_HTTP_BODY=true +OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="user-agent,content-type,x-request-id,referer" +OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="content-type,content-length" MONGO_URI="mongodb://hyperdx:hyperdx@localhost:${HDX_DEV_MONGO_PORT}/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256" NODE_ENV=development OTEL_SERVICE_NAME="hdx-oss-dev-api" @@ -22,7 +23,7 @@ USAGE_STATS_ENABLED=false NODE_OPTIONS="--max-http-header-size=131072" ENABLE_SWAGGER=true DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""}] -DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"},{"kind":"promql","name":"PromQL","connection":"Local ClickHouse","from":{"databaseName":"default","tableName":"metrics_ts"},"timestampValueExpression":"timestamp"}] +DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","severityNumberExpression":"SeverityNumber","eventNameExpression":"EventName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","metadataMaterializedViews":{"kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"Logs"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","resourceAttributesExpression":"ResourceAttributes","connection":"Local ClickHouse","traceSourceId":"Traces"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","severityNumberExpression":"SeverityNumber","eventNameExpression":"EventName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"JSON Logs"},{"kind":"promql","name":"Time Series","connection":"Local ClickHouse","from":{"databaseName":"default","tableName":"metrics_ts"},"timestampValueExpression":"timestamp"}] INGESTION_API_KEY="super-secure-ingestion-api-key" HYPERDX_API_KEY=$INGESTION_API_KEY From 10ccf3293f8bc018fddb3b5cd42b4797e16a11d3 Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Thu, 19 Nov 2026 09:30:59 +1100 Subject: [PATCH 6/7] chore(dev): bump DocumentDB to v1.2-0 and address review feedback - Build DocumentDB v1.2-0 (pinned to its commit) instead of v1.0-RC2. - Publish DocumentDB on loopback only in the dev, integration and e2e compose files; the credentials are well known. - Name the shared DocumentDB credentials and TLS options in the e2e helpers and the alert seed script, and type the drop retry's error. - Keep the entrypoint's TERM trap from aborting before Postgres stops. - Update the e2e README and the AGENTS.md data-flow line for DocumentDB. --- AGENTS.md | 2 +- docker-compose.ci.yml | 2 +- docker-compose.dev.yml | 2 +- docker/documentdb/Dockerfile | 4 +-- docker/documentdb/entrypoint.sh | 2 +- packages/api/.env.development | 1 + packages/api/.env.e2e | 2 +- packages/api/.env.test | 1 + packages/api/scripts/seed-alerts.ts | 7 +++- packages/app/tests/e2e/README.md | 40 ++++++++++------------ packages/app/tests/e2e/docker-compose.yml | 2 +- packages/app/tests/e2e/utils/db-helpers.ts | 20 ++++++----- 12 files changed, 47 insertions(+), 38 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3590fde95d..6ad0d13690 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,7 +34,7 @@ This is a **monorepo** with six packages: comparison workflow. **Data flow**: Apps → OpenTelemetry Collector → ClickHouse (telemetry data) / -MongoDB (configuration/metadata) +a MongoDB-compatible store (configuration/metadata; DocumentDB in local dev) ## Development Setup (HOW) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index fc0ec66a8f..c47ecf97cd 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -41,7 +41,7 @@ services: DOCDB_USER: hyperdx DOCDB_PASSWORD: hyperdx ports: - - '${HDX_CI_MONGO_PORT:-39999}:10260' + - '127.0.0.1:${HDX_CI_MONGO_PORT:-39999}:10260' networks: - internal networks: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 4eaf9c3036..bb09bc74cc 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -17,7 +17,7 @@ services: volumes: - .volumes/docdb_dev_${HDX_DEV_SLOT:-0}:/data/docdb ports: - - '${HDX_DEV_MONGO_PORT:-27017}:10260' + - '127.0.0.1:${HDX_DEV_MONGO_PORT:-27017}:10260' networks: - internal otel-collector: diff --git a/docker/documentdb/Dockerfile b/docker/documentdb/Dockerfile index a17c9c9b0b..1da60fb441 100644 --- a/docker/documentdb/Dockerfile +++ b/docker/documentdb/Dockerfile @@ -1,9 +1,9 @@ # DocumentDB (MIT): Postgres extensions plus a Mongo wire-protocol gateway, used # in place of MongoDB (SSPL). Built from source because upstream ships glibc # packages only; Alpine 3.24 matches the all-in-one base and its PostGIS targets PG18. -ARG DOCUMENTDB_VERSION=v1.0-RC2 +ARG DOCUMENTDB_VERSION=v1.2-0 # Tags can move; the build fails unless the tag still resolves to this commit. -ARG DOCUMENTDB_COMMIT=b0cb0dae85e8746f4e0b79c847894179b2f80d64 +ARG DOCUMENTDB_COMMIT=965b6e7ad1c49aefb983a412a3fea0ff637cd7f1 FROM alpine:3.24 AS builder ARG DOCUMENTDB_VERSION diff --git a/docker/documentdb/entrypoint.sh b/docker/documentdb/entrypoint.sh index 02da89a67f..d7b294b23a 100755 --- a/docker/documentdb/entrypoint.sh +++ b/docker/documentdb/entrypoint.sh @@ -49,7 +49,7 @@ export DOCUMENTDB_PG_URL_FILE=/tmp/pg-url DOCUMENTDB_TLS_STATE_DIR=/tmp/tls documentdb-gateway run --config /etc/documentdb/gateway/SetupConfiguration.json & gateway=$! # Stop Postgres cleanly on docker stop so persistent volumes skip crash recovery. -trap 'kill "$gateway" 2>/dev/null; pg_ctl -D "$PGDATA" -m fast stop >/dev/null; exit 0' TERM INT +trap 'kill "$gateway" 2>/dev/null || true; pg_ctl -D "$PGDATA" -m fast stop >/dev/null; exit 0' TERM INT # Keep the gateway's exit code so a crash shows as a failed container, not a clean exit. rc=0 wait "$gateway" || rc=$? diff --git a/packages/api/.env.development b/packages/api/.env.development index 0c856a56a5..54c44226db 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -7,6 +7,7 @@ FRONTEND_URL="http://localhost:${HYPERDX_APP_PORT}" CLICKSTACK_CAPTURE_HTTP_BODY=true OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="user-agent,content-type,x-request-id,referer" OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="content-type,content-length" +# DocumentDB credentials match DOCDB_USER/DOCDB_PASSWORD in the docker-compose files. MONGO_URI="mongodb://hyperdx:hyperdx@localhost:${HDX_DEV_MONGO_PORT}/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256" NODE_ENV=development OTEL_SERVICE_NAME="hdx-oss-dev-api" diff --git a/packages/api/.env.e2e b/packages/api/.env.e2e index 8b93a929c7..a47b442e66 100644 --- a/packages/api/.env.e2e +++ b/packages/api/.env.e2e @@ -10,7 +10,7 @@ CLICKHOUSE_PASSWORD= CLICKHOUSE_USER=default RUN_SCHEDULED_TASKS_EXTERNALLY=true FRONTEND_URL=http://localhost:${HDX_E2E_APP_PORT:-21300} -# DocumentDB (Mongo wire protocol) connection string +# DocumentDB (Mongo wire protocol); credentials match DOCDB_USER/DOCDB_PASSWORD in the compose files. MONGO_URI=mongodb://hyperdx:hyperdx@localhost:${HDX_E2E_MONGO_PORT:-21100}/hyperdx-e2e?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256 NODE_ENV=test PORT=${HDX_E2E_API_PORT:-21000} diff --git a/packages/api/.env.test b/packages/api/.env.test index ddba188320..bf91334d19 100644 --- a/packages/api/.env.test +++ b/packages/api/.env.test @@ -7,6 +7,7 @@ CLICKHOUSE_PASSWORD=api CLICKHOUSE_USER=api RUN_SCHEDULED_TASKS_EXTERNALLY=true FRONTEND_URL=http://app:8080 +# DocumentDB credentials match DOCDB_USER/DOCDB_PASSWORD in the docker-compose files. MONGO_URI=mongodb://hyperdx:hyperdx@localhost:${HDX_CI_MONGO_PORT:-39999}/hyperdx-test?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256 NODE_ENV=test PORT=${HDX_CI_API_PORT:-19000} diff --git a/packages/api/scripts/seed-alerts.ts b/packages/api/scripts/seed-alerts.ts index fa68ca4e18..532176fb7b 100644 --- a/packages/api/scripts/seed-alerts.ts +++ b/packages/api/scripts/seed-alerts.ts @@ -119,6 +119,11 @@ function parseArgs(argv: string[]): Args { return args; } +// Matches DOCDB_USER/DOCDB_PASSWORD in docker-compose.dev.yml and MONGO_URI in .env.development. +const DEV_DOCDB_AUTH = 'hyperdx:hyperdx'; +const DEV_DOCDB_OPTIONS = + 'tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256'; + /** * URI of this worktree's dev DocumentDB. `yarn seed:alerts` runs through * scripts/with-slots.sh, which sets HDX_DEV_MONGO_PORT. @@ -130,7 +135,7 @@ function devSlotMongoUri(): string { 'HDX_DEV_MONGO_PORT is not set. Run this with `yarn seed:alerts`, or pass --mongo-uri.', ); } - return `mongodb://hyperdx:hyperdx@localhost:${port}/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256`; + return `mongodb://${DEV_DOCDB_AUTH}@localhost:${port}/hyperdx?${DEV_DOCDB_OPTIONS}`; } function randomInt(minInclusive: number, maxInclusive: number): number { diff --git a/packages/app/tests/e2e/README.md b/packages/app/tests/e2e/README.md index a7e5922cab..da79b6eac7 100644 --- a/packages/app/tests/e2e/README.md +++ b/packages/app/tests/e2e/README.md @@ -14,11 +14,11 @@ feature-specific test suites. ### Default: Full-Stack Mode -By default, `make e2e` runs tests in **full-stack mode** with MongoDB + API + +By default, `make e2e` runs tests in **full-stack mode** with DocumentDB + API + local Docker ClickHouse for maximum consistency and real backend features: ```bash -# Run all tests (full-stack with MongoDB + API + local Docker ClickHouse) +# Run all tests (full-stack with DocumentDB + API + local Docker ClickHouse) make e2e # For UI, specific tests, or other options, use the script from repo root: @@ -31,10 +31,10 @@ make e2e ### Optional: Local Mode (Frontend Only) For faster iteration during development, use the script with `--local` to skip -MongoDB and run frontend-only tests: +DocumentDB and run frontend-only tests: ```bash -# From repo root - run local tests (no MongoDB, frontend only) +# From repo root - run local tests (no DocumentDB, frontend only) ./scripts/test-e2e.sh --local ./scripts/test-e2e.sh --local --ui ./scripts/test-e2e.sh --local --grep "@search" @@ -95,12 +95,13 @@ In the Playwright UI sidebar, click the **eye icon** next to a test (or file/des #### Full-Stack Mode (Default) -**Default behavior** - runs with real backend (MongoDB + API) and demo +**Default behavior** - runs with real backend (DocumentDB + API) and demo ClickHouse data. **What it includes:** -- MongoDB (port 29998) - authentication, teams, users, persistence +- DocumentDB (`HDX_E2E_MONGO_PORT`, 21100 + slot) - authentication, teams, + users, persistence. The API reaches it over the Mongo wire protocol - API Server (port 29000) - full backend logic - App Server (port 28081) - frontend - **Local Docker ClickHouse** (localhost:8123) - seeded E2E test data (logs/traces/metrics/K8s). Seeded timestamps span a past+future window (~1h past, ~2h future from seed time) so relative ranges like "last 5 minutes" keep finding data. If you run tests more than ~2 hours after the last seed, re-run the global setup (or full test run) to re-seed. @@ -121,7 +122,7 @@ make e2e #### Local Mode (for testing frontend-only features) -**Frontend + ClickHouse mode** - skips MongoDB/API, uses local Docker ClickHouse +**Frontend + ClickHouse mode** - skips DocumentDB/API, uses local Docker ClickHouse with seeded test data. **Use for:** @@ -394,21 +395,19 @@ Tests use the extended base test from `utils/base-test.ts` which provides: **Local Environment (make e2e):** -- MongoDB: 29998 (custom port to avoid conflicts) +- DocumentDB: `HDX_E2E_MONGO_PORT` (21100 + slot), mapped to the gateway's + port 10260 - API Server: 29000 - App Server: 28081 **CI Environment (GitHub Actions):** -- MongoDB: 27017 (default, accessed via service name `mongodb`) +- DocumentDB: 21100, from the same compose file (`.github/actions/e2e-setup`) - API Server: 29000 - App Server: 28081 -The MongoDB port differs between local and CI to: - -- Avoid conflicts with existing MongoDB instances locally (port 27017) -- Use standard ports in isolated CI containers (port 27017) -- CI accesses MongoDB via hostname `mongodb` instead of `localhost` +Locally the port is offset by the worktree slot so several worktrees can run +e2e at once; CI runs one stack per runner at slot 0. ### Playwright Configuration @@ -428,13 +427,12 @@ multiple servers: - Check development server started successfully - Verify environment variables in `.env.e2e` -**MongoDB connection issues (full-stack mode):** +**DocumentDB connection issues (full-stack mode):** -- Check port 29998 is available locally: `lsof -i :29998` -- View MongoDB logs: - `docker compose -p e2e -f tests/e2e/docker-compose.yml logs` -- MongoDB is auto-managed by `make e2e` (default) -- Note: CI uses port 27017 internally (accessed via service name) +- Check the slot's port is free: `lsof -i :$HDX_E2E_MONGO_PORT` +- View DocumentDB logs (a failed Postgres start prints its log here): + `docker compose -p e2e- -f tests/e2e/docker-compose.yml logs db` +- DocumentDB is started by `make e2e`, which waits for its healthcheck **Sources don't appear in UI:** @@ -462,7 +460,7 @@ For intermittent failures: Tests run in **full-stack mode** on CI (GitHub Actions) with: -- MongoDB service container for authentication and persistence +- DocumentDB container for authentication and persistence - Local Docker ClickHouse for telemetry data (same as local mode) - 60-second test timeout (same as local) - Multiple retry attempts (2 retries on CI vs 1 locally) diff --git a/packages/app/tests/e2e/docker-compose.yml b/packages/app/tests/e2e/docker-compose.yml index a4e811101a..d3c6653a14 100644 --- a/packages/app/tests/e2e/docker-compose.yml +++ b/packages/app/tests/e2e/docker-compose.yml @@ -7,7 +7,7 @@ services: DOCDB_USER: hyperdx DOCDB_PASSWORD: hyperdx ports: - - ${HDX_E2E_MONGO_PORT:-21100}:10260 + - 127.0.0.1:${HDX_E2E_MONGO_PORT:-21100}:10260 networks: - internal deploy: diff --git a/packages/app/tests/e2e/utils/db-helpers.ts b/packages/app/tests/e2e/utils/db-helpers.ts index 947f4a4974..4da807f215 100644 --- a/packages/app/tests/e2e/utils/db-helpers.ts +++ b/packages/app/tests/e2e/utils/db-helpers.ts @@ -3,16 +3,22 @@ * usable in full-stack mode (DocumentDB via docker-compose); there is no database in * local mode, so callers must gate on `{ tag: ['@full-stack'] }`. */ -import { type Db, MongoClient } from 'mongodb'; +import { type Db, MongoClient, MongoServerError } from 'mongodb'; export { ObjectId } from 'mongodb'; const E2E_DB_NAME = 'hyperdx-e2e'; +// Matches DOCDB_USER/DOCDB_PASSWORD in the e2e compose file and MONGO_URI in packages/api/.env.e2e. +const DOCDB_AUTH = 'hyperdx:hyperdx'; +const DOCDB_OPTIONS = + 'tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256'; +// DocumentDB's transient dropDatabase failures: WriteConflict and InternalError. +const TRANSIENT_DROP_CODES = [112, 1]; // Built from the e2e slot port rather than MONGO_URI, so a MONGO_URI exported in a // developer's shell can never point these destructive helpers at another database. function e2eMongoUri(): string { - return `mongodb://hyperdx:hyperdx@localhost:${process.env.HDX_E2E_MONGO_PORT || '21100'}/${E2E_DB_NAME}?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256`; + return `mongodb://${DOCDB_AUTH}@localhost:${process.env.HDX_E2E_MONGO_PORT || '21100'}/${E2E_DB_NAME}?${DOCDB_OPTIONS}`; } /** Runs `fn` against the e2e database over the Mongo wire protocol. */ @@ -25,17 +31,15 @@ export async function withE2eDb(fn: (db: Db) => Promise): Promise { } } -/** - * Drops the e2e database. DocumentDB intermittently fails this while the API's - * background index builds run (WriteConflict 112, or InternalError 1). - */ +/** Drops the e2e database, retrying while the API's background index builds run. */ export async function dropE2eDatabase(): Promise { for (let attempt = 1; ; attempt++) { try { await withE2eDb(db => db.dropDatabase()); return; - } catch (e: any) { - if (attempt >= 5 || ![1, 112].includes(e?.code)) throw e; + } catch (e: unknown) { + const code = e instanceof MongoServerError ? e.code : undefined; + if (attempt >= 5 || !TRANSIENT_DROP_CODES.includes(Number(code))) throw e; await new Promise(resolve => setTimeout(resolve, 200 * attempt)); } } From 42087e938a479ae926a5703e95ea01903fea3d96 Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Fri, 20 Nov 2026 08:53:45 +1100 Subject: [PATCH 7/7] docs: describe the local DocumentDB stack in the remaining dev docs Contributing, eval and Playwright docs still said the local dev, e2e and eval stacks run MongoDB. Also document a psql one-liner for reaching the Postgres under DocumentDB. --- .claude/agents/playwright-test-generator.md | 2 +- .claude/agents/playwright-test-planner.md | 2 +- .claude/skills/playwright/SKILL.md | 2 +- CONTRIBUTING.md | 12 +++++++----- agent_docs/development.md | 2 ++ agent_docs/evals.md | 6 +++--- packages/hdx-eval/README.md | 4 ++-- 7 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.claude/agents/playwright-test-generator.md b/.claude/agents/playwright-test-generator.md index ac3f237d4b..79d6b3e642 100644 --- a/.claude/agents/playwright-test-generator.md +++ b/.claude/agents/playwright-test-generator.md @@ -58,7 +58,7 @@ The webhook API enforces uniqueness on `(team, service, url)`. Hardcoded URLs wi - Assert successful chart loads by checking `.recharts-responsive-container` is visible ### Tags -- `{ tag: '@full-stack' }` for tests requiring the backend (MongoDB + API) +- `{ tag: '@full-stack' }` for tests requiring the backend (DocumentDB + API) - Feature tags: `@dashboard`, `@alerts`, `@search`, etc. diff --git a/.claude/agents/playwright-test-planner.md b/.claude/agents/playwright-test-planner.md index 5c0d2e2784..5cab5c7826 100644 --- a/.claude/agents/playwright-test-planner.md +++ b/.claude/agents/playwright-test-planner.md @@ -63,6 +63,6 @@ HyperDX is an observability platform. Key pages: `/search` (logs/traces), `/dash - Components: `packages/app/tests/e2e/components/` ### Plan Requirements for HyperDX -- Scenarios must be independent and assume a fresh DB state (the test runner clears MongoDB before each run) +- Scenarios must be independent and assume a fresh DB state (the test runner clears DocumentDB before each run) - Note when a scenario creates shared resources (e.g. webhooks, saved searches) that could conflict with parallel runs — flag these for data isolation - Reference existing page objects when describing steps so the generator agent knows what abstractions are available \ No newline at end of file diff --git a/.claude/skills/playwright/SKILL.md b/.claude/skills/playwright/SKILL.md index 070d678076..a0dfc76b19 100644 --- a/.claude/skills/playwright/SKILL.md +++ b/.claude/skills/playwright/SKILL.md @@ -78,7 +78,7 @@ The webhook API enforces uniqueness on `(team, service, url)`. A hardcoded URL w - Assert successful chart loads by checking `.recharts-responsive-container` is visible ### Tags -- `{ tag: '@full-stack' }` — tests requiring MongoDB + API backend +- `{ tag: '@full-stack' }` — tests requiring DocumentDB + API backend - Feature tags: `@dashboard`, `@alerts`, `@search`, etc. ### Imports diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fd948ba21e..a2e95a986b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -178,7 +178,8 @@ Service Descriptions: instrumented applications and forwards it to ClickHouse for storage. Includes OpAMP supervisor that dynamically pulls configuration from ClickStack API. - ClickHouse (ch-server): ClickHouse database, stores all telemetry. -- MongoDB (db): Stores user/saved search/alert/dashboard data. +- MongoDB (db): Stores user/saved search/alert/dashboard data. Local dev runs + DocumentDB instead, which the API reaches over the Mongo wire protocol. - ClickStack API (api): Node.js API, executes ClickHouse queries on behalf of the frontend and serves the frontend. serves the frontend. Can also run alert checker. @@ -199,7 +200,8 @@ yarn dev ``` This will start the Node.js API, Next.js frontend locally and the OpenTelemetry -collector and ClickHouse server in Docker. +collector, ClickHouse server and DocumentDB in Docker. To browse DocumentDB, see +the connection string in `agent_docs/development.md`. Each worktree automatically gets unique ports so multiple developers (or agents) can run `yarn dev` simultaneously without conflicts. A dev portal at @@ -236,8 +238,8 @@ see them reflected in real-time. The development stack mounts volumes locally for persisting storage under `.volumes`. Each worktree gets its own volume directory (e.g. -`.volumes/ch_data_dev_89`). Clear the `.volumes` directory to reset ClickHouse -and MongoDB storage. +`.volumes/ch_data_dev_89`, `.volumes/docdb_dev_89`). Clear the `.volumes` +directory to reset ClickHouse and DocumentDB storage. ### Windows @@ -257,7 +259,7 @@ simultaneously with the dev stack and across multiple worktrees. ### E2E Tests -E2E tests run against a full local stack (MongoDB + ClickHouse + API). Docker +E2E tests run against a full local stack (DocumentDB + ClickHouse + API). Docker must be running. ```bash diff --git a/agent_docs/development.md b/agent_docs/development.md index f3c281c4d2..e7feee067f 100644 --- a/agent_docs/development.md +++ b/agent_docs/development.md @@ -38,6 +38,8 @@ yarn dev # or equivalently: make dev `mongodb://hyperdx:hyperdx@localhost:/?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256`. Everything works except Compass's Performance tab (the gateway doesn't implement `serverStatus` or `top`). + For the Postgres underneath (documents are BSON rows in `documentdb_data`), + run `docker exec -it -u postgres hdx-dev--db-1 psql -h /tmp -U postgres -d postgres`. Dev data lives in `.volumes/docdb_dev_`; data from the old MongoDB volumes (`.volumes/db_dev_`) is not migrated, so the dev stack starts empty. - Hot reload enabled for all services in development diff --git a/agent_docs/evals.md b/agent_docs/evals.md index d7e091aa3f..f57003a1ee 100644 --- a/agent_docs/evals.md +++ b/agent_docs/evals.md @@ -28,7 +28,7 @@ compare any two (or more) MCP servers side by side. | Requirement | Notes | |---|---| -| `yarn dev` running | Each worktree needs `yarn dev` running with a fixed slot. This handles `yarn install`, `yarn build:common-utils`, Docker containers (ClickHouse, MongoDB), and the API server. See [Development Setup](development.md). | +| `yarn dev` running | Each worktree needs `yarn dev` running with a fixed slot. This handles `yarn install`, `yarn build:common-utils`, Docker containers (ClickHouse, DocumentDB), and the API server. See [Development Setup](development.md). | | `ANTHROPIC_API_KEY` or `AI_API_KEY` | Auto-loaded from `.env.local` at the monorepo root. Check `.env.local` in other worktrees for existing keys if missing. | | `claude` CLI | Installed globally (`which claude`). The harness spawns it in streaming JSON mode. | | `uv` | Only needed for `stdio`-type MCPs (e.g. raw ClickHouse MCP). Install via `curl -LsSf https://astral.sh/uv/install.sh \| sh`. | @@ -75,7 +75,7 @@ dual-slot comparisons you must seed each instance explicitly (step 4b). ### 1. Start both dev stacks -`yarn dev` starts Docker infra (ClickHouse, MongoDB, OTel Collector) **and** +`yarn dev` starts Docker infra (ClickHouse, DocumentDB, OTel Collector) **and** the API + App servers in one command. Run it from each worktree with a fixed slot: @@ -326,7 +326,7 @@ Always use the `bash -c 'export HDX_DEV_SLOT=... && source scripts/dev-env.sh ### Access keys go stale after container restarts -When MongoDB is recreated, the eval account's access key changes. Re-run +When DocumentDB is recreated, the eval account's access key changes. Re-run `setup-hyperdx` against the affected instance and rebuild the combined config. ### False timeouts diff --git a/packages/hdx-eval/README.md b/packages/hdx-eval/README.md index db6f9e634a..6f93b28f9f 100644 --- a/packages/hdx-eval/README.md +++ b/packages/hdx-eval/README.md @@ -13,7 +13,7 @@ selected at run time. - **`yarn dev` running** on at least one worktree — this handles `yarn install`, `yarn build:common-utils`, Docker containers (ClickHouse, - MongoDB), and the API server. See `agent_docs/development.md` for setup. + DocumentDB), and the API server. See `agent_docs/development.md` for setup. - **`claude` CLI installed** (the harness spawns it in streaming JSON mode) - **`ANTHROPIC_API_KEY`** or **`AI_API_KEY`** in `.env.local` at the monorepo root (for `run` and `grade` commands) @@ -57,7 +57,7 @@ yarn workspace @hyperdx/hdx-eval viewer For reliable A/B comparison of two HyperDX branches (e.g. feature vs main), we recommend reserving two fixed dev slots. This gives you two fully isolated -HyperDX stacks (separate ClickHouse, MongoDB, API) that can run +HyperDX stacks (separate ClickHouse, DocumentDB, API) that can run simultaneously. ### 1. Start two dev stacks with fixed slots