From d9b100064395ba44cdf99f1164a14cbde47aa7ba Mon Sep 17 00:00:00 2001 From: Jeremy Stein Date: Wed, 2 Sep 2026 14:23:07 +0100 Subject: [PATCH 1/7] Configure grafana to be reverse proxy-friendly and connect it to the same docker network as the Emap portal --- README.md | 15 +++++++++++++-- docker-compose.lgtm.yml | 17 +++++++++++++++++ docs/deployment.md | 3 ++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f8bd860..f490d23 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,8 @@ mkdir config cp waveform-controller/config.EXAMPLE/controller.env.EXAMPLE config/controller.env cp waveform-controller/config.EXAMPLE/exporter.env.EXAMPLE config/exporter.env cp waveform-controller/config.EXAMPLE/hasher.env.EXAMPLE config/hasher.env +cp waveform-controller/config.EXAMPLE/monitoring.env.EXAMPLE config/monitoring.env +cp waveform-controller/config.EXAMPLE/lgtm.env.EXAMPLE config/lgtm.env ``` From the new config files, remove the comments telling you not to put secrets in it, as instructed. @@ -118,13 +120,22 @@ messages. mkdir waveform-export ``` +##### Create docker network +If not already done, create the docker network used for bridging the Emap +portal with the local LGTM (grafana) instance: + +`docker create network emap-portal-grafana` + ##### run it! Build and start the hasher, controller and exporter with docker. +(the `--profile lgtm` brings up the local telemetry collector, +which is optional and soon to be replaced +with a GAE-level lgtm instance) ``` cd waveform-controller -docker compose build -docker compose up -d +docker compose -f docker-compose.yml -f docker-compose.lgtm.yml --profile lgtm build +docker compose -f docker-compose.yml -f docker-compose.lgtm.yml --profile lgtm up -d ``` For more complex deployment scenarios, such as where there is existing data you need to preserve, diff --git a/docker-compose.lgtm.yml b/docker-compose.lgtm.yml index 4dbade2..626ff33 100644 --- a/docker-compose.lgtm.yml +++ b/docker-compose.lgtm.yml @@ -18,6 +18,18 @@ services: restart: unless-stopped profiles: - lgtm + environment: + # set this to match the emap portal page + GF_SERVER_ROOT_URL: "%(protocol)s://localhost:7100/local-grafana/" + GF_SERVER_SERVE_FROM_SUB_PATH: "true" + + # Grafana is only accessible on localhost or through password-protected + # Emap portal, so some non-admin access is fine. + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: "Editor" + networks: + - default + - emap-portal-grafana ports: # Grafana UI - "127.0.0.1:3000:3000" @@ -35,3 +47,8 @@ services: volumes: lgtm-data: + +networks: + # allow the emap-portal to connect to us + emap-portal-grafana: + external: true diff --git a/docs/deployment.md b/docs/deployment.md index c35bf33..3c34968 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -135,7 +135,8 @@ Bring up any Emap services that we brought down: Emap repo: `emap docker up -d` Bring up the waveform controller/export if you brought them down. -Waveform repo: `docker compose up -d` +Waveform repo: `docker compose -f docker-compose.yml -f docker-compose.lgtm.yml --profile lgtm up -d` + ### Replay old HL7 data From 65c6eae01020fcf8b9520062ca9b0c52c3973b35 Mon Sep 17 00:00:00 2001 From: Jeremy Stein Date: Wed, 2 Sep 2026 14:59:58 +0100 Subject: [PATCH 2/7] Make public hostname configurable so can deploy on GAE --- config.EXAMPLE/lgtm.env.EXAMPLE | 8 ++++++++ docker-compose.lgtm.yml | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 config.EXAMPLE/lgtm.env.EXAMPLE diff --git a/config.EXAMPLE/lgtm.env.EXAMPLE b/config.EXAMPLE/lgtm.env.EXAMPLE new file mode 100644 index 0000000..c59163f --- /dev/null +++ b/config.EXAMPLE/lgtm.env.EXAMPLE @@ -0,0 +1,8 @@ +# This is an EXAMPLE file, do not put real secrets in here. +# Copy it to ../config/lgtm.env and then DELETE THIS COMMENT. + +# Change the URL hostname for each deployment, to match the hostname +# that the browser sees when accessing this site. Eg. the GAE hostname. +# The URL path should not routinely be changed, as it needs to match +# what the reverse proxy (nginx) is serving us on. +GF_SERVER_ROOT_URL: "%(protocol)s://localhost:7100/local-grafana/" diff --git a/docker-compose.lgtm.yml b/docker-compose.lgtm.yml index 626ff33..1cc99f9 100644 --- a/docker-compose.lgtm.yml +++ b/docker-compose.lgtm.yml @@ -18,9 +18,9 @@ services: restart: unless-stopped profiles: - lgtm + env_file: + - ../config/lgtm.env environment: - # set this to match the emap portal page - GF_SERVER_ROOT_URL: "%(protocol)s://localhost:7100/local-grafana/" GF_SERVER_SERVE_FROM_SUB_PATH: "true" # Grafana is only accessible on localhost or through password-protected From bdfc8c88bb5d318d9edd0a61398e64d7848bb11e Mon Sep 17 00:00:00 2001 From: Jeremy Stein Date: Wed, 2 Sep 2026 16:02:08 +0100 Subject: [PATCH 3/7] Although ":" works in compose env files, let's use "=" for consistency with the others --- config.EXAMPLE/lgtm.env.EXAMPLE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.EXAMPLE/lgtm.env.EXAMPLE b/config.EXAMPLE/lgtm.env.EXAMPLE index c59163f..cb6cd3d 100644 --- a/config.EXAMPLE/lgtm.env.EXAMPLE +++ b/config.EXAMPLE/lgtm.env.EXAMPLE @@ -5,4 +5,4 @@ # that the browser sees when accessing this site. Eg. the GAE hostname. # The URL path should not routinely be changed, as it needs to match # what the reverse proxy (nginx) is serving us on. -GF_SERVER_ROOT_URL: "%(protocol)s://localhost:7100/local-grafana/" +GF_SERVER_ROOT_URL="%(protocol)s://localhost:7100/local-grafana/" From b34d9395066f62e7d345a5e93b4d3eb63f0f4ad8 Mon Sep 17 00:00:00 2001 From: Jeremy Stein Date: Wed, 2 Sep 2026 16:02:39 +0100 Subject: [PATCH 4/7] Allow customizing of Grafana admin password --- config.EXAMPLE/lgtm.env.EXAMPLE | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config.EXAMPLE/lgtm.env.EXAMPLE b/config.EXAMPLE/lgtm.env.EXAMPLE index cb6cd3d..cacffef 100644 --- a/config.EXAMPLE/lgtm.env.EXAMPLE +++ b/config.EXAMPLE/lgtm.env.EXAMPLE @@ -6,3 +6,7 @@ # The URL path should not routinely be changed, as it needs to match # what the reverse proxy (nginx) is serving us on. GF_SERVER_ROOT_URL="%(protocol)s://localhost:7100/local-grafana/" + +GF_SECURITY_ADMIN_USER=admin +# please set to something secure +GF_SECURITY_ADMIN_PASSWORD=admin From 6f039ced42698f68a825e448d1a5b60f60dab04b Mon Sep 17 00:00:00 2001 From: Jeremy Stein Date: Wed, 2 Sep 2026 17:36:24 +0100 Subject: [PATCH 5/7] Update README.md Co-authored-by: Stephen Thompson --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f490d23..833acb7 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ mkdir waveform-export If not already done, create the docker network used for bridging the Emap portal with the local LGTM (grafana) instance: -`docker create network emap-portal-grafana` +`docker network create emap-portal-grafana` ##### run it! From 28072670ac2cfef4f30770bec77ab94b4c9f1489 Mon Sep 17 00:00:00 2001 From: Jeremy Stein Date: Wed, 2 Sep 2026 17:39:09 +0100 Subject: [PATCH 6/7] Remove some port forwards as they clash with existing services on the GAE and aren't necessary --- docker-compose.lgtm.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker-compose.lgtm.yml b/docker-compose.lgtm.yml index 1cc99f9..23e2e5c 100644 --- a/docker-compose.lgtm.yml +++ b/docker-compose.lgtm.yml @@ -37,10 +37,6 @@ services: - "127.0.0.1:4317:4317" # OTLP HTTP (OpenTelemetry default) - "127.0.0.1:4318:4318" - # Optional: query backends directly (Grafana already has them as datasources) - - "127.0.0.1:3200:3200" # Tempo - - "127.0.0.1:4040:4040" # Pyroscope - - "127.0.0.1:9090:9090" # Prometheus / Mimir volumes: # https://github.com/grafana/docker-otel-lgtm#persist-data-across-container-instantiation - lgtm-data:/data From 9b0e98ae597778ca68144225c57e4e84cea18853 Mon Sep 17 00:00:00 2001 From: Jeremy Stein Date: Wed, 2 Sep 2026 17:39:41 +0100 Subject: [PATCH 7/7] Add missing part of re-deploy process document --- docs/deployment.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/deployment.md b/docs/deployment.md index 3c34968..2a7f8ba 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -110,7 +110,31 @@ When the core service comes back up, it would continue to update the non-wavefor ### Waveform controller/exporter (ie. this repo) -You may need to delete files in the host directory `waveform-export`, which +#### Bring down all waveform containers and rebuild +``` +docker compose -f docker-compose.yml -f docker-compose.lgtm.yml --profile lgtm down +# check out desired code (example) +git checkout my_branch +git pull +# rebuild +docker compose -f docker-compose.yml -f docker-compose.lgtm.yml --profile lgtm build +``` + +#### Reconfigure + +Check all example config files in `config.EXAMPLE`. This shows what config files are expected to be present +in this version of the code. + +Copy any that don't exist in `../config` as per the instructions in each file. + +Then make sure that any new/deleted variables are adjusted as appropriate. + +You could run this command for each file: +`vimdiff config.EXAMPLE/exporter.env.EXAMPLE ../config/exporter.env` + +#### Reset data + +To force re-processing, you would need to delete files in the host directory `waveform-export`, which is bind mounted by the `waveform-controller` and `waveform-exporter` containers. Snakemake won't regenerate files if the timestamps of upstream