From c4a69c589b328ca1d5645307e3eb3146b1d77e57 Mon Sep 17 00:00:00 2001
From: Simon van Lierde <simonvanlierde@gmail.com>
Date: Tue, 8 Sep 2026 14:06:18 +0000
Subject: [PATCH] feat(alerting): add relab/prod project coverage

- add project-relab-prod.yaml silence rule
- exempt relab/prod from the uncovered-project rule in coverage.yaml
---
 config/grafana/alerting/coverage.yaml         |  4 +-
 .../grafana/alerting/project-relab-prod.yaml  | 57 +++++++++++++++++++
 2 files changed, 59 insertions(+), 2 deletions(-)
 create mode 100644 config/grafana/alerting/project-relab-prod.yaml

diff --git a/config/grafana/alerting/coverage.yaml b/config/grafana/alerting/coverage.yaml
index 58b2cb1..82109f5 100644
--- a/config/grafana/alerting/coverage.yaml
+++ b/config/grafana/alerting/coverage.yaml
@@ -4,7 +4,7 @@
 # Fires on any series whose project/env pair has no rendered rule file: a project that
 # ships telemetry but was never bootstrapped has no ProjectTelemetrySilent rule.
 #
-# Covered right now: relab/staging (plus demo/demo, exempt: see below)
+# Covered right now: relab/prod, relab/staging (plus demo/demo, exempt: see below)
 
 apiVersion: 1
 
@@ -40,7 +40,7 @@ groups:
               # would leave ProjectTelemetrySilent firing forever after `just
               # demo-down`, so it is exempt as a pair; a real project named demo in
               # a real env is still caught.
-              expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) ({__name__=~"telemetry_.+_total", project="demo", env="demo"} or {__name__=~"telemetry_.+_total", project="relab",env="staging"}))
+              expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) ({__name__=~"telemetry_.+_total", project="demo", env="demo"} or {__name__=~"telemetry_.+_total", project="relab",env="prod"} or {__name__=~"telemetry_.+_total", project="relab",env="staging"}))
           - refId: FIRING
             datasourceUid: __expr__
             model:
diff --git a/config/grafana/alerting/project-relab-prod.yaml b/config/grafana/alerting/project-relab-prod.yaml
new file mode 100644
index 0000000..3480b3b
--- /dev/null
+++ b/config/grafana/alerting/project-relab-prod.yaml
@@ -0,0 +1,57 @@
+# Rendered by bootstrap.sh into config/grafana/alerting/. Do not edit the rendered
+# files by hand; edit this template so every project gets the fix.
+#
+# Nothing on a project host can detect its own absence, so this rule lives here.
+#
+# bootstrap.sh reads the marker below to regenerate coverage.yaml. It carries the pair,
+# not the filename, because both halves may contain a dash.
+# COVERS: relab prod
+
+apiVersion: 1
+
+groups:
+  - orgId: 1
+    name: project-relab-prod
+    folder: Stack alerts
+    interval: 1m
+    rules:
+      # absent() yields nothing while telemetry flows, so NoData is the HEALTHY state
+      # and must map to OK.
+      - uid: proj-silent-relab-prod
+        title: ProjectTelemetrySilent
+        condition: FIRING
+        for: 15m
+        noDataState: OK
+        execErrState: Error
+        isPaused: false
+        data:
+          - refId: QUERY
+            datasourceUid: prometheus
+            relativeTimeRange:
+              from: 3600
+              to: 0
+            model:
+              refId: QUERY
+              instant: true
+              editorMode: code
+              # The collector's ingest counters, as in coverage.yaml.tmpl. absent() loads
+              # every series its selector matches, once a minute per project, so it is
+              # bounded to a few per service rather than everything the project sends.
+              expr: absent({__name__=~"telemetry_.+_total", project="relab",env="prod"})
+          - refId: FIRING
+            datasourceUid: __expr__
+            model:
+              refId: FIRING
+              type: threshold
+              expression: QUERY
+              conditions:
+                - evaluator:
+                    type: gt
+                    params: [0]
+        labels:
+          severity: critical
+          project: relab
+          env: prod
+        annotations:
+          summary: "No telemetry from relab/prod in 15m"
+          description: "Host down, Docker down, the agent down, tunnel down, token rotated wrong, or the collector is rejecting this project's data. Nothing on the relab side can detect this on its own. Expect this ~20m after the last sample: absent() needs the series to go stale first."