Add relab/prod alerting coverage

config/grafana/alerting/coverage.yaml

@@ -4,7 +4,7 @@
 # Fires on any series whose project/env pair has no rendered rule file: a project that
 # ships telemetry but was never bootstrapped has no ProjectTelemetrySilent rule.
 #
-# Covered right now: relab/staging (plus demo/demo, exempt: see below)
+# Covered right now: relab/prod, relab/staging (plus demo/demo, exempt: see below)
 
 apiVersion: 1
 
@@ -40,7 +40,7 @@ groups:
               # would leave ProjectTelemetrySilent firing forever after `just
               # demo-down`, so it is exempt as a pair; a real project named demo in
               # a real env is still caught.
-              expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) ({__name__=~"telemetry_.+_total", project="demo", env="demo"} or {__name__=~"telemetry_.+_total", project="relab",env="staging"}))
+              expr: count by (project, env) ({__name__=~"telemetry_.+_total", project!="", env!=""} unless on (project, env) ({__name__=~"telemetry_.+_total", project="demo", env="demo"} or {__name__=~"telemetry_.+_total", project="relab",env="prod"} or {__name__=~"telemetry_.+_total", project="relab",env="staging"}))
           - refId: FIRING
             datasourceUid: __expr__
             model:

config/grafana/alerting/project-relab-prod.yaml

@@ -0,0 +1,57 @@
+# Rendered by bootstrap.sh into config/grafana/alerting/. Do not edit the rendered
+# files by hand; edit this template so every project gets the fix.
+#
+# Nothing on a project host can detect its own absence, so this rule lives here.
+#
+# bootstrap.sh reads the marker below to regenerate coverage.yaml. It carries the pair,
+# not the filename, because both halves may contain a dash.
+# COVERS: relab prod
+
+apiVersion: 1
+
+groups:
+  - orgId: 1
+    name: project-relab-prod
+    folder: Stack alerts
+    interval: 1m
+    rules:
+      # absent() yields nothing while telemetry flows, so NoData is the HEALTHY state
+      # and must map to OK.
+      - uid: proj-silent-relab-prod
+        title: ProjectTelemetrySilent
+        condition: FIRING
+        for: 15m
+        noDataState: OK
+        execErrState: Error
+        isPaused: false
+        data:
+          - refId: QUERY
+            datasourceUid: prometheus
+            relativeTimeRange:
+              from: 3600
+              to: 0
+            model:
+              refId: QUERY
+              instant: true
+              editorMode: code
+              # The collector's ingest counters, as in coverage.yaml.tmpl. absent() loads
+              # every series its selector matches, once a minute per project, so it is
+              # bounded to a few per service rather than everything the project sends.
+              expr: absent({__name__=~"telemetry_.+_total", project="relab",env="prod"})
+          - refId: FIRING
+            datasourceUid: __expr__
+            model:
+              refId: FIRING
+              type: threshold
+              expression: QUERY
+              conditions:
+                - evaluator:
+                    type: gt
+                    params: [0]
+        labels:
+          severity: critical
+          project: relab
+          env: prod
+        annotations:
+          summary: "No telemetry from relab/prod in 15m"
+          description: "Host down, Docker down, the agent down, tunnel down, token rotated wrong, or the collector is rejecting this project's data. Nothing on the relab side can detect this on its own. Expect this ~20m after the last sample: absent() needs the series to go stale first."