ci: pin apt hosts through DNS over HTTPS when the runner can't resolve them

.github/workflows/ci.yml

@@ -31,6 +31,8 @@ jobs:
       # The landing page in site/, which the Pages workflow deploys from main.
       - run: pnpm --filter @bugpatrol/site typecheck
       - run: pnpm --filter @bugpatrol/site build
+      # Runner DNS sometimes loses the Azure apt mirror; see the script.
+      - run: bash scripts/apt-mirror-fallback.sh
       # The capture tests drive a real browser, because font loading and layout
       # only mean anything against a real rendering engine.
       - run: pnpm --filter @bugpatrol/capture exec playwright install --with-deps chromium

.github/workflows/determinism.yml

@@ -34,6 +34,8 @@ jobs:
       - run: pnpm install --frozen-lockfile
       - run: pnpm build
 
+      # Runner DNS sometimes loses the Azure apt mirror; see the script.
+      - run: bash scripts/apt-mirror-fallback.sh
       - name: Install the pinned browser build
         run: pnpm --filter @bugpatrol/capture exec playwright install --with-deps chromium
 
@@ -59,4 +61,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - run: docker build -t bugpatrol-runner:ci -f images/runner/Dockerfile .
+      - run: bash scripts/apt-mirror-fallback.sh
+      # Word splitting is intended: the variable is a list of --add-host flags.
+      - run: docker build $APT_MIRROR_ADD_HOSTS -t bugpatrol-runner:ci -f images/runner/Dockerfile .

.github/workflows/release.yml

@@ -39,6 +39,8 @@ jobs:
       - run: npm install -g npm@^11.5.1
       - run: pnpm install --frozen-lockfile
       - run: pnpm build
+      # Runner DNS sometimes loses the Azure apt mirror; see the script.
+      - run: bash scripts/apt-mirror-fallback.sh
       # Capture tests need Chromium's real font and layout engine.
       - run: pnpm --filter @bugpatrol/capture exec playwright install --with-deps chromium
       # The recording tests encode real video.

scripts/apt-mirror-fallback.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# GitHub's Ubuntu runners fetch apt packages from azure.archive.ubuntu.com, and
+# the Playwright image the runner image builds from names the same mirror. When
+# the runner's resolver loses the ubuntu.com names, every apt step retries for
+# minutes and fails the job though nothing in the commit is wrong. So any apt
+# host the runner cannot resolve is looked up through public DNS over HTTPS and
+# pinned in /etc/hosts for this job only.
+#
+# It changes nothing while the names resolve. `docker build` does not read the
+# host's /etc/hosts, so the pins are also exported as APT_MIRROR_ADD_HOSTS, a
+# list of --add-host flags.
+set -uo pipefail
+
+flags=()
+for host in azure.archive.ubuntu.com archive.ubuntu.com security.ubuntu.com; do
+  if getent hosts "$host" >/dev/null; then
+    continue
+  fi
+  ip=""
+  # DNS over HTTPS: the runner resolves these providers' own names fine.
+  for doh in https://cloudflare-dns.com/dns-query https://dns.google/resolve; do
+    ip=$(curl -fsS --max-time 10 -H "accept: application/dns-json" "$doh?name=$host&type=A" |
+      jq -r '[.Answer[]? | select(.type == 1) | .data][0] // empty')
+    [ -n "$ip" ] && break
+  done
+  if [ -z "$ip" ]; then
+    echo "::warning::$host does not resolve here or through public DNS; apt steps that need it will fail."
+    continue
+  fi
+  echo "::warning::$host does not resolve on this runner; pinned to $ip from public DNS."
+  echo "$ip $host" | sudo tee -a /etc/hosts >/dev/null
+  flags+=("--add-host=$host:$ip")
+done
+
+if [ -n "${GITHUB_ENV:-}" ]; then
+  echo "APT_MIRROR_ADD_HOSTS=${flags[*]}" >>"$GITHUB_ENV"
+fi