ci(ios): stop noise updates of the doc site screenshots

.github/workflows/ios-actions.yml

@@ -74,6 +74,10 @@ jobs:
           UDID=$("$GITHUB_WORKSPACE/scripts/ci/pick-ios-simulator.sh")
           echo "DETOX_DEVICE_UDID=$UDID" >> "$GITHUB_ENV"
           xcrun simctl boot "$UDID"
+        # The doc site screenshots change size and rendering with the device and iOS version
+        env:
+          IOS_SIMULATOR_DEVICE: ${{ inputs.E2E_MODE == 'doc-screenshots' && 'iPhone 17' || '' }}
+          IOS_SIMULATOR_RUNTIME: ${{ inputs.E2E_MODE == 'doc-screenshots' && '26.5' || '' }}
 
       # Fork PRs get no secrets; fall back to the public rotated CI token (see rnmapbox/ci-config)
       - name: Access Token

.github/workflows/ios-nightly-percy.yml

@@ -100,6 +100,9 @@ jobs:
     needs: [check, ios]
     if: always() && needs.check.outputs.run == 'true'
     runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
     env:
       BRANCH: auto/example-screenshots
     steps:
@@ -122,6 +125,23 @@ jobs:
           ssh-key: ${{ secrets.MAPS_DOCS_DEPLOY_KEY }}
           path: maps-docs
 
+      # Skipped nightlies (no new commits) succeed without a capture, so look further back
+      - name: Download the previous capture
+        if: steps.download.outcome == 'success'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          for RUN in $(gh run list --repo "$GITHUB_REPOSITORY" --workflow ios-nightly-percy.yml \
+            --branch main --status success --limit 20 --json databaseId --jq '.[].databaseId'); do
+            if gh run download "$RUN" --repo "$GITHUB_REPOSITORY" --name doc-screenshots --dir previous; then
+              echo "Comparing with the capture of $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$RUN"
+              echo "PREVIOUS_CAPTURE=previous" >> "$GITHUB_ENV"
+              exit 0
+            fi
+            rm -rf previous
+          done
+          echo "No previous capture found"
+
       - uses: actions/setup-node@v4
         if: steps.download.outcome == 'success'
         with:
@@ -132,7 +152,8 @@ jobs:
         run: |
           npm install --no-save --prefix "$RUNNER_TEMP/deps" pngjs@7.0.0
           NODE_PATH="$RUNNER_TEMP/deps/node_modules" node maps/scripts/ci/update-doc-screenshots.js \
-            captured maps-docs/examples-screenshots maps/docs/examples.json "$RUNNER_TEMP/summary.md"
+            captured maps-docs/examples-screenshots maps/docs/examples.json "$RUNNER_TEMP/summary.md" \
+            ${PREVIOUS_CAPTURE:+"$PREVIOUS_CAPTURE"}
         env:
           REFRESH_UNSTABLE: ${{ inputs.refresh_unstable }}
 

scripts/ci/pick-ios-simulator.sh

@@ -1,10 +1,12 @@
 #!/bin/sh
 # Prints the UDID of a plain iPhone simulator (e.g. "iPhone 16", no Pro/Plus/SE/e)
 # on the newest installed iOS runtime, creating one if the image has none.
+# IOS_SIMULATOR_DEVICE and IOS_SIMULATOR_RUNTIME (e.g. "iPhone 17" and "26.5") pin both instead,
+# failing if the runner image lacks them.
 set -eu
 
 PICK=$(python3 -c '
-import json, re, subprocess
+import json, os, re, subprocess, sys
 
 def simctl(*args):
     return json.loads(subprocess.check_output(["xcrun", "simctl", "list", *args, "-j"]))
@@ -15,14 +17,32 @@ def plain(name):
 def model(name):
     return int(name.split()[1])
 
+def version(runtime):
+    return [int(p) for p in runtime["version"].split(".")]
+
+pinned_device = os.environ.get("IOS_SIMULATOR_DEVICE")
+pinned_runtime = os.environ.get("IOS_SIMULATOR_RUNTIME")
 runtimes = [r for r in simctl("runtimes", "available")["runtimes"] if r["platform"] == "iOS"]
-runtime = max(runtimes, key=lambda r: [int(p) for p in r["version"].split(".")])
-devices = [d for d in simctl("devices", "available")["devices"].get(runtime["identifier"], []) if plain(d["name"])]
+if pinned_runtime:
+    matching = [r for r in runtimes if (r["version"] + ".").startswith(pinned_runtime + ".")]
+    if not matching:
+        available = ", ".join(r["version"] for r in runtimes)
+        sys.exit(f"The iOS {pinned_runtime} simulator runtime is not on this runner image, it has: {available}. Update IOS_SIMULATOR_RUNTIME.")
+    runtimes = matching
+runtime = max(runtimes, key=version)
+types = [t for t in runtime["supportedDeviceTypes"] if (t["name"] == pinned_device if pinned_device else plain(t["name"]))]
+if not types:
+    available = ", ".join(t["name"] for t in runtime["supportedDeviceTypes"] if t["name"].startswith("iPhone"))
+    wanted = pinned_device or "plain iPhone"
+    found = runtime["version"]
+    sys.exit(f"The iOS {found} simulator runtime has no {wanted}, it has: {available}. Update IOS_SIMULATOR_DEVICE.")
+device_type = types[0] if pinned_device else max(types, key=lambda t: model(t["name"]))
+devices = [d for d in simctl("devices", "available")["devices"].get(runtime["identifier"], [])
+           if d["deviceTypeIdentifier"] == device_type["identifier"]]
 if devices:
-    print(max(devices, key=lambda d: model(d["name"]))["udid"])
+    print(devices[0]["udid"])
 else:
-    types = [t for t in runtime["supportedDeviceTypes"] if plain(t["name"])]
-    print("create", max(types, key=lambda t: model(t["name"]))["identifier"], runtime["identifier"])
+    print("create", device_type["identifier"], runtime["identifier"])
 ')
 
 case "$PICK" in

scripts/ci/update-doc-screenshots.js

@@ -1,9 +1,12 @@
 /**
- * @file node update-doc-screenshots.js <captured-dir> <docs-screenshots-dir> <examples.json> <summary.md>
+ * @file node update-doc-screenshots.js <captured-dir> <docs-screenshots-dir> <examples.json> <summary.md> [<previous-captured-dir>]
  * Copies the captured example screenshots that differ meaningfully into the doc site's
  * examples-screenshots folder, drops screenshots of removed examples, keeps screenshots.json
  * in sync and writes a markdown summary of what changed (empty if nothing did).
  *
+ * With the previous night's capture, a changed screenshot is only copied if it matches that capture
+ * too, so one-off rendering and loading glitches never reach the doc site.
+ *
  * excludeFromPercy examples differ by up to ~80% between runs (animations, timers), so they are only
  * replaced when new, when the screenshot size changed or with REFRESH_UNSTABLE=true.
  */
@@ -12,13 +15,13 @@ const fs = require('fs');
 const path = require('path');
 const { PNG } = require('pngjs');
 
-const [capturedDir, docsDir, examplesJSONPath, summaryPath] =
+const [capturedDir, docsDir, examplesJSONPath, summaryPath, previousDir] =
   process.argv.slice(2);
 
 // The status bar can't be fully frozen (the location arrow comes and goes)
 const IGNORE_TOP_FRACTION = 0.075;
-const CHANNEL_TOLERANCE = 16;
-const THRESHOLD = 0.005;
+const CHANNEL_TOLERANCE = 32;
+const THRESHOLD = 0.0025;
 const refreshUnstable = process.env.REFRESH_UNSTABLE === 'true';
 
 function diffFraction(aPath, bPath) {
@@ -48,6 +51,7 @@ const changed = [];
 const added = [];
 const removed = [];
 const missing = [];
+const unstableChanges = [];
 const expected = new Set();
 
 for (const { groupName, examples: groupExamples } of examples) {
@@ -68,16 +72,34 @@ for (const { groupName, examples: groupExamples } of examples) {
     if (entry && fs.existsSync(dest)) {
       const fraction = diffFraction(captured, dest);
       const unstable = metadata.excludeFromPercy && !refreshUnstable;
-      if (fraction === null) {
-        changed.push(`${key} (size changed)`);
-      } else {
-        const percent = `${(fraction * 100).toFixed(1)}%`;
-        console.log(`${key}: ${percent}${unstable ? ' (unstable, kept)' : ''}`);
+      const change =
+        fraction === null
+          ? 'size changed'
+          : `${(fraction * 100).toFixed(2)}% of the pixels`;
+      if (fraction !== null) {
+        console.log(`${key}: ${change}${unstable ? ' (unstable, kept)' : ''}`);
         if (unstable || fraction <= THRESHOLD) {
           continue;
         }
-        changed.push(`${key} (${percent} of the pixels)`);
       }
+      if (previousDir && !metadata.excludeFromPercy) {
+        const previous = path.join(previousDir, groupName, `${name}.png`);
+        const sinceLastNight = fs.existsSync(previous)
+          ? diffFraction(captured, previous)
+          : null;
+        if (sinceLastNight === null || sinceLastNight > THRESHOLD) {
+          const reason =
+            sinceLastNight === null
+              ? 'no matching capture last night'
+              : `${(sinceLastNight * 100).toFixed(
+                  2,
+                )}% off last night's capture`;
+          console.log(`${key}: ${reason}, not updated`);
+          unstableChanges.push(`${key} (${change}, ${reason})`);
+          continue;
+        }
+      }
+      changed.push(`${key} (${change})`);
     } else {
       added.push(key);
     }
@@ -117,6 +139,12 @@ for (const file of fs.readdirSync(docsDir, { recursive: true })) {
   }
 }
 
+if (!previousDir) {
+  console.log(
+    "No previous capture, changed screenshots are updated without checking that they're stable",
+  );
+}
+
 if (missing.length > 0) {
   console.log(`Not captured, kept as is: ${missing.join(', ')}`);
 }
@@ -126,6 +154,9 @@ const sections = [
   ['New', added],
   ['Removed', removed],
 ].filter(([, keys]) => keys.length > 0);
+if (sections.length > 0 && unstableChanges.length > 0) {
+  sections.push(['Unstable, not updated', unstableChanges]);
+}
 
 if (sections.length === 0) {
   fs.writeFileSync(summaryPath, '');