diff --git a/.github/workflows/ios-actions.yml b/.github/workflows/ios-actions.yml index 840dd015a..28c7b5c35 100644 --- a/.github/workflows/ios-actions.yml +++ b/.github/workflows/ios-actions.yml @@ -74,6 +74,10 @@ jobs: UDID=$("$GITHUB_WORKSPACE/scripts/ci/pick-ios-simulator.sh") echo "DETOX_DEVICE_UDID=$UDID" >> "$GITHUB_ENV" xcrun simctl boot "$UDID" + # The doc site screenshots change size and rendering with the device and iOS version + env: + IOS_SIMULATOR_DEVICE: ${{ inputs.E2E_MODE == 'doc-screenshots' && 'iPhone 17' || '' }} + IOS_SIMULATOR_RUNTIME: ${{ inputs.E2E_MODE == 'doc-screenshots' && '26.5' || '' }} # Fork PRs get no secrets; fall back to the public rotated CI token (see rnmapbox/ci-config) - name: Access Token diff --git a/.github/workflows/ios-nightly-percy.yml b/.github/workflows/ios-nightly-percy.yml index 75c24e6a9..b2ed71997 100644 --- a/.github/workflows/ios-nightly-percy.yml +++ b/.github/workflows/ios-nightly-percy.yml @@ -100,6 +100,9 @@ jobs: needs: [check, ios] if: always() && needs.check.outputs.run == 'true' runs-on: ubuntu-latest + permissions: + actions: read + contents: read env: BRANCH: auto/example-screenshots steps: @@ -122,6 +125,23 @@ jobs: ssh-key: ${{ secrets.MAPS_DOCS_DEPLOY_KEY }} path: maps-docs + # Skipped nightlies (no new commits) succeed without a capture, so look further back + - name: Download the previous capture + if: steps.download.outcome == 'success' + env: + GH_TOKEN: ${{ github.token }} + run: | + for RUN in $(gh run list --repo "$GITHUB_REPOSITORY" --workflow ios-nightly-percy.yml \ + --branch main --status success --limit 20 --json databaseId --jq '.[].databaseId'); do + if gh run download "$RUN" --repo "$GITHUB_REPOSITORY" --name doc-screenshots --dir previous; then + echo "Comparing with the capture of $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$RUN" + echo "PREVIOUS_CAPTURE=previous" >> "$GITHUB_ENV" + exit 0 + fi + rm -rf previous + done + echo "No previous capture found" + - uses: actions/setup-node@v4 if: steps.download.outcome == 'success' with: @@ -132,7 +152,8 @@ jobs: run: | npm install --no-save --prefix "$RUNNER_TEMP/deps" pngjs@7.0.0 NODE_PATH="$RUNNER_TEMP/deps/node_modules" node maps/scripts/ci/update-doc-screenshots.js \ - captured maps-docs/examples-screenshots maps/docs/examples.json "$RUNNER_TEMP/summary.md" + captured maps-docs/examples-screenshots maps/docs/examples.json "$RUNNER_TEMP/summary.md" \ + ${PREVIOUS_CAPTURE:+"$PREVIOUS_CAPTURE"} env: REFRESH_UNSTABLE: ${{ inputs.refresh_unstable }} diff --git a/scripts/ci/pick-ios-simulator.sh b/scripts/ci/pick-ios-simulator.sh index c85748250..9367e2a89 100755 --- a/scripts/ci/pick-ios-simulator.sh +++ b/scripts/ci/pick-ios-simulator.sh @@ -1,10 +1,12 @@ #!/bin/sh # Prints the UDID of a plain iPhone simulator (e.g. "iPhone 16", no Pro/Plus/SE/e) # on the newest installed iOS runtime, creating one if the image has none. +# IOS_SIMULATOR_DEVICE and IOS_SIMULATOR_RUNTIME (e.g. "iPhone 17" and "26.5") pin both instead, +# failing if the runner image lacks them. set -eu PICK=$(python3 -c ' -import json, re, subprocess +import json, os, re, subprocess, sys def simctl(*args): return json.loads(subprocess.check_output(["xcrun", "simctl", "list", *args, "-j"])) @@ -15,14 +17,32 @@ def plain(name): def model(name): return int(name.split()[1]) +def version(runtime): + return [int(p) for p in runtime["version"].split(".")] + +pinned_device = os.environ.get("IOS_SIMULATOR_DEVICE") +pinned_runtime = os.environ.get("IOS_SIMULATOR_RUNTIME") runtimes = [r for r in simctl("runtimes", "available")["runtimes"] if r["platform"] == "iOS"] -runtime = max(runtimes, key=lambda r: [int(p) for p in r["version"].split(".")]) -devices = [d for d in simctl("devices", "available")["devices"].get(runtime["identifier"], []) if plain(d["name"])] +if pinned_runtime: + matching = [r for r in runtimes if (r["version"] + ".").startswith(pinned_runtime + ".")] + if not matching: + available = ", ".join(r["version"] for r in runtimes) + sys.exit(f"The iOS {pinned_runtime} simulator runtime is not on this runner image, it has: {available}. Update IOS_SIMULATOR_RUNTIME.") + runtimes = matching +runtime = max(runtimes, key=version) +types = [t for t in runtime["supportedDeviceTypes"] if (t["name"] == pinned_device if pinned_device else plain(t["name"]))] +if not types: + available = ", ".join(t["name"] for t in runtime["supportedDeviceTypes"] if t["name"].startswith("iPhone")) + wanted = pinned_device or "plain iPhone" + found = runtime["version"] + sys.exit(f"The iOS {found} simulator runtime has no {wanted}, it has: {available}. Update IOS_SIMULATOR_DEVICE.") +device_type = types[0] if pinned_device else max(types, key=lambda t: model(t["name"])) +devices = [d for d in simctl("devices", "available")["devices"].get(runtime["identifier"], []) + if d["deviceTypeIdentifier"] == device_type["identifier"]] if devices: - print(max(devices, key=lambda d: model(d["name"]))["udid"]) + print(devices[0]["udid"]) else: - types = [t for t in runtime["supportedDeviceTypes"] if plain(t["name"])] - print("create", max(types, key=lambda t: model(t["name"]))["identifier"], runtime["identifier"]) + print("create", device_type["identifier"], runtime["identifier"]) ') case "$PICK" in diff --git a/scripts/ci/update-doc-screenshots.js b/scripts/ci/update-doc-screenshots.js index 433cf21fd..c9e3f083e 100644 --- a/scripts/ci/update-doc-screenshots.js +++ b/scripts/ci/update-doc-screenshots.js @@ -1,9 +1,12 @@ /** - * @file node update-doc-screenshots.js + * @file node update-doc-screenshots.js [] * Copies the captured example screenshots that differ meaningfully into the doc site's * examples-screenshots folder, drops screenshots of removed examples, keeps screenshots.json * in sync and writes a markdown summary of what changed (empty if nothing did). * + * With the previous night's capture, a changed screenshot is only copied if it matches that capture + * too, so one-off rendering and loading glitches never reach the doc site. + * * excludeFromPercy examples differ by up to ~80% between runs (animations, timers), so they are only * replaced when new, when the screenshot size changed or with REFRESH_UNSTABLE=true. */ @@ -12,13 +15,13 @@ const fs = require('fs'); const path = require('path'); const { PNG } = require('pngjs'); -const [capturedDir, docsDir, examplesJSONPath, summaryPath] = +const [capturedDir, docsDir, examplesJSONPath, summaryPath, previousDir] = process.argv.slice(2); // The status bar can't be fully frozen (the location arrow comes and goes) const IGNORE_TOP_FRACTION = 0.075; -const CHANNEL_TOLERANCE = 16; -const THRESHOLD = 0.005; +const CHANNEL_TOLERANCE = 32; +const THRESHOLD = 0.0025; const refreshUnstable = process.env.REFRESH_UNSTABLE === 'true'; function diffFraction(aPath, bPath) { @@ -48,6 +51,7 @@ const changed = []; const added = []; const removed = []; const missing = []; +const unstableChanges = []; const expected = new Set(); for (const { groupName, examples: groupExamples } of examples) { @@ -68,16 +72,34 @@ for (const { groupName, examples: groupExamples } of examples) { if (entry && fs.existsSync(dest)) { const fraction = diffFraction(captured, dest); const unstable = metadata.excludeFromPercy && !refreshUnstable; - if (fraction === null) { - changed.push(`${key} (size changed)`); - } else { - const percent = `${(fraction * 100).toFixed(1)}%`; - console.log(`${key}: ${percent}${unstable ? ' (unstable, kept)' : ''}`); + const change = + fraction === null + ? 'size changed' + : `${(fraction * 100).toFixed(2)}% of the pixels`; + if (fraction !== null) { + console.log(`${key}: ${change}${unstable ? ' (unstable, kept)' : ''}`); if (unstable || fraction <= THRESHOLD) { continue; } - changed.push(`${key} (${percent} of the pixels)`); } + if (previousDir && !metadata.excludeFromPercy) { + const previous = path.join(previousDir, groupName, `${name}.png`); + const sinceLastNight = fs.existsSync(previous) + ? diffFraction(captured, previous) + : null; + if (sinceLastNight === null || sinceLastNight > THRESHOLD) { + const reason = + sinceLastNight === null + ? 'no matching capture last night' + : `${(sinceLastNight * 100).toFixed( + 2, + )}% off last night's capture`; + console.log(`${key}: ${reason}, not updated`); + unstableChanges.push(`${key} (${change}, ${reason})`); + continue; + } + } + changed.push(`${key} (${change})`); } else { added.push(key); } @@ -117,6 +139,12 @@ for (const file of fs.readdirSync(docsDir, { recursive: true })) { } } +if (!previousDir) { + console.log( + "No previous capture, changed screenshots are updated without checking that they're stable", + ); +} + if (missing.length > 0) { console.log(`Not captured, kept as is: ${missing.join(', ')}`); } @@ -126,6 +154,9 @@ const sections = [ ['New', added], ['Removed', removed], ].filter(([, keys]) => keys.length > 0); +if (sections.length > 0 && unstableChanges.length > 0) { + sections.push(['Unstable, not updated', unstableChanges]); +} if (sections.length === 0) { fs.writeFileSync(summaryPath, '');