test(webkit): record API observations and add a macOS pilot

.github/workflows/webkit.yml

@@ -0,0 +1,48 @@
+name: WebKit pilot
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - develop
+      - 'codex/**'
+      - 'feature/**'
+
+permissions:
+  contents: read
+
+# Pilot coverage is independent of reusable CI and release.yml: failures remain visible
+# without becoming a dependency of the release/publish job.
+jobs:
+  webkit:
+    name: System WebKit (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-15, macos-26]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+      - name: Use Node.js
+        uses: actions/setup-node@v7
+        with:
+          node-version: 24
+          cache: npm
+      - name: Install dependencies
+        run: npm ci
+      - name: Check generated sources
+        run: npm run generate:check
+      - name: Build
+        run: npm run build
+      - name: Run system WebKit contracts and record observations
+        run: npm run test:webkit -- --report-dir .output/webkit/${{ matrix.os }}
+      - name: Upload observations even when a scenario fails
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: webkit-${{ matrix.os }}-${{ github.run_attempt }}
+          path: .output/webkit/${{ matrix.os }}
+          if-no-files-found: warn

docs/safari.md

@@ -111,3 +111,23 @@ including `webRequest`. A namespace absent from a differently configured host mu
 Safari API without further investigation. Automatically recorded observations and human-reviewed findings are
 separate concerns; a callable inventory is not a runtime conformance result for every wrapper.
 
+## Saved observations and pilot CI
+
+`npm run test:webkit` writes `report.json` and `report.md` under the gitignored `.output/webkit/` directory, including
+failures after runner startup. Override the directory with `--report-dir <path>`. Raw reports are not committed.
+
+The report records system WebKit/macOS versions, the source revision and dirty flag, requested manifest permissions
+and host permissions, **actually granted** permissions/origins, and scenario results. The inventory resolves native
+method references from the production TypeScript source and enumerates callable properties of each exposed namespace,
+including non-enumerable and inherited properties. Wrapper references are source observations, not proof of execution.
+
+Each method has one of three states: `namespace-unavailable`, `method-unavailable` (namespace present), or
+`method-present`. Namespace absence is not attributed to Safari itself: permissions, manifest, host and build flags
+can all affect it. The fixture requests and grants `webRequest`, so its namespace is not withheld by a missing request.
+Human findings about adapted incorrect calls, source-only checks and remaining gaps are maintained separately in
+`tests/webkit/review.json`; they are never inferred from presence alone.
+
+The separate **WebKit pilot** workflow runs on `macos-15` and `macos-26`, uploads the report even after failure, and
+prints the actual system versions. It is not called by reusable CI or `release.yml`, so it is not a release dependency.
+GitHub-hosted execution must be assessed from its own run; the local macOS 27.2 result does not establish runner
+stability or the results of older system WebKit builds.

tests/webkit/observations.test.ts

@@ -0,0 +1,38 @@
+import {expect, test} from "@jest/globals";
+
+import {observeMethods} from "./observations";
+
+test("inventory distinguishes missing namespaces, missing methods and callable methods without invoking them", () => {
+    let calls = 0;
+
+    const inherited = {inherited() {
+        calls++;
+    }};
+
+    const tabs = Object.create(inherited);
+
+    Object.defineProperty(tabs, "query", {value() {
+        calls++;
+    }, enumerable: false});
+
+    const observations = observeMethods({tabs}, {tabs: {discard: []}, identity: {launchWebAuthFlow: []}});
+    expect(observations.find(row => row.namespace === "identity")?.methods[0].state).toBe("namespace-unavailable");
+    const methods = observations.find(row => row.namespace === "tabs")!.methods;
+
+    expect(methods.map(row => [row.name, row.state])).toEqual([
+        ["discard", "method-unavailable"], ["inherited", "method-present"], ["query", "method-present"],
+    ]);
+
+    expect(calls).toBe(0);
+});
+
+test("inaccessible namespaces retain their error without claiming browser non-support", () => {
+    const api = Object.defineProperty({}, "cookies", {get() {
+        throw new Error("Permission withheld");
+    }});
+
+    const [row] = observeMethods(api, {cookies: {get: []}});
+    expect(row.state).toBe("namespace-unavailable");
+    expect(row.accessError).toContain("Permission withheld");
+    expect(row.methods[0].state).toBe("namespace-unavailable");
+});

tests/webkit/observations.ts

@@ -0,0 +1,95 @@
+export interface WrapperReference {
+    file: string;
+    line: number;
+    wrapper: string;
+}
+
+export type WrapperInventory = Record<string, Record<string, WrapperReference[]>>;
+export type MethodState = "namespace-unavailable" | "method-unavailable" | "method-present";
+
+function keys(value: object): string[] {
+    const names = new Set<string>();
+
+    for (let current: object | null = value; current && current !== Object.prototype; current = Object.getPrototypeOf(current)) {
+        for (const name of Object.getOwnPropertyNames(current)) {
+            if (name !== "constructor") {
+                names.add(name);
+            }
+        }
+    }
+
+    return [...names].sort();
+}
+
+/** Presence observations only: no method is called and no absence is attributed to the browser. */
+export function observeMethods(api: object, wrappers: WrapperInventory) {
+    const namespaces = new Set(Object.keys(wrappers));
+
+    for (const name of keys(api)) {
+        try {
+            const value = Reflect.get(api, name);
+
+            if (value && typeof value === "object") {
+                namespaces.add(name);
+            }
+        } catch {
+            namespaces.add(name);
+        }
+    }
+
+    return [...namespaces].sort().map(namespace => {
+        let native: object | undefined;
+        let accessError: string | undefined;
+
+        try {
+            const value = Reflect.get(api, namespace);
+
+            if (value && (typeof value === "object" || typeof value === "function")) {
+                native = value;
+            }
+        } catch (error) {
+            accessError = String(error);
+        }
+
+        const methods = new Set(Object.keys(wrappers[namespace] ?? {}));
+
+        if (native) {
+            for (const name of keys(native)) {
+                try {
+                    if (typeof Reflect.get(native, name) === "function") {
+                        methods.add(name);
+                    }
+                } catch {
+                    // A guarded wrapper reference below still records an inaccessible member.
+                }
+            }
+        }
+
+        return {
+            namespace,
+            state: native ? "namespace-present" : "namespace-unavailable",
+            ...(accessError ? {accessError} : {}),
+            methods: [...methods].sort().map(method => {
+                let valueType = "undefined";
+                let methodError: string | undefined;
+
+                if (native) {
+                    try {
+                        valueType = typeof Reflect.get(native, method);
+                    } catch (error) {
+                        methodError = String(error);
+                    }
+                }
+
+                const state: MethodState = !native ? "namespace-unavailable"
+                    : valueType === "function" ? "method-present" : "method-unavailable";
+
+                return {
+                    name: method, state, valueType,
+                    wrapperReferences: wrappers[namespace]?.[method] ?? [],
+                    ...(methodError ? {accessError: methodError} : {}),
+                };
+            }),
+        };
+    });
+}

tests/webkit/probe.ts

@@ -1,11 +1,14 @@
 import {
     browser, clearAlarm, clearAllAlarm, createAlarm, createAlarmIfNotExists, disableAction,
-    enableAction, getAlarm, getAllAlarm, getBadgeText, getCurrentTab, getDnrDynamicRules,
+    enableAction, getAlarm, getAllAlarm, getAllPermissions, getBadgeText, getCurrentTab, getDnrDynamicRules,
     getRegisteredContentScripts, isActionEnabled, isAvailableAction, isAvailableCookies,
     isAvailableDnr, isAvailableI18n, isAvailablePermissions, isAvailableRuntime, isAvailableTabs,
     isAvailableWebRequest, queryTabs, removeCookie, setBadgeText,
 } from "../../dist/index.js";
 import {assert, rejects} from "../browser/extension/assert";
+import {observeMethods, type WrapperInventory} from "./observations";
+
+declare const __WK_WRAPPER_INVENTORY__: WrapperInventory;
 
 const results: {id: string; status: string; error?: string}[] = [];
 const native = browser();
@@ -87,7 +90,12 @@ async function main() {
 
     const host = globalThis as typeof globalThis & {webkit: {messageHandlers: {result: {postMessage(value: unknown): void}}}};
 
-    host.webkit.messageHandlers.result.postMessage({results});
+    host.webkit.messageHandlers.result.postMessage({
+        results,
+        grantedPermissions: await getAllPermissions(),
+        aliasesShareObject: native === globalThis.chrome,
+        observations: observeMethods(native, __WK_WRAPPER_INVENTORY__),
+    });
 }
 
 void main();

tests/webkit/review.json

@@ -0,0 +1,22 @@
+{
+  "kind": "human-reviewed-findings",
+  "reviewedOn": "2026-11-05",
+  "incorrectCallsAdapted": [
+    "alarms.create: legacy WebKit rejects a callback; modern WebKit returns a Promise",
+    "action.isEnabled: WebKit needs an object; Firefox's Promise interface rejects a callback",
+    "alarms.get and tabs.getCurrent: normalize null to declared undefined",
+    "alarms.clear: prefer native boolean and use WebKit pre-read when no result is reported",
+    "alarms.clearAll: use the nonempty pre-read collection in every browser; ignore inconsistent native results",
+    "cookies.remove: preserve details and normalize absent native results to null"
+  ],
+  "sourceOnly": [
+    "Presence of a callable property does not validate argument options, result fields or errors",
+    "Newer Promise-returning WebKit alarms.create is covered by unit tests and upstream source review"
+  ],
+  "remainingGaps": [
+    "Successful execution and errors for every remaining public wrapper",
+    "Actual delayed alarm firing and event payloads",
+    "Safari application UI, user gestures, popups and iOS",
+    "Host/build flags controlling optional APIs and versions not executed by this run"
+  ]
+}

tests/webkit/run.mjs

@@ -1,55 +1,112 @@
 import assert from "node:assert/strict";
 import {execFile} from "node:child_process";
-import {mkdir, mkdtemp, rm, writeFile} from "node:fs/promises";
+import {mkdir, mkdtemp, readFile, rm, writeFile} from "node:fs/promises";
 import {tmpdir} from "node:os";
 import {dirname, join, resolve} from "node:path";
 import {fileURLToPath} from "node:url";
-import {promisify} from "node:util";
+import {parseArgs, promisify} from "node:util";
 
 import {build} from "tsup";
 
+import {collectWrapperMethods} from "./wrappers.mjs";
+
 const run = promisify(execFile);
 const directory = dirname(fileURLToPath(import.meta.url));
 const root = resolve(directory, "../..");
+const {values} = parseArgs({options: {"report-dir": {type: "string"}}});
+const reportDirectory = resolve(root, values["report-dir"] ?? ".output/webkit");
 assert.equal(process.platform, "darwin", "System WebKit tests require macOS 15.4+ and the Xcode Command Line Tools");
 const temporary = await mkdtemp(join(tmpdir(), "addon-browser-webkit-"));
 
+const manifest = {
+    manifest_version: 3, name: "Isolated WebKit contract tests", version: "0.0.1", action: {},
+    permissions: ["alarms", "tabs", "cookies", "contextMenus", "scripting", "declarativeNetRequest", "webRequest"],
+    host_permissions: ["https://example.test/*"],
+};
+
+const report = {
+    schemaVersion: 1,
+    capturedAt: new Date().toISOString(),
+    status: "failed",
+    host: "Standalone WKWebExtensionController; nonpersistent data stores; minimal window/tab delegates",
+    requestedManifest: manifest,
+    humanReview: JSON.parse(await readFile(join(directory, "review.json"), "utf8")),
+};
+
 try {
+    const [{stdout: macOS}, {stdout: macOSBuild}, {stdout: webKit}, {stdout: revision}, {stdout: dirty}] = await Promise.all([
+        run("sw_vers", ["-productVersion"]), run("sw_vers", ["-buildVersion"]),
+        run("/usr/libexec/PlistBuddy", ["-c", "Print :CFBundleVersion", "/System/Library/Frameworks/WebKit.framework/Resources/Info.plist"]),
+        run("git", ["rev-parse", "HEAD"], {cwd: root}), run("git", ["status", "--porcelain"], {cwd: root}),
+    ]);
+
+    report.versions = {macOS: macOS.trim(), macOSBuild: macOSBuild.trim(), webKit: webKit.trim()};
+    report.source = {revision: revision.trim(), workingTreeDirty: dirty.trim() !== ""};
+    const inventory = collectWrapperMethods(root);
+    report.wrapperInventory = inventory;
     await run(process.execPath, [join(root, "node_modules/typescript/bin/tsc"), "-p", join(directory, "tsconfig.json")]);
     const extension = join(temporary, "extension");
     await mkdir(extension);
-
-    await writeFile(join(extension, "manifest.json"), JSON.stringify({
-        manifest_version: 3, name: "Isolated WebKit contract tests", version: "0.0.1", action: {},
-        permissions: ["alarms", "tabs", "cookies", "contextMenus", "scripting", "declarativeNetRequest", "webRequest"],
-        host_permissions: ["https://example.test/*"],
-    }));
-
+    await writeFile(join(extension, "manifest.json"), JSON.stringify(manifest));
     await writeFile(join(extension, "probe.html"), '<!doctype html><meta charset="utf-8"><script src="probe.js"></script>');
 
     await build({
         config: false, entry: {probe: join(directory, "probe.ts")}, outDir: extension,
         bundle: true, format: ["iife"], platform: "browser", target: "es2022",
-        dts: false, sourcemap: false, silent: true, outExtension: () => ({js: ".js"}),
+        dts: false, sourcemap: false, silent: true,
+        outExtension: () => ({js: ".js"}),
+        define: {__WK_WRAPPER_INVENTORY__: JSON.stringify(inventory)},
     });
 
     const host = join(temporary, "host");
     await run("xcrun", ["clang", "-fobjc-arc", "-framework", "Cocoa", "-framework", "WebKit", join(directory, "host.m"), "-o", host]);
+    const {stdout, stderr} = await run(host, [extension], {timeout: 50000, maxBuffer: 4 * 1024 * 1024});
+    Object.assign(report, JSON.parse(stdout));
 
-    const [{stdout: macOS}, {stdout: webKit}, {stdout}] = await Promise.all([
-        run("sw_vers", ["-productVersion"]),
-        run("/usr/libexec/PlistBuddy", ["-c", "Print :CFBundleVersion", "/System/Library/Frameworks/WebKit.framework/Resources/Info.plist"]),
-        run(host, [extension], {timeout: 50000, maxBuffer: 4 * 1024 * 1024}),
-    ]);
+    if (stderr.trim()) {
+        report.hostDiagnostics = stderr.trim();
+    }
 
-    const {results} = JSON.parse(stdout);
-    assert.deepEqual(results.map(result => result.id), ["alarms", "action", "absent-results", "capabilities", "callback-errors"]);
-    console.log(`System WebKit ${webKit.trim()} on macOS ${macOS.trim()}`);
+    assert.deepEqual(report.results.map(result => result.id), ["alarms", "action", "absent-results", "capabilities", "callback-errors"]);
+    assert.ok(Array.isArray(report.grantedPermissions?.permissions), "Granted permissions must be recorded");
+    assert.ok(Array.isArray(report.grantedPermissions?.origins), "Granted host permissions must be recorded");
+    console.log(`System WebKit ${report.versions.webKit} on macOS ${report.versions.macOS}`);
 
-    for (const result of results) {
+    for (const result of report.results) {
         assert.equal(result.status, "passed", `${result.id}: ${result.error ?? "did not pass"}`);
         console.log(`PASS webkit ${result.id}`);
     }
+
+    report.status = "passed";
+} catch (error) {
+    report.failure = String(error);
+
+    if (error.stderr) {
+        report.hostDiagnostics = error.stderr;
+    }
+
+    throw error;
 } finally {
+    await mkdir(reportDirectory, {recursive: true});
+    await writeFile(join(reportDirectory, "report.json"), JSON.stringify(report, null, 2) + "\n");
+
+    const lines = ["# System WebKit observations", "", `Captured: ${report.capturedAt}`, `Status: ${report.status}`,
+        `Versions: ${JSON.stringify(report.versions ?? {})}`, `Source: ${JSON.stringify(report.source ?? {})}`,
+        `Requested permissions: ${JSON.stringify(manifest.permissions)}`,
+        `Requested host permissions: ${JSON.stringify(manifest.host_permissions)}`,
+        `Granted permissions/origins: ${JSON.stringify(report.grantedPermissions ?? "not recorded")}`,
+        "", report.host, "", "## Runtime scenarios", "",
+        ...(report.results ?? []).map(result => `- ${result.id}: ${result.status}${result.error ? ` — ${result.error}` : ""}`),
+        "", "## Observed presence", "", "Absence is not attributed to a browser limitation; permissions, manifest, host and build can affect exposure.", "",
+        "| Namespace | Method | State | Wrapper references |", "| --- | --- | --- | --- |",
+        ...(report.observations ?? []).flatMap(row => row.methods.map(method =>
+            `| ${row.namespace} | ${method.name} | ${method.state} | ${method.wrapperReferences.map(ref => `${ref.file}:${ref.line} (${ref.wrapper})`).join(", ")} |`)),
+        "", "## Human-reviewed findings", "", `Reviewed: ${report.humanReview.reviewedOn}. These sections are static findings, not automatic deductions.`,
+        ...["incorrectCallsAdapted", "sourceOnly", "remainingGaps"].flatMap(section => ["", `### ${section}`, "", ...report.humanReview[section].map(value => `- ${value}`)]),
+        ...(report.failure ? ["", `Failure: ${report.failure}`] : []), "",
+    ];
+
+    await writeFile(join(reportDirectory, "report.md"), lines.join("\n"));
+    console.log(`WebKit report: ${reportDirectory}`);
     await rm(temporary, {recursive: true, force: true});
 }