diff --git a/.github/workflows/webkit.yml b/.github/workflows/webkit.yml new file mode 100644 index 0000000..239d4eb --- /dev/null +++ b/.github/workflows/webkit.yml @@ -0,0 +1,48 @@ +name: WebKit pilot + +on: + workflow_dispatch: + pull_request: + push: + branches: + - develop + - 'codex/**' + - 'feature/**' + +permissions: + contents: read + +# Pilot coverage is independent of reusable CI and release.yml: failures remain visible +# without becoming a dependency of the release/publish job. +jobs: + webkit: + name: System WebKit (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + os: [macos-15, macos-26] + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: Use Node.js + uses: actions/setup-node@v7 + with: + node-version: 24 + cache: npm + - name: Install dependencies + run: npm ci + - name: Check generated sources + run: npm run generate:check + - name: Build + run: npm run build + - name: Run system WebKit contracts and record observations + run: npm run test:webkit -- --report-dir .output/webkit/${{ matrix.os }} + - name: Upload observations even when a scenario fails + if: always() + uses: actions/upload-artifact@v7 + with: + name: webkit-${{ matrix.os }}-${{ github.run_attempt }} + path: .output/webkit/${{ matrix.os }} + if-no-files-found: warn diff --git a/docs/safari.md b/docs/safari.md index 917eed5..7b9607c 100644 --- a/docs/safari.md +++ b/docs/safari.md @@ -111,3 +111,23 @@ including `webRequest`. A namespace absent from a differently configured host mu Safari API without further investigation. Automatically recorded observations and human-reviewed findings are separate concerns; a callable inventory is not a runtime conformance result for every wrapper. +## Saved observations and pilot CI + +`npm run test:webkit` writes `report.json` and `report.md` under the gitignored `.output/webkit/` directory, including +failures after runner startup. Override the directory with `--report-dir `. Raw reports are not committed. + +The report records system WebKit/macOS versions, the source revision and dirty flag, requested manifest permissions +and host permissions, **actually granted** permissions/origins, and scenario results. The inventory resolves native +method references from the production TypeScript source and enumerates callable properties of each exposed namespace, +including non-enumerable and inherited properties. Wrapper references are source observations, not proof of execution. + +Each method has one of three states: `namespace-unavailable`, `method-unavailable` (namespace present), or +`method-present`. Namespace absence is not attributed to Safari itself: permissions, manifest, host and build flags +can all affect it. The fixture requests and grants `webRequest`, so its namespace is not withheld by a missing request. +Human findings about adapted incorrect calls, source-only checks and remaining gaps are maintained separately in +`tests/webkit/review.json`; they are never inferred from presence alone. + +The separate **WebKit pilot** workflow runs on `macos-15` and `macos-26`, uploads the report even after failure, and +prints the actual system versions. It is not called by reusable CI or `release.yml`, so it is not a release dependency. +GitHub-hosted execution must be assessed from its own run; the local macOS 27.2 result does not establish runner +stability or the results of older system WebKit builds. diff --git a/tests/webkit/observations.test.ts b/tests/webkit/observations.test.ts new file mode 100644 index 0000000..3598218 --- /dev/null +++ b/tests/webkit/observations.test.ts @@ -0,0 +1,38 @@ +import {expect, test} from "@jest/globals"; + +import {observeMethods} from "./observations"; + +test("inventory distinguishes missing namespaces, missing methods and callable methods without invoking them", () => { + let calls = 0; + + const inherited = {inherited() { + calls++; + }}; + + const tabs = Object.create(inherited); + + Object.defineProperty(tabs, "query", {value() { + calls++; + }, enumerable: false}); + + const observations = observeMethods({tabs}, {tabs: {discard: []}, identity: {launchWebAuthFlow: []}}); + expect(observations.find(row => row.namespace === "identity")?.methods[0].state).toBe("namespace-unavailable"); + const methods = observations.find(row => row.namespace === "tabs")!.methods; + + expect(methods.map(row => [row.name, row.state])).toEqual([ + ["discard", "method-unavailable"], ["inherited", "method-present"], ["query", "method-present"], + ]); + + expect(calls).toBe(0); +}); + +test("inaccessible namespaces retain their error without claiming browser non-support", () => { + const api = Object.defineProperty({}, "cookies", {get() { + throw new Error("Permission withheld"); + }}); + + const [row] = observeMethods(api, {cookies: {get: []}}); + expect(row.state).toBe("namespace-unavailable"); + expect(row.accessError).toContain("Permission withheld"); + expect(row.methods[0].state).toBe("namespace-unavailable"); +}); diff --git a/tests/webkit/observations.ts b/tests/webkit/observations.ts new file mode 100644 index 0000000..5bdf37a --- /dev/null +++ b/tests/webkit/observations.ts @@ -0,0 +1,95 @@ +export interface WrapperReference { + file: string; + line: number; + wrapper: string; +} + +export type WrapperInventory = Record>; +export type MethodState = "namespace-unavailable" | "method-unavailable" | "method-present"; + +function keys(value: object): string[] { + const names = new Set(); + + for (let current: object | null = value; current && current !== Object.prototype; current = Object.getPrototypeOf(current)) { + for (const name of Object.getOwnPropertyNames(current)) { + if (name !== "constructor") { + names.add(name); + } + } + } + + return [...names].sort(); +} + +/** Presence observations only: no method is called and no absence is attributed to the browser. */ +export function observeMethods(api: object, wrappers: WrapperInventory) { + const namespaces = new Set(Object.keys(wrappers)); + + for (const name of keys(api)) { + try { + const value = Reflect.get(api, name); + + if (value && typeof value === "object") { + namespaces.add(name); + } + } catch { + namespaces.add(name); + } + } + + return [...namespaces].sort().map(namespace => { + let native: object | undefined; + let accessError: string | undefined; + + try { + const value = Reflect.get(api, namespace); + + if (value && (typeof value === "object" || typeof value === "function")) { + native = value; + } + } catch (error) { + accessError = String(error); + } + + const methods = new Set(Object.keys(wrappers[namespace] ?? {})); + + if (native) { + for (const name of keys(native)) { + try { + if (typeof Reflect.get(native, name) === "function") { + methods.add(name); + } + } catch { + // A guarded wrapper reference below still records an inaccessible member. + } + } + } + + return { + namespace, + state: native ? "namespace-present" : "namespace-unavailable", + ...(accessError ? {accessError} : {}), + methods: [...methods].sort().map(method => { + let valueType = "undefined"; + let methodError: string | undefined; + + if (native) { + try { + valueType = typeof Reflect.get(native, method); + } catch (error) { + methodError = String(error); + } + } + + const state: MethodState = !native ? "namespace-unavailable" + : valueType === "function" ? "method-present" : "method-unavailable"; + + return { + name: method, state, valueType, + wrapperReferences: wrappers[namespace]?.[method] ?? [], + ...(methodError ? {accessError: methodError} : {}), + }; + }), + }; + }); +} diff --git a/tests/webkit/probe.ts b/tests/webkit/probe.ts index 3a1d380..ca66f9f 100644 --- a/tests/webkit/probe.ts +++ b/tests/webkit/probe.ts @@ -1,11 +1,14 @@ import { browser, clearAlarm, clearAllAlarm, createAlarm, createAlarmIfNotExists, disableAction, - enableAction, getAlarm, getAllAlarm, getBadgeText, getCurrentTab, getDnrDynamicRules, + enableAction, getAlarm, getAllAlarm, getAllPermissions, getBadgeText, getCurrentTab, getDnrDynamicRules, getRegisteredContentScripts, isActionEnabled, isAvailableAction, isAvailableCookies, isAvailableDnr, isAvailableI18n, isAvailablePermissions, isAvailableRuntime, isAvailableTabs, isAvailableWebRequest, queryTabs, removeCookie, setBadgeText, } from "../../dist/index.js"; import {assert, rejects} from "../browser/extension/assert"; +import {observeMethods, type WrapperInventory} from "./observations"; + +declare const __WK_WRAPPER_INVENTORY__: WrapperInventory; const results: {id: string; status: string; error?: string}[] = []; const native = browser(); @@ -87,7 +90,12 @@ async function main() { const host = globalThis as typeof globalThis & {webkit: {messageHandlers: {result: {postMessage(value: unknown): void}}}}; - host.webkit.messageHandlers.result.postMessage({results}); + host.webkit.messageHandlers.result.postMessage({ + results, + grantedPermissions: await getAllPermissions(), + aliasesShareObject: native === globalThis.chrome, + observations: observeMethods(native, __WK_WRAPPER_INVENTORY__), + }); } void main(); diff --git a/tests/webkit/review.json b/tests/webkit/review.json new file mode 100644 index 0000000..6bb892b --- /dev/null +++ b/tests/webkit/review.json @@ -0,0 +1,22 @@ +{ + "kind": "human-reviewed-findings", + "reviewedOn": "2026-11-05", + "incorrectCallsAdapted": [ + "alarms.create: legacy WebKit rejects a callback; modern WebKit returns a Promise", + "action.isEnabled: WebKit needs an object; Firefox's Promise interface rejects a callback", + "alarms.get and tabs.getCurrent: normalize null to declared undefined", + "alarms.clear: prefer native boolean and use WebKit pre-read when no result is reported", + "alarms.clearAll: use the nonempty pre-read collection in every browser; ignore inconsistent native results", + "cookies.remove: preserve details and normalize absent native results to null" + ], + "sourceOnly": [ + "Presence of a callable property does not validate argument options, result fields or errors", + "Newer Promise-returning WebKit alarms.create is covered by unit tests and upstream source review" + ], + "remainingGaps": [ + "Successful execution and errors for every remaining public wrapper", + "Actual delayed alarm firing and event payloads", + "Safari application UI, user gestures, popups and iOS", + "Host/build flags controlling optional APIs and versions not executed by this run" + ] +} diff --git a/tests/webkit/run.mjs b/tests/webkit/run.mjs index 9184e8c..a0b7eda 100644 --- a/tests/webkit/run.mjs +++ b/tests/webkit/run.mjs @@ -1,55 +1,112 @@ import assert from "node:assert/strict"; import {execFile} from "node:child_process"; -import {mkdir, mkdtemp, rm, writeFile} from "node:fs/promises"; +import {mkdir, mkdtemp, readFile, rm, writeFile} from "node:fs/promises"; import {tmpdir} from "node:os"; import {dirname, join, resolve} from "node:path"; import {fileURLToPath} from "node:url"; -import {promisify} from "node:util"; +import {parseArgs, promisify} from "node:util"; import {build} from "tsup"; +import {collectWrapperMethods} from "./wrappers.mjs"; + const run = promisify(execFile); const directory = dirname(fileURLToPath(import.meta.url)); const root = resolve(directory, "../.."); +const {values} = parseArgs({options: {"report-dir": {type: "string"}}}); +const reportDirectory = resolve(root, values["report-dir"] ?? ".output/webkit"); assert.equal(process.platform, "darwin", "System WebKit tests require macOS 15.4+ and the Xcode Command Line Tools"); const temporary = await mkdtemp(join(tmpdir(), "addon-browser-webkit-")); +const manifest = { + manifest_version: 3, name: "Isolated WebKit contract tests", version: "0.0.1", action: {}, + permissions: ["alarms", "tabs", "cookies", "contextMenus", "scripting", "declarativeNetRequest", "webRequest"], + host_permissions: ["https://example.test/*"], +}; + +const report = { + schemaVersion: 1, + capturedAt: new Date().toISOString(), + status: "failed", + host: "Standalone WKWebExtensionController; nonpersistent data stores; minimal window/tab delegates", + requestedManifest: manifest, + humanReview: JSON.parse(await readFile(join(directory, "review.json"), "utf8")), +}; + try { + const [{stdout: macOS}, {stdout: macOSBuild}, {stdout: webKit}, {stdout: revision}, {stdout: dirty}] = await Promise.all([ + run("sw_vers", ["-productVersion"]), run("sw_vers", ["-buildVersion"]), + run("/usr/libexec/PlistBuddy", ["-c", "Print :CFBundleVersion", "/System/Library/Frameworks/WebKit.framework/Resources/Info.plist"]), + run("git", ["rev-parse", "HEAD"], {cwd: root}), run("git", ["status", "--porcelain"], {cwd: root}), + ]); + + report.versions = {macOS: macOS.trim(), macOSBuild: macOSBuild.trim(), webKit: webKit.trim()}; + report.source = {revision: revision.trim(), workingTreeDirty: dirty.trim() !== ""}; + const inventory = collectWrapperMethods(root); + report.wrapperInventory = inventory; await run(process.execPath, [join(root, "node_modules/typescript/bin/tsc"), "-p", join(directory, "tsconfig.json")]); const extension = join(temporary, "extension"); await mkdir(extension); - - await writeFile(join(extension, "manifest.json"), JSON.stringify({ - manifest_version: 3, name: "Isolated WebKit contract tests", version: "0.0.1", action: {}, - permissions: ["alarms", "tabs", "cookies", "contextMenus", "scripting", "declarativeNetRequest", "webRequest"], - host_permissions: ["https://example.test/*"], - })); - + await writeFile(join(extension, "manifest.json"), JSON.stringify(manifest)); await writeFile(join(extension, "probe.html"), ''); await build({ config: false, entry: {probe: join(directory, "probe.ts")}, outDir: extension, bundle: true, format: ["iife"], platform: "browser", target: "es2022", - dts: false, sourcemap: false, silent: true, outExtension: () => ({js: ".js"}), + dts: false, sourcemap: false, silent: true, + outExtension: () => ({js: ".js"}), + define: {__WK_WRAPPER_INVENTORY__: JSON.stringify(inventory)}, }); const host = join(temporary, "host"); await run("xcrun", ["clang", "-fobjc-arc", "-framework", "Cocoa", "-framework", "WebKit", join(directory, "host.m"), "-o", host]); + const {stdout, stderr} = await run(host, [extension], {timeout: 50000, maxBuffer: 4 * 1024 * 1024}); + Object.assign(report, JSON.parse(stdout)); - const [{stdout: macOS}, {stdout: webKit}, {stdout}] = await Promise.all([ - run("sw_vers", ["-productVersion"]), - run("/usr/libexec/PlistBuddy", ["-c", "Print :CFBundleVersion", "/System/Library/Frameworks/WebKit.framework/Resources/Info.plist"]), - run(host, [extension], {timeout: 50000, maxBuffer: 4 * 1024 * 1024}), - ]); + if (stderr.trim()) { + report.hostDiagnostics = stderr.trim(); + } - const {results} = JSON.parse(stdout); - assert.deepEqual(results.map(result => result.id), ["alarms", "action", "absent-results", "capabilities", "callback-errors"]); - console.log(`System WebKit ${webKit.trim()} on macOS ${macOS.trim()}`); + assert.deepEqual(report.results.map(result => result.id), ["alarms", "action", "absent-results", "capabilities", "callback-errors"]); + assert.ok(Array.isArray(report.grantedPermissions?.permissions), "Granted permissions must be recorded"); + assert.ok(Array.isArray(report.grantedPermissions?.origins), "Granted host permissions must be recorded"); + console.log(`System WebKit ${report.versions.webKit} on macOS ${report.versions.macOS}`); - for (const result of results) { + for (const result of report.results) { assert.equal(result.status, "passed", `${result.id}: ${result.error ?? "did not pass"}`); console.log(`PASS webkit ${result.id}`); } + + report.status = "passed"; +} catch (error) { + report.failure = String(error); + + if (error.stderr) { + report.hostDiagnostics = error.stderr; + } + + throw error; } finally { + await mkdir(reportDirectory, {recursive: true}); + await writeFile(join(reportDirectory, "report.json"), JSON.stringify(report, null, 2) + "\n"); + + const lines = ["# System WebKit observations", "", `Captured: ${report.capturedAt}`, `Status: ${report.status}`, + `Versions: ${JSON.stringify(report.versions ?? {})}`, `Source: ${JSON.stringify(report.source ?? {})}`, + `Requested permissions: ${JSON.stringify(manifest.permissions)}`, + `Requested host permissions: ${JSON.stringify(manifest.host_permissions)}`, + `Granted permissions/origins: ${JSON.stringify(report.grantedPermissions ?? "not recorded")}`, + "", report.host, "", "## Runtime scenarios", "", + ...(report.results ?? []).map(result => `- ${result.id}: ${result.status}${result.error ? ` — ${result.error}` : ""}`), + "", "## Observed presence", "", "Absence is not attributed to a browser limitation; permissions, manifest, host and build can affect exposure.", "", + "| Namespace | Method | State | Wrapper references |", "| --- | --- | --- | --- |", + ...(report.observations ?? []).flatMap(row => row.methods.map(method => + `| ${row.namespace} | ${method.name} | ${method.state} | ${method.wrapperReferences.map(ref => `${ref.file}:${ref.line} (${ref.wrapper})`).join(", ")} |`)), + "", "## Human-reviewed findings", "", `Reviewed: ${report.humanReview.reviewedOn}. These sections are static findings, not automatic deductions.`, + ...["incorrectCallsAdapted", "sourceOnly", "remainingGaps"].flatMap(section => ["", `### ${section}`, "", ...report.humanReview[section].map(value => `- ${value}`)]), + ...(report.failure ? ["", `Failure: ${report.failure}`] : []), "", + ]; + + await writeFile(join(reportDirectory, "report.md"), lines.join("\n")); + console.log(`WebKit report: ${reportDirectory}`); await rm(temporary, {recursive: true, force: true}); } diff --git a/tests/webkit/wrappers.mjs b/tests/webkit/wrappers.mjs new file mode 100644 index 0000000..789e956 --- /dev/null +++ b/tests/webkit/wrappers.mjs @@ -0,0 +1,106 @@ +import {relative, resolve} from "node:path"; + +import ts from "typescript"; + +/** Resolve native call symbols instead of maintaining a browser-method allowlist. */ +export function collectWrapperMethods(root) { + const configPath = resolve(root, "tsconfig.json"); + const config = ts.readConfigFile(configPath, ts.sys.readFile); + + if (config.error) { + throw new Error(ts.flattenDiagnosticMessageText(config.error.messageText, "\n")); + } + + const parsed = ts.parseJsonConfigFileContent(config.config, ts.sys, root); + const program = ts.createProgram(parsed.fileNames, parsed.options); + const checker = program.getTypeChecker(); + const inventory = {}; + const unresolved = []; + + function add(namespace, method, source, node) { + const file = relative(root, source.fileName).replaceAll("\\", "/"); + const line = source.getLineAndCharacterOfPosition(node.getStart(source)).line + 1; + let owner; + + for (let parent = node; parent; parent = parent.parent) { + if (ts.isVariableDeclaration(parent) && parent.parent?.parent?.modifiers?.some(value => value.kind === ts.SyntaxKind.ExportKeyword)) { + owner = parent; + } + + if (ts.isFunctionDeclaration(parent) && parent.modifiers?.some(value => value.kind === ts.SyntaxKind.ExportKeyword)) { + owner = parent; + } + } + + const wrapper = owner?.name?.getText(source) ?? ""; + const references = (inventory[namespace] ??= {})[method] ??= []; + + if (!references.some(value => value.file === file && value.line === line)) { + references.push({file, line, wrapper}); + } + } + + function names(type) { + return (type.isUnion() ? type.types : [type]).flatMap(member => member.isStringLiteral() ? [member.value] : []); + } + + function symbolMethod(type, method, source, node) { + for (const member of type.isUnion() ? type.types : [type]) { + const symbol = checker.getPropertyOfType(member, method); + const match = symbol && /^(?:chrome|browser|opr)\.([\w$]+)\.([\w$]+)$/.exec(checker.getFullyQualifiedName(symbol)); + + if (match) { + add(match[1], match[2], source, node); + } + } + } + + for (const source of program.getSourceFiles()) { + const path = relative(root, source.fileName).replaceAll("\\", "/"); + + if (!path.startsWith("src/api/") || source.isDeclarationFile || path.endsWith(".test.ts")) { + continue; + } + + const visit = node => { + if (ts.isPropertyAccessExpression(node) && checker.getTypeAtLocation(node).getCallSignatures().length) { + symbolMethod(checker.getTypeAtLocation(node.expression), node.name.text, source, node); + } + + if (ts.isCallExpression(node)) { + if (ts.isIdentifier(node.expression) && node.expression.text === "requireApiMethod") { + const [, namespace, method] = node.arguments; + const namespaces = namespace ? names(checker.getTypeAtLocation(namespace)) : []; + const methods = method ? names(checker.getTypeAtLocation(method)) : []; + + if (!namespaces.length || !methods.length) { + unresolved.push({file: path, call: node.getText(source)}); + } + + for (const ns of namespaces) { + for (const name of methods) { + add(ns, name, source, node); + } + } + } + } + + ts.forEachChild(node, visit); + }; + + visit(source); + } + + if (unresolved.length) { + throw new Error(`Unresolved native method guards: ${JSON.stringify(unresolved)}`); + } + + if (!Object.keys(inventory).length) { + throw new Error("No native wrapper methods were discovered"); + } + + // Keep output inspectable and deterministic within a given source revision. + return Object.fromEntries(Object.entries(inventory).sort().map(([namespace, methods]) => [namespace, + Object.fromEntries(Object.entries(methods).sort().map(([method, refs]) => [method, refs.sort((a, b) => a.file.localeCompare(b.file) || a.line - b.line)])), + ])); +} diff --git a/tests/webkit/wrappers.test.mjs b/tests/webkit/wrappers.test.mjs new file mode 100644 index 0000000..493bcb7 --- /dev/null +++ b/tests/webkit/wrappers.test.mjs @@ -0,0 +1,17 @@ +import {fileURLToPath} from "node:url"; + +import {expect, test} from "@jest/globals"; + +import {collectWrapperMethods} from "./wrappers.mjs"; + +test("discovers direct calls, guarded calls, union namespaces and function references from production sources", () => { + const root = fileURLToPath(new URL("../..", import.meta.url)); + const methods = collectWrapperMethods(root); + expect(methods.alarms.create.some(ref => ref.wrapper === "createAlarm")).toBe(true); + expect(methods.alarms.clear.some(ref => ref.wrapper === "clearAlarm")).toBe(true); + expect(methods.action.isEnabled.some(ref => ref.wrapper === "isActionEnabled")).toBe(true); + expect(methods.browserAction.isEnabled.some(ref => ref.wrapper === "isActionEnabled")).toBe(true); + expect(methods.cookies.getPartitionKey.some(ref => ref.wrapper === "getCookiePartitionKey")).toBe(true); + expect(methods.i18n.detectLanguage).toHaveLength(1); + expect(Object.keys(methods).length).toBeGreaterThan(25); +});