diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee797116d..87f0a8947 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -272,6 +272,10 @@ jobs: if: ${{ !cancelled() && needs.plan.outputs.site == 'true' && needs.build.result == 'success' }} runs-on: ubuntu-latest timeout-minutes: 20 + outputs: + # A re-run of failed jobs keeps this job's earlier attempt, so the smoke + # and the publisher find the candidate by this name, not by recomputing it. + candidate: site-stable-${{ github.sha }}-${{ github.run_attempt }} steps: - uses: actions/checkout@v6 - uses: ./.github/actions/setup @@ -287,6 +291,16 @@ jobs: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} run: pnpm site:build + # This branch publishes the Stable channel at /ExoJS/. Its site build + # writes no site-build.json, so the finished tree is recorded here. + - name: Record the Stable site identity + run: node scripts/pages/stamp-stable-site.ts --dir site/dist --sha "$GITHUB_SHA" + + # The tree must match its own site-build.json, carry this commit, channel + # and base, contain no links or special files and fit the site budget. + - name: Verify the site candidate + run: node scripts/pages/channel-artifact.ts verify --dir site/dist --channel stable --sha "$GITHUB_SHA" + - name: Upload tooling logs on failure if: ${{ failure() }} uses: actions/upload-artifact@v7 @@ -297,12 +311,14 @@ jobs: include-hidden-files: true if-no-files-found: warn - # Deploy Pages ships exactly this artifact; it builds nothing itself. + # Deploy Pages ships exactly this artifact; it builds nothing itself. A + # push to main is a publishable Stable candidate and is kept for 90 days, + # so the publisher never depends on a short-lived download. - uses: actions/upload-artifact@v7 with: - name: site-dist-${{ github.sha }} + name: site-stable-${{ github.sha }}-${{ github.run_attempt }} path: site/dist/ - retention-days: 1 + retention-days: ${{ github.event_name == 'push' && 90 || 1 }} if-no-files-found: error # Every catalog example through the real playground route, against the site @@ -323,7 +339,7 @@ jobs: browser: chromium - uses: actions/download-artifact@v8 with: - name: site-dist-${{ github.sha }} + name: ${{ needs.site.outputs.candidate }} path: site/dist - run: pnpm test:examples:smoke --renderer webgl2 ${{ needs.plan.outputs.smokeSample == 'true' && '--sample' || '' }} diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index d39beba64..797f76043 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -1,83 +1,125 @@ name: Deploy Pages -# Deploys the site EXACTLY as CI built and validated it — this workflow builds -# no source code. CI's `site` job (and a release's `prepare` job) runs the site -# gates and `site:build` and uploads `site/dist/**` as `site-dist-`; here -# that same artifact is downloaded from the run that triggered this one and -# handed to Pages unchanged. +# The one publisher of the project site. GitHub Pages serves a single tree, so +# every deployment carries both channels: Stable at /ExoJS/ and Next at +# /ExoJS/next/, each exactly as a green CI push to main or next (or a stable +# Release) built and smoked it. This workflow builds no site code. # -# The defect this prevents: the workflow used to check the sources out and -# rebuild the library and the site itself, so what went live was a second build -# that no gate had ever seen. A green CI run said nothing about the deployed -# bytes. +# The triggering run is only a wake-up call. On every start the publisher +# re-decides BOTH channels from GitHub's own metadata (scripts/pages/ +# select-channels.ts): each channel gets the newest qualified candidate on its +# branch's current history, or keeps what is already published. A queued or +# re-run event can therefore neither roll a channel back nor drop the other +# channel's update, and a green run without a site candidate changes nothing. # -# Fail-closed by construction: there is no `workflow_dispatch` path, because a -# manual run has no triggering run to read the artifact from. If the artifact is -# missing or belongs to a different commit, the download step fails and nothing -# is deployed. To re-deploy, re-run the CI workflow for the commit. +# Trust: only successful same-repository push runs of CI (main/next) and +# stable Release tags qualify; a pull request, a fork or a manual dispatch +# never does. Candidates are downloaded by the trusted scripts of the +# default-branch checkout below, never executed, and checked against the +# digest GitHub reports, their own site-build.json and the archive rules +# before anything is composed. Any doubt deploys nothing and leaves the live +# site as it is. +# +# A deployment also keeps its composed tree (the Pages artifact) and its +# channels.json for 90 days, so a channel whose candidate expired can be +# carried over unchanged from the last published snapshot. on: workflow_run: workflows: ['Release', 'CI'] types: [completed] -permissions: - contents: read - # Reading an artifact from ANOTHER workflow run needs `actions: read` plus an - # explicit token on the download step. - actions: read - pages: write - id-token: write +permissions: {} +# One queue for the whole site, and a running deployment is never cancelled. +# Ordering does not rest on this: each run re-decides both channels when it +# starts. concurrency: - group: pages-${{ github.event.workflow_run.head_sha }} - cancel-in-progress: true + group: pages-combined + cancel-in-progress: false jobs: - deploy: - name: Deploy to GitHub Pages + compose: + name: Compose Stable and Next if: >- ${{ github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' && - ( - github.event.workflow_run.name == 'Release' || - ( - github.event.workflow_run.name == 'CI' && - github.event.workflow_run.event == 'push' && - github.event.workflow_run.head_branch == 'main' - ) - ) + github.event.workflow_run.event == 'push' && + github.event.workflow_run.repository.full_name == github.repository && + github.event.workflow_run.head_repository.full_name == github.repository }} runs-on: ubuntu-latest - timeout-minutes: 10 - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} + timeout-minutes: 30 + permissions: + contents: read + # Listing runs and reading artifacts of other workflow runs. + actions: read + outputs: + deploy: ${{ steps.select.outputs.deploy }} steps: - # `run-id` pins the download to the exact run that triggered this one, and - # the commit in the artifact name pins it to that run's commit — an older - # or unrelated build cannot be picked up. A missing artifact fails here. - - name: Download validated site artifact - uses: actions/download-artifact@v8 + # The publisher's own scripts, from the default branch - never from a + # candidate. Dependency-free TypeScript that node strips: no install. + - uses: actions/checkout@v6 with: - name: site-dist-${{ github.event.workflow_run.head_sha }} - run-id: ${{ github.event.workflow_run.id }} - github-token: ${{ github.token }} - path: site-dist + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + sparse-checkout-cone-mode: false + sparse-checkout: | + /.nvmrc + /scripts/pages/ + + - uses: actions/setup-node@v6 + with: + node-version-file: .nvmrc + + - name: Select both channels + id: select + env: + GITHUB_TOKEN: ${{ github.token }} + TRIGGER_RUN_ID: ${{ github.event.workflow_run.id }} + run: node scripts/pages/select-channels.ts --out "$RUNNER_TEMP/selection.json" - - name: Verify the artifact carries a built site - run: | - test -f site-dist/index.html || { - echo "::error::site-dist-${{ github.event.workflow_run.head_sha }} has no index.html — refusing to deploy." - exit 1 - } + - name: Fetch, verify and compose the channels + if: steps.select.outputs.deploy == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node scripts/pages/compose-site.ts + --selection "$RUNNER_TEMP/selection.json" + --work "$RUNNER_TEMP/channels" + --out "$RUNNER_TEMP/site" + --channels-out "$RUNNER_TEMP/channels-record" - name: Package for Pages + if: steps.select.outputs.deploy == 'true' uses: actions/upload-pages-artifact@v5 with: - path: site-dist + path: ${{ runner.temp }}/site + retention-days: 90 + - name: Keep the deployed pairing + if: steps.select.outputs.deploy == 'true' + uses: actions/upload-artifact@v7 + with: + name: pages-channels-${{ github.run_id }} + path: ${{ runner.temp }}/channels-record/channels.json + retention-days: 90 + if-no-files-found: error + + deploy: + name: Deploy to GitHub Pages + needs: compose + if: needs.compose.outputs.deploy == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: - name: Deploy id: deployment uses: actions/deploy-pages@v5 diff --git a/scripts/pages/archive-safety.ts b/scripts/pages/archive-safety.ts new file mode 100644 index 000000000..f1336058e --- /dev/null +++ b/scripts/pages/archive-safety.ts @@ -0,0 +1,548 @@ +import { lstatSync, mkdirSync, readdirSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve, sep } from 'node:path'; +import { crc32, inflateRawSync } from 'node:zlib'; + +import { validateSitePath } from './site-manifest.ts'; + +/** + * Inspection and extraction of the archives the Pages publisher handles: the + * zip of a channel candidate artifact and the tar inside a Pages artifact. + * + * Every entry is inspected before anything is written, and extraction trusts + * nothing it inspected: sizes and checksums are re-checked while inflating, so + * a header that lies about its size cannot unpack past the budget. + * + * Dependency-free and erasable TypeScript, run by the publisher with plain + * `node`. + */ + +export type ArchiveEntryType = 'file' | 'directory' | 'symlink' | 'hardlink' | 'special'; + +export interface ArchiveEntry { + /** Normalized root-relative path, without a trailing slash. */ + name: string; + type: ArchiveEntryType; + /** Declared unpacked size. */ + size: number; +} + +export interface ZipEntry extends ArchiveEntry { + method: number; + compressedSize: number; + crc32: number; + encrypted: boolean; + localHeaderOffset: number; + rawName: Buffer; +} + +export interface TarEntry extends ArchiveEntry { + dataOffset: number; +} + +export interface ArchiveLimits { + /** Largest total of declared unpacked sizes accepted. */ + maxUnpackedBytes: number; + /** Largest number of entries accepted. */ + maxEntries?: number; +} + +const DEFAULT_MAX_ENTRIES = 250_000; + +export class ArchiveSafetyError extends Error { + readonly problems: readonly string[]; + + constructor(problems: readonly string[]) { + super(`[archive-safety] archive refused:\n ${problems.join('\n ')}`); + this.name = 'ArchiveSafetyError'; + this.problems = problems; + } +} + +const S_IFMT = 0o170000; + +const typeFromMode = (mode: number): ArchiveEntryType | null => { + switch (mode & S_IFMT) { + case 0: + return null; + case 0o100000: + return 'file'; + case 0o040000: + return 'directory'; + case 0o120000: + return 'symlink'; + default: + return 'special'; + } +}; + +const ZIP_END = 0x06054b50; +const ZIP64_END = 0x06064b50; +const ZIP64_LOCATOR = 0x07064b50; +const ZIP_CENTRAL = 0x02014b50; +const ZIP_LOCAL = 0x04034b50; +const U16_MAX = 0xffff; +const U32_MAX = 0xffffffff; + +const readU64 = (buffer: Buffer, offset: number): number => { + const value = buffer.readBigUInt64LE(offset); + + if (value > BigInt(Number.MAX_SAFE_INTEGER)) { + throw new ArchiveSafetyError([`zip field at ${offset} exceeds the safe integer range`]); + } + + return Number(value); +}; + +const findZipEnd = (zip: Buffer): number => { + const lowest = Math.max(0, zip.length - 22 - U16_MAX); + + for (let offset = zip.length - 22; offset >= lowest; offset--) { + if (zip.readUInt32LE(offset) === ZIP_END) { + return offset; + } + } + + throw new ArchiveSafetyError(['not a zip archive: no end of central directory record']); +}; + +/** The central directory of `zip`, with entry types and normalized names. */ +export const readZipEntries = (zip: Buffer): ZipEntry[] => { + if (zip.length < 22) { + throw new ArchiveSafetyError(['not a zip archive: too short']); + } + + const end = findZipEnd(zip); + let count = zip.readUInt16LE(end + 10); + let directorySize = zip.readUInt32LE(end + 12); + let directoryOffset = zip.readUInt32LE(end + 16); + + if (count === U16_MAX || directorySize === U32_MAX || directoryOffset === U32_MAX) { + const locator = end - 20; + + if (locator < 0 || zip.readUInt32LE(locator) !== ZIP64_LOCATOR) { + throw new ArchiveSafetyError(['zip64 archive without a zip64 end locator']); + } + + const record = readU64(zip, locator + 8); + + if (record + 56 > zip.length || zip.readUInt32LE(record) !== ZIP64_END) { + throw new ArchiveSafetyError(['zip64 end record missing']); + } + + count = readU64(zip, record + 32); + directorySize = readU64(zip, record + 40); + directoryOffset = readU64(zip, record + 48); + } + + if (directoryOffset + directorySize > zip.length) { + throw new ArchiveSafetyError(['zip central directory lies outside the archive']); + } + + const entries: ZipEntry[] = []; + let offset = directoryOffset; + + for (let index = 0; index < count; index++) { + if (offset + 46 > zip.length || zip.readUInt32LE(offset) !== ZIP_CENTRAL) { + throw new ArchiveSafetyError([`zip central directory entry ${index} is corrupt`]); + } + + const madeBy = zip.readUInt16LE(offset + 4) >> 8; + const flags = zip.readUInt16LE(offset + 8); + const method = zip.readUInt16LE(offset + 10); + const crc = zip.readUInt32LE(offset + 16); + let compressedSize = zip.readUInt32LE(offset + 20); + let size = zip.readUInt32LE(offset + 24); + const nameLength = zip.readUInt16LE(offset + 28); + const extraLength = zip.readUInt16LE(offset + 30); + const commentLength = zip.readUInt16LE(offset + 32); + const external = zip.readUInt32LE(offset + 38); + let localHeaderOffset = zip.readUInt32LE(offset + 42); + const rawName = zip.subarray(offset + 46, offset + 46 + nameLength); + const extra = zip.subarray(offset + 46 + nameLength, offset + 46 + nameLength + extraLength); + + if (size === U32_MAX || compressedSize === U32_MAX || localHeaderOffset === U32_MAX) { + for (let cursor = 0; cursor + 4 <= extra.length;) { + const id = extra.readUInt16LE(cursor); + const length = extra.readUInt16LE(cursor + 2); + + if (id === 0x0001) { + let field = cursor + 4; + + if (size === U32_MAX) { + size = readU64(extra, field); + field += 8; + } + + if (compressedSize === U32_MAX) { + compressedSize = readU64(extra, field); + field += 8; + } + + if (localHeaderOffset === U32_MAX) { + localHeaderOffset = readU64(extra, field); + } + } + + cursor += 4 + length; + } + } + + const decoded = rawName.toString('utf8'); + const isDirectoryName = decoded.endsWith('/'); + const modeType = madeBy === 3 ? typeFromMode(external >>> 16) : null; + const type: ArchiveEntryType = modeType ?? ((external & 0x10) !== 0 || isDirectoryName ? 'directory' : 'file'); + + entries.push({ + name: isDirectoryName ? decoded.slice(0, -1) : decoded, + type: type === 'file' && isDirectoryName ? 'directory' : type, + size, + method, + compressedSize, + crc32: crc, + encrypted: (flags & 0x0001) !== 0, + localHeaderOffset, + rawName, + }); + offset += 46 + nameLength + extraLength + commentLength; + } + + return entries; +}; + +const parseTarNumber = (field: Buffer): number => { + if ((field[0]! & 0x80) !== 0) { + let value = 0; + + for (let index = 1; index < field.length; index++) { + value = value * 256 + field[index]!; + } + + return value; + } + + const text = field + .toString('ascii') + .replace(/[\0 ]+$/, '') + .trim(); + + return text === '' ? 0 : Number.parseInt(text, 8); +}; + +const tarString = (field: Buffer): string => { + const end = field.indexOf(0); + + return field.subarray(0, end === -1 ? field.length : end).toString('utf8'); +}; + +const parsePax = (data: Buffer): Record => { + const records: Record = {}; + let cursor = 0; + + while (cursor < data.length) { + const space = data.indexOf(0x20, cursor); + const length = Number.parseInt(data.subarray(cursor, space).toString('ascii'), 10); + + if (space === -1 || !Number.isFinite(length) || length <= 0 || cursor + length > data.length) { + throw new ArchiveSafetyError(['corrupt pax extended header']); + } + + const record = data.subarray(space + 1, cursor + length - 1).toString('utf8'); + const equals = record.indexOf('='); + + if (equals > 0) { + records[record.slice(0, equals)] = record.slice(equals + 1); + } + + cursor += length; + } + + return records; +}; + +const normalizeTarName = (name: string): string => { + let normalized = name; + + while (normalized.startsWith('./')) { + normalized = normalized.slice(2); + } + + return normalized.endsWith('/') ? normalized.slice(0, -1) : normalized; +}; + +/** The members of a ustar/pax/GNU tar, without the `./` root entry. */ +export const readTarEntries = (tar: Buffer): TarEntry[] => { + const entries: TarEntry[] = []; + let offset = 0; + let pending: Record = {}; + let longName: string | null = null; + + while (offset + 512 <= tar.length) { + const header = tar.subarray(offset, offset + 512); + + if (header.every(byte => byte === 0)) { + return entries; + } + + let sum = 0; + + for (let index = 0; index < 512; index++) { + sum += index >= 148 && index < 156 ? 0x20 : header[index]!; + } + + if (sum !== parseTarNumber(header.subarray(148, 156))) { + throw new ArchiveSafetyError([`tar header at ${offset} has a bad checksum`]); + } + + const flag = String.fromCharCode(header[156]!); + let size = parseTarNumber(header.subarray(124, 136)); + const dataOffset = offset + 512; + + if (dataOffset + size > tar.length) { + throw new ArchiveSafetyError([`tar member at ${offset} is truncated`]); + } + + const data = tar.subarray(dataOffset, dataOffset + size); + offset = dataOffset + Math.ceil(size / 512) * 512; + + if (flag === 'x') { + pending = { ...pending, ...parsePax(data) }; + continue; + } + + if (flag === 'g') { + continue; + } + + if (flag === 'L') { + longName = tarString(data); + continue; + } + + if (flag === 'K') { + continue; + } + + const magic = header.subarray(257, 262).toString('ascii'); + const prefix = magic === 'ustar' ? tarString(header.subarray(345, 500)) : ''; + const baseName = tarString(header.subarray(0, 100)); + const name = pending['path'] ?? longName ?? (prefix === '' ? baseName : `${prefix}/${baseName}`); + + if (pending['size'] !== undefined) { + size = Number.parseInt(pending['size'], 10); + } + + pending = {}; + longName = null; + + const type: ArchiveEntryType = + flag === '0' || flag === '\0' ? 'file' : flag === '5' ? 'directory' : flag === '2' ? 'symlink' : flag === '1' ? 'hardlink' : 'special'; + const normalized = normalizeTarName(name); + + if (normalized === '' || normalized === '.') { + continue; + } + + entries.push({ name: normalized, type, size: type === 'file' ? size : 0, dataOffset }); + } + + throw new ArchiveSafetyError(['tar archive ends without its end-of-archive blocks']); +}; + +/** Every reason the listed entries may not be unpacked; empty when they may. */ +export const checkArchiveEntries = (entries: readonly ArchiveEntry[], limits: ArchiveLimits): string[] => { + const problems: string[] = []; + const seen = new Set(); + let total = 0; + + if (entries.length > (limits.maxEntries ?? DEFAULT_MAX_ENTRIES)) { + problems.push(`${entries.length} entries exceed the limit of ${limits.maxEntries ?? DEFAULT_MAX_ENTRIES}`); + } + + for (const entry of entries) { + const pathProblem = validateSitePath(entry.name); + + if (pathProblem !== null) { + problems.push(`unsafe path: ${pathProblem}`); + continue; + } + + if (seen.has(entry.name)) { + problems.push(`duplicate entry ${entry.name}`); + } + + seen.add(entry.name); + + if (entry.type === 'symlink') { + problems.push(`symlink entry ${entry.name}`); + } else if (entry.type === 'hardlink') { + problems.push(`hardlink entry ${entry.name}`); + } else if (entry.type === 'special') { + problems.push(`special file entry ${entry.name}`); + } + + const zip = entry as Partial; + + if (zip.encrypted) { + problems.push(`encrypted entry ${entry.name}`); + } + + if (zip.method !== undefined && entry.type === 'file' && zip.method !== 0 && zip.method !== 8) { + problems.push(`entry ${entry.name} uses unsupported compression method ${zip.method}`); + } + + total += entry.type === 'file' ? entry.size : 0; + } + + if (total > limits.maxUnpackedBytes) { + problems.push(`declared unpacked size ${total} exceeds the budget of ${limits.maxUnpackedBytes} bytes`); + } + + return problems; +}; + +const assertSafe = (entries: readonly ArchiveEntry[], limits: ArchiveLimits): void => { + const problems = checkArchiveEntries(entries, limits); + + if (problems.length > 0) { + throw new ArchiveSafetyError(problems); + } +}; + +const inflateZipEntry = (zip: Buffer, entry: ZipEntry): Buffer => { + const local = entry.localHeaderOffset; + + if (local + 30 > zip.length || zip.readUInt32LE(local) !== ZIP_LOCAL) { + throw new ArchiveSafetyError([`local header of ${entry.name} is corrupt`]); + } + + const nameLength = zip.readUInt16LE(local + 26); + const extraLength = zip.readUInt16LE(local + 28); + + if (!zip.subarray(local + 30, local + 30 + nameLength).equals(entry.rawName)) { + throw new ArchiveSafetyError([`local header of ${entry.name} names a different file`]); + } + + const start = local + 30 + nameLength + extraLength; + + if (start + entry.compressedSize > zip.length) { + throw new ArchiveSafetyError([`data of ${entry.name} lies outside the archive`]); + } + + const data = zip.subarray(start, start + entry.compressedSize); + let output: Buffer; + + if (entry.method === 0) { + output = data; + } else { + try { + output = inflateRawSync(data, { maxOutputLength: Math.max(1, entry.size) }); + } catch { + throw new ArchiveSafetyError([`entry ${entry.name} inflates past its declared size of ${entry.size} bytes or is corrupt`]); + } + } + + if (output.length !== entry.size) { + throw new ArchiveSafetyError([`entry ${entry.name} unpacks to ${output.length} bytes, not its declared size of ${entry.size}`]); + } + + if (crc32(output) !== entry.crc32) { + throw new ArchiveSafetyError([`entry ${entry.name} fails its CRC-32 check`]); + } + + return output; +}; + +const insideDestination = (dest: string, name: string): string => { + const root = resolve(dest); + const target = resolve(root, ...name.split('/')); + + if (!target.startsWith(root + sep)) { + throw new ArchiveSafetyError([`entry ${name} resolves outside the destination`]); + } + + return target; +}; + +const writeEntries = (dest: string, entries: readonly ArchiveEntry[], read: (entry: ArchiveEntry) => Buffer): void => { + mkdirSync(dest, { recursive: true }); + + for (const entry of entries) { + const target = insideDestination(dest, entry.name); + + if (entry.type === 'directory') { + mkdirSync(target, { recursive: true }); + continue; + } + + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, read(entry), { flag: 'wx' }); + } +}; + +/** Unpacks `zip` into `dest` after inspecting every entry. */ +export const extractZip = (zip: Buffer, dest: string, limits: ArchiveLimits): void => { + const entries = readZipEntries(zip); + assertSafe(entries, limits); + writeEntries(dest, entries, entry => inflateZipEntry(zip, entry as ZipEntry)); +}; + +/** The bytes of the single file `name` in `zip`, after inspecting every entry. */ +export const readZipFile = (zip: Buffer, name: string, limits: ArchiveLimits): Buffer => { + const entries = readZipEntries(zip); + assertSafe(entries, limits); + const entry = entries.find(candidate => candidate.name === name && candidate.type === 'file'); + + if (entry === undefined) { + throw new ArchiveSafetyError([`archive has no file ${name}`]); + } + + return inflateZipEntry(zip, entry); +}; + +/** Unpacks `tar` into `dest` after inspecting every member. */ +export const extractTar = (tar: Buffer, dest: string, limits: ArchiveLimits): void => { + const entries = readTarEntries(tar); + assertSafe(entries, limits); + writeEntries(dest, entries, entry => { + const member = entry as TarEntry; + + return tar.subarray(member.dataOffset, member.dataOffset + member.size); + }); +}; + +/** + * Every entry below `root` the Pages artifact must not contain: a symbolic + * link, a hardlinked file, a special file or a name that is not a site path. + */ +export const checkTreeSafety = (root: string): string[] => { + const problems: string[] = []; + + const visit = (directory: string, prefix: string): void => { + for (const name of readdirSync(directory)) { + const rel = prefix === '' ? name : `${prefix}/${name}`; + const pathProblem = validateSitePath(rel); + + if (pathProblem !== null) { + problems.push(`unsafe path: ${pathProblem}`); + continue; + } + + const abs = join(directory, name); + const stats = lstatSync(abs); + + if (stats.isSymbolicLink()) { + problems.push(`symlink ${rel}`); + } else if (stats.isDirectory()) { + visit(abs, rel); + } else if (!stats.isFile()) { + problems.push(`special file ${rel}`); + } else if (stats.nlink > 1) { + problems.push(`hardlink ${rel} (${stats.nlink} links)`); + } + } + }; + + visit(root, ''); + + return problems; +}; diff --git a/scripts/pages/channel-artifact.ts b/scripts/pages/channel-artifact.ts new file mode 100644 index 000000000..7755d796e --- /dev/null +++ b/scripts/pages/channel-artifact.ts @@ -0,0 +1,158 @@ +import { createHash } from 'node:crypto'; +import { pathToFileURL } from 'node:url'; + +import { checkTreeSafety, extractZip } from './archive-safety.ts'; +import { evaluateSiteSize, SITE_SIZE_REFUSE_BYTES } from './site-budget.ts'; +import { SITE_CHANNEL_BASE, SITE_CHANNELS, type SiteChannel } from './site-channel.ts'; +import { readSiteBuildManifest, SITE_BUILD_FILE, type SiteBuildManifest, verifySiteTree } from './site-manifest.ts'; + +/** + * One channel candidate: the zip of a built site tree with its + * `site-build.json`, uploaded by CI or Release as `site---`. + * + * Dependency-free and erasable TypeScript, run by CI after the site build and + * by the publisher with plain `node`. + */ + +const CANDIDATE_NAME = /^site-(stable|next)-([0-9a-f]{40})-([1-9][0-9]{0,3})$/; + +/** The artifact name a run uploads its channel tree under. */ +export const candidateArtifactName = (channel: SiteChannel, sha: string, attempt: number): string => `site-${channel}-${sha}-${attempt}`; + +export const parseCandidateArtifactName = (name: string): { channel: SiteChannel; sha: string; attempt: number } | null => { + const match = CANDIDATE_NAME.exec(name); + + return match === null ? null : { channel: match[1] as SiteChannel, sha: match[2]!, attempt: Number(match[3]) }; +}; + +/** Why `bytes` do not match the `sha256:` digest GitHub reported, or `null`. */ +export const verifyDigest = (bytes: Buffer, digest: string): string | null => { + if (!/^sha256:[0-9a-f]{64}$/.test(digest)) { + return `artifact digest ${JSON.stringify(digest)} is not a sha256 digest`; + } + + const actual = `sha256:${createHash('sha256').update(bytes).digest('hex')}`; + + return actual === digest ? null : `artifact digest mismatch: GitHub reported ${digest}, the download hashes to ${actual}`; +}; + +export interface ExpectedChannel { + channel: SiteChannel; + sourceSha: string; + /** Also require this exact content, for a snapshot that is being kept. */ + contentHash?: string; +} + +/** Checks a channel tree on disk against its manifest and the identity it must have. */ +export const verifyChannelTree = (dir: string, expected: ExpectedChannel): { problems: string[]; manifest: SiteBuildManifest | null } => { + let manifest: SiteBuildManifest; + + try { + manifest = readSiteBuildManifest(dir); + } catch (error) { + return { problems: [`${SITE_BUILD_FILE}: ${error instanceof Error ? error.message : String(error)}`], manifest: null }; + } + + const problems: string[] = []; + + if (manifest.channel !== expected.channel) { + problems.push(`channel ${manifest.channel} is not the expected ${expected.channel}`); + } + + if (manifest.localTest) { + problems.push(`the tree is a local test build for ${manifest.basePath}`); + } else if (manifest.basePath !== SITE_CHANNEL_BASE[expected.channel]) { + problems.push(`basePath ${manifest.basePath} is not the ${expected.channel} base`); + } + + if (manifest.sourceSha !== expected.sourceSha) { + problems.push(`sourceSha ${manifest.sourceSha} is not the run's commit ${expected.sourceSha}`); + } + + if (expected.contentHash !== undefined && manifest.contentHash !== expected.contentHash) { + problems.push(`contentHash ${manifest.contentHash} is not the published ${expected.contentHash}`); + } + + problems.push(...verifySiteTree(dir, manifest), ...checkTreeSafety(dir)); + + return { problems, manifest }; +}; + +const CHANNEL_LIMITS = { maxUnpackedBytes: SITE_SIZE_REFUSE_BYTES }; + +/** + * Verifies a downloaded candidate zip against its reported digest, unpacks it + * into `outDir` after inspecting every entry, and checks the unpacked tree. + * Throws on any problem. + */ +export const openChannelArchive = (zip: Buffer, options: ExpectedChannel & { digest: string; outDir: string }): { manifest: SiteBuildManifest } => { + const digestProblem = verifyDigest(zip, options.digest); + + if (digestProblem !== null) { + throw new Error(`[channel-artifact] ${digestProblem}`); + } + + extractZip(zip, options.outDir, CHANNEL_LIMITS); + const { problems, manifest } = verifyChannelTree(options.outDir, options); + + if (problems.length > 0 || manifest === null) { + throw new Error(`[channel-artifact] ${options.channel} candidate refused:\n ${problems.join('\n ')}`); + } + + return { manifest }; +}; + +const argument = (name: string): string => { + const index = process.argv.indexOf(name); + const value = index === -1 ? undefined : process.argv[index + 1]; + + if (value === undefined || value.startsWith('--')) { + throw new Error(`[channel-artifact] missing ${name}`); + } + + return value; +}; + +/** + * `verify --dir --channel --sha `: the check CI runs + * on its freshly built tree before uploading it as a candidate. + */ +const main = (): void => { + if (process.argv[2] !== 'verify') { + throw new Error('[channel-artifact] usage: channel-artifact.ts verify --dir --channel --sha '); + } + + const channel = argument('--channel'); + + if (!(SITE_CHANNELS as readonly string[]).includes(channel)) { + throw new Error(`[channel-artifact] unknown channel ${JSON.stringify(channel)}`); + } + + const dir = argument('--dir'); + const { problems, manifest } = verifyChannelTree(dir, { channel: channel as SiteChannel, sourceSha: argument('--sha') }); + const budget = evaluateSiteSize(manifest?.totalBytes ?? 0); + + if (budget.level === 'refuse') { + problems.push(`the ${channel} tree alone: ${budget.message}`); + } else if (budget.level === 'warn') { + console.log(`::warning::The ${channel} site tree alone: ${budget.message}`); + } + + if (problems.length > 0 || manifest === null) { + throw new Error(`[channel-artifact] the ${channel} tree is not a publishable candidate:\n ${problems.join('\n ')}`); + } + + console.log( + `[channel-artifact] ${channel} ${manifest.basePath} ${manifest.sourceSha} ${manifest.packageVersion}: ` + + `${manifest.fileCount} files, ${manifest.totalBytes} bytes, content ${manifest.contentHash}`, + ); +}; + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/scripts/pages/channels-document.ts b/scripts/pages/channels-document.ts new file mode 100644 index 000000000..cd4712721 --- /dev/null +++ b/scripts/pages/channels-document.ts @@ -0,0 +1,219 @@ +import { createHash } from 'node:crypto'; + +import { SITE_CHANNEL_BASE, type SiteChannel } from './site-channel.ts'; +import type { SiteBuildManifest } from './site-manifest.ts'; + +/** + * `channels.json`: what the composed Pages site contains, written by the + * composer at the site root. It records, per channel, the commit, version, + * content hash and base of the deployed tree and the run and artifact its + * bytes came from. + * + * Dependency-free and erasable TypeScript, run by the publisher with plain + * `node`. + */ + +export const CHANNELS_FILE = 'channels.json'; + +export const CHANNEL_PATH: Readonly> = { stable: '', next: 'next/' }; + +/** Where a channel's bytes came from, as GitHub reported it. */ +export interface ChannelSource { + kind: 'ci' | 'release'; + workflowId: number; + runId: number; + runAttempt: number; + artifactId: number; + artifactName: string; + /** `sha256:` of the artifact zip. */ + digest: string; +} + +export interface ChannelEntry { + channel: SiteChannel; + /** Location inside the composed tree: `''` or `'next/'`. */ + path: '' | 'next/'; + basePath: string; + sourceSha: string; + packageVersion: string; + assetProfile: string; + createdAt: string; + contentHash: string; + fileCount: number; + totalBytes: number; + source: ChannelSource; +} + +export interface ChannelsDocument { + schemaVersion: 1; + /** Identifies the pair of channel contents; equal pairings deploy nothing new. */ + pairing: string; + channels: { stable: ChannelEntry; next: ChannelEntry | null }; +} + +interface ChannelRecord { + manifest: SiteBuildManifest; + source: ChannelSource; +} + +const entryFor = (channel: SiteChannel, { manifest, source }: ChannelRecord): ChannelEntry => ({ + channel, + path: CHANNEL_PATH[channel], + basePath: manifest.basePath, + sourceSha: manifest.sourceSha, + packageVersion: manifest.packageVersion, + assetProfile: manifest.assetProfile, + createdAt: manifest.createdAt, + contentHash: manifest.contentHash, + fileCount: manifest.fileCount, + totalBytes: manifest.totalBytes, + source, +}); + +export const pairingOf = (stable: Pick, next: Pick | null): string => + createHash('sha256') + .update(JSON.stringify([stable.sourceSha, stable.contentHash, next?.sourceSha ?? null, next?.contentHash ?? null])) + .digest('hex'); + +export const buildChannelsDocument = (records: { stable: ChannelRecord; next: ChannelRecord | null }): ChannelsDocument => { + const stable = entryFor('stable', records.stable); + const next = records.next === null ? null : entryFor('next', records.next); + + return { schemaVersion: 1, pairing: pairingOf(stable, next), channels: { stable, next } }; +}; + +export const serializeChannelsDocument = (document: ChannelsDocument): string => `${JSON.stringify(document, null, 2)}\n`; + +const HEX64 = /^[0-9a-f]{64}$/; +const SHA1 = /^[0-9a-f]{40}$/; + +const parseSource = (value: unknown, label: string, problems: string[]): ChannelSource | null => { + const raw = (typeof value === 'object' && value !== null ? value : {}) as Record; + + const integer = (key: string): number => { + const field = raw[key]; + + if (typeof field !== 'number' || !Number.isSafeInteger(field) || field <= 0) { + problems.push(`${label}.source.${key} is not a positive integer`); + + return 0; + } + + return field; + }; + + if (raw['kind'] !== 'ci' && raw['kind'] !== 'release') { + problems.push(`${label}.source.kind is not ci or release`); + } + + if (typeof raw['artifactName'] !== 'string' || typeof raw['digest'] !== 'string' || !/^sha256:[0-9a-f]{64}$/.test(raw['digest'])) { + problems.push(`${label}.source needs an artifactName and a sha256 digest`); + } + + const source: ChannelSource = { + kind: raw['kind'] as ChannelSource['kind'], + workflowId: integer('workflowId'), + runId: integer('runId'), + runAttempt: integer('runAttempt'), + artifactId: integer('artifactId'), + artifactName: raw['artifactName'] as string, + digest: raw['digest'] as string, + }; + + return source; +}; + +const parseEntry = (value: unknown, channel: SiteChannel, problems: string[]): ChannelEntry | null => { + if (typeof value !== 'object' || value === null) { + problems.push(`channels.${channel} is not an object`); + + return null; + } + + const raw = value as Record; + const label = `channels.${channel}`; + + if (raw['channel'] !== channel) { + problems.push(`${label}.channel is not ${channel}`); + } + + if (raw['path'] !== CHANNEL_PATH[channel]) { + problems.push(`${label}.path is not ${JSON.stringify(CHANNEL_PATH[channel])}`); + } + + if (raw['basePath'] !== SITE_CHANNEL_BASE[channel]) { + problems.push(`${label}.basePath is not ${SITE_CHANNEL_BASE[channel]}`); + } + + if (typeof raw['sourceSha'] !== 'string' || !SHA1.test(raw['sourceSha'])) { + problems.push(`${label}.sourceSha is not a commit SHA`); + } + + if (typeof raw['contentHash'] !== 'string' || !HEX64.test(raw['contentHash'])) { + problems.push(`${label}.contentHash is not a SHA-256 hex digest`); + } + + for (const key of ['packageVersion', 'assetProfile', 'createdAt']) { + if (typeof raw[key] !== 'string') { + problems.push(`${label}.${key} is not a string`); + } + } + + for (const key of ['fileCount', 'totalBytes']) { + if (typeof raw[key] !== 'number' || !Number.isSafeInteger(raw[key]) || (raw[key] as number) < 0) { + problems.push(`${label}.${key} is not a non-negative integer`); + } + } + + const source = parseSource(raw['source'], label, problems); + + return { + channel, + path: CHANNEL_PATH[channel], + basePath: raw['basePath'] as string, + sourceSha: raw['sourceSha'] as string, + packageVersion: raw['packageVersion'] as string, + assetProfile: raw['assetProfile'] as string, + createdAt: raw['createdAt'] as string, + contentHash: raw['contentHash'] as string, + fileCount: raw['fileCount'] as number, + totalBytes: raw['totalBytes'] as number, + source: source as ChannelSource, + }; +}; + +/** Validates an untrusted, JSON-parsed `channels.json`; throws listing every problem. */ +export const parseChannelsDocument = (value: unknown): ChannelsDocument => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error(`[compose-site] ${CHANNELS_FILE} is not a JSON object`); + } + + const raw = value as Record; + const problems: string[] = []; + const channels = (typeof raw['channels'] === 'object' && raw['channels'] !== null ? raw['channels'] : {}) as Record; + + if (raw['schemaVersion'] !== 1) { + problems.push('schemaVersion is not 1'); + } + + if (typeof raw['pairing'] !== 'string' || !HEX64.test(raw['pairing'])) { + problems.push('pairing is not a SHA-256 hex digest'); + } + + const stable = channels['stable'] == null ? null : parseEntry(channels['stable'], 'stable', problems); + const next = channels['next'] == null ? null : parseEntry(channels['next'], 'next', problems); + + if (stable === null) { + problems.push('channels.stable is missing; Stable is always published'); + } + + if (stable !== null && raw['pairing'] !== pairingOf(stable, next)) { + problems.push('pairing does not match the channel identities'); + } + + if (problems.length > 0) { + throw new Error(`[compose-site] invalid ${CHANNELS_FILE}:\n ${problems.join('\n ')}`); + } + + return { schemaVersion: 1, pairing: raw['pairing'] as string, channels: { stable: stable!, next } }; +}; diff --git a/scripts/pages/compose-site.ts b/scripts/pages/compose-site.ts new file mode 100644 index 000000000..b1064e335 --- /dev/null +++ b/scripts/pages/compose-site.ts @@ -0,0 +1,427 @@ +import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { dirname, extname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { checkTreeSafety, extractTar, readZipFile } from './archive-safety.ts'; +import { type ExpectedChannel, openChannelArchive, verifyChannelTree, verifyDigest } from './channel-artifact.ts'; +import { buildChannelsDocument, CHANNELS_FILE, type ChannelsDocument, type ChannelSource, serializeChannelsDocument } from './channels-document.ts'; +import { createGitHubClient, type GitHubClient } from './github-api.ts'; +import { type ChannelPlan, parseSelection, type Transport } from './select-channels.ts'; +import { evaluateSiteSize, SITE_SIZE_REFUSE_BYTES, type SiteSizeThresholds, type SiteSizeVerdict } from './site-budget.ts'; +import { SITE_CHANNEL_BASE, type SiteChannel } from './site-channel.ts'; +import { readSiteBuildManifest, SITE_BUILD_FILE, type SiteBuildManifest, verifySiteTree } from './site-manifest.ts'; + +export { + buildChannelsDocument, + type ChannelEntry, + CHANNELS_FILE, + type ChannelsDocument, + type ChannelSource, + pairingOf, + parseChannelsDocument, +} from './channels-document.ts'; + +/** + * Joins the Stable and the Next channel tree into the one tree GitHub Pages + * serves: Stable at the root, Next under `next/`. The composer adds + * `channels.json` and `.nojekyll` and nothing else, never runs or rebuilds + * site code, and proves that both channels arrive byte for byte. + * + * Dependency-free and erasable TypeScript, run by the publisher with plain + * `node` from the trusted default-branch checkout. + */ + +export const NOJEKYLL_FILE = '.nojekyll'; + +const COMPOSER_FILES = new Set([CHANNELS_FILE, NOJEKYLL_FILE]); + +export interface CrossChannelReference { + file: string; + url: string; +} + +const URL_ATTRIBUTE = /\b(?:href|src|action|poster|content)=(["'])([^"']*)\1/g; + +/** + * URLs in the HTML of the channel tree at `dir` that point into the other + * channel. The bare base of the other channel is allowed: it is the explicit + * channel switch. + */ +export const findCrossChannelReferences = (dir: string, ownBase: string, otherBase: string): CrossChannelReference[] => { + const hits: CrossChannelReference[] = []; + const nested = ownBase.length > otherBase.length && ownBase.startsWith(otherBase); + + const visit = (directory: string, prefix: string): void => { + for (const entry of readdirSync(directory, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : 1))) { + const rel = prefix === '' ? entry.name : `${prefix}/${entry.name}`; + const abs = join(directory, entry.name); + + if (entry.isDirectory()) { + visit(abs, rel); + continue; + } + + if (extname(entry.name) !== '.html') { + continue; + } + + for (const match of readFileSync(abs, 'utf8').matchAll(URL_ATTRIBUTE)) { + const url = match[2]!; + + if (url.startsWith(otherBase) && url.length > otherBase.length && !(nested && url.startsWith(ownBase))) { + hits.push({ file: rel, url }); + } + } + } + }; + + visit(dir, ''); + + return hits; +}; + +const treeBytes = (root: string): number => { + let bytes = 0; + + const visit = (directory: string): void => { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const abs = join(directory, entry.name); + + if (entry.isDirectory()) { + visit(abs); + } else { + bytes += statSync(abs).size; + } + } + }; + + visit(root); + + return bytes; +}; + +const copyListed = (fromDir: string, toDir: string, manifest: SiteBuildManifest): void => { + for (const path of [...manifest.files.map(file => file.path), SITE_BUILD_FILE]) { + const target = join(toDir, ...path.split('/')); + mkdirSync(dirname(target), { recursive: true }); + copyFileSync(join(fromDir, ...path.split('/')), target); + } +}; + +const isReserved = (path: string): boolean => path.startsWith('next/') || COMPOSER_FILES.has(path); + +export interface ComposeInput { + dir: string; + source: ChannelSource; +} + +export interface ComposeOptions { + outDir: string; + stable: ComposeInput; + next: ComposeInput | null; + budget?: SiteSizeThresholds; +} + +export interface ComposeReport { + problems: string[]; + bytes: number; + budget: SiteSizeVerdict; + channels: ChannelsDocument | null; +} + +const checkInput = (channel: SiteChannel, input: ComposeInput, problems: string[]): SiteBuildManifest | null => { + let manifest: SiteBuildManifest; + + try { + manifest = readSiteBuildManifest(input.dir); + } catch (error) { + problems.push(`${channel}: ${error instanceof Error ? error.message : String(error)}`); + + return null; + } + + if (manifest.channel !== channel) { + problems.push(`${channel}: the tree was built for channel ${manifest.channel}`); + } + + if (manifest.localTest) { + problems.push(`${channel}: the tree is a local test build for ${manifest.basePath}`); + } else if (manifest.basePath !== SITE_CHANNEL_BASE[channel]) { + problems.push(`${channel}: the tree was built for base ${manifest.basePath}, not ${SITE_CHANNEL_BASE[channel]}`); + } + + problems.push(...verifySiteTree(input.dir, manifest).map(problem => `${channel}: ${problem}`)); + + return manifest; +}; + +/** + * Writes the composed tree to `outDir`, which must be empty or absent. Returns + * every problem found; the tree may only be uploaded when there is none. + */ +export const composeSite = (options: ComposeOptions): ComposeReport => { + const outDir = resolve(options.outDir); + + if (existsSync(outDir) && readdirSync(outDir).length > 0) { + throw new Error(`[compose-site] the output directory ${outDir} must be empty`); + } + + const problems: string[] = []; + const stable = checkInput('stable', options.stable, problems); + const next = options.next === null ? null : checkInput('next', options.next, problems); + + for (const file of stable?.files ?? []) { + if (isReserved(file.path)) { + problems.push(`stable: ${file.path} is a reserved path of the composed site`); + } + } + + const empty: ComposeReport = { problems, bytes: 0, budget: evaluateSiteSize(0, options.budget), channels: null }; + + if (problems.length > 0 || stable === null || (options.next !== null && next === null)) { + return empty; + } + + mkdirSync(outDir, { recursive: true }); + copyListed(options.stable.dir, outDir, stable); + + if (next !== null && options.next !== null) { + copyListed(options.next.dir, join(outDir, 'next'), next); + } + + const channels = buildChannelsDocument({ + stable: { manifest: stable, source: options.stable.source }, + next: next === null || options.next === null ? null : { manifest: next, source: options.next.source }, + }); + writeFileSync(join(outDir, NOJEKYLL_FILE), ''); + writeFileSync(join(outDir, CHANNELS_FILE), serializeChannelsDocument(channels)); + + problems.push(...verifySiteTree(outDir, stable, { allowExtra: isReserved }).map(problem => `composed stable: ${problem}`)); + + if (next !== null) { + problems.push(...verifySiteTree(join(outDir, 'next'), next).map(problem => `composed next: ${problem}`)); + + for (const hit of findCrossChannelReferences(join(outDir, 'next'), SITE_CHANNEL_BASE.next, SITE_CHANNEL_BASE.stable)) { + problems.push(`composed next: next/${hit.file} references the Stable channel at ${hit.url}`); + } + } + + for (const hit of findCrossChannelReferences(outDir, SITE_CHANNEL_BASE.stable, SITE_CHANNEL_BASE.next)) { + if (!hit.file.startsWith('next/')) { + problems.push(`composed stable: ${hit.file} references the Next channel at ${hit.url}`); + } + } + + problems.push(...checkTreeSafety(outDir).map(problem => `composed tree: ${problem}`)); + + const bytes = treeBytes(outDir); + const budget = evaluateSiteSize(bytes, options.budget); + + if (budget.level === 'refuse') { + problems.push(`composed tree: ${budget.message}; the upload is refused by the site budget`); + } + + return { problems, bytes, budget, channels }; +}; + +/** + * Copies one channel out of a composed tree into `destDir`, exactly the files + * its `site-build.json` lists. Throws unless the channel is intact. + */ +export const extractComposedChannel = (composedDir: string, path: '' | 'next/', destDir: string): SiteBuildManifest => { + const root = path === '' ? composedDir : join(composedDir, 'next'); + const manifest = readSiteBuildManifest(root); + const problems = verifySiteTree(root, manifest, path === '' ? { allowExtra: isReserved } : {}); + + if (problems.length > 0) { + throw new Error(`[compose-site] the ${path === '' ? 'stable' : 'next'} channel of the snapshot is not intact:\n ${problems.join('\n ')}`); + } + + copyListed(root, destDir, manifest); + + return manifest; +}; + +// A Pages artifact is a zip around one tar of the whole composed tree; the tar +// adds a header block per member on top of the content budget. +const PAGES_CONTAINER_LIMITS = { maxUnpackedBytes: SITE_SIZE_REFUSE_BYTES + 200_000_000 }; + +export interface SnapshotCache { + /** Where unpacked Pages artifacts are kept, one directory per artifact id. */ + dir: string; +} + +export interface FetchedChannel { + dir: string; + manifest: SiteBuildManifest; + source: ChannelSource; + transport: string; +} + +const openPagesSnapshot = async (client: GitHubClient, transport: Extract, cache: SnapshotCache): Promise => { + const dir = join(cache.dir, String(transport.artifactId)); + + if (existsSync(dir)) { + return dir; + } + + const zip = await client.download(`/repos/${client.repository}/actions/artifacts/${transport.artifactId}/zip`); + const digestProblem = verifyDigest(zip, transport.digest); + + if (digestProblem !== null) { + throw new Error(`[compose-site] ${digestProblem}`); + } + + const tar = readZipFile(zip, 'artifact.tar', PAGES_CONTAINER_LIMITS); + extractTar(tar, dir, PAGES_CONTAINER_LIMITS); + + return dir; +}; + +/** + * Fetches the bytes one channel plan names into `dir` and proves they are the + * planned snapshot. A kept channel tries its transports in order and takes + * the first that verifies; throws when none does. + */ +export const fetchChannelSnapshot = async (client: GitHubClient, plan: ChannelPlan, dir: string, cache: SnapshotCache): Promise => { + const downloadCandidate = async (artifactId: number, digest: string, expected: ExpectedChannel): Promise => { + const zip = await client.download(`/repos/${client.repository}/actions/artifacts/${artifactId}/zip`); + + return openChannelArchive(zip, { ...expected, digest, outDir: dir }).manifest; + }; + + if (plan.action === 'absent') { + throw new Error(`[compose-site] ${plan.channel} is absent from the selection`); + } + + if (plan.action === 'update') { + const manifest = await downloadCandidate(plan.source.artifactId, plan.source.digest, { + channel: plan.channel, + sourceSha: plan.sourceSha, + }); + + return { dir, manifest, source: plan.source, transport: `candidate ${plan.source.artifactName}` }; + } + + const failures: string[] = []; + + for (const transport of plan.transports) { + rmSync(dir, { recursive: true, force: true }); + + try { + if (transport.kind === 'snapshot') { + const snapshot = await openPagesSnapshot(client, transport, cache); + mkdirSync(dir, { recursive: true }); + extractComposedChannel(snapshot, transport.path, dir); + const { problems, manifest } = verifyChannelTree(dir, { + channel: plan.channel, + sourceSha: plan.published.sourceSha, + contentHash: plan.published.contentHash, + }); + + if (problems.length > 0 || manifest === null) { + throw new Error(problems.join('; ')); + } + + return { dir, manifest, source: plan.published.source, transport: `published snapshot of run ${transport.runId}` }; + } + + const exact = transport.kind === 'artifact'; + const manifest = await downloadCandidate(transport.artifactId, transport.digest, { + channel: plan.channel, + sourceSha: plan.published.sourceSha, + ...(exact ? { contentHash: plan.published.contentHash } : {}), + }); + const source: ChannelSource = exact + ? plan.published.source + : { + kind: 'release', + workflowId: transport.workflowId, + runId: transport.runId, + runAttempt: transport.runAttempt, + artifactId: transport.artifactId, + artifactName: transport.artifactName, + digest: transport.digest, + }; + + return { dir, manifest, source, transport: `${transport.kind} ${transport.artifactName}` }; + } catch (error) { + failures.push(`${transport.kind} ${transport.artifactId}: ${error instanceof Error ? error.message : String(error)}`); + console.log(`::warning::${plan.channel}: ${failures.at(-1)}`); + } + } + + throw new Error(`[compose-site] no trusted source of the published ${plan.channel} snapshot verified:\n ${failures.join('\n ')}`); +}; + +const argument = (name: string): string => { + const index = process.argv.indexOf(name); + const value = index === -1 ? undefined : process.argv[index + 1]; + + if (value === undefined || value.startsWith('--')) { + throw new Error(`[compose-site] missing ${name}`); + } + + return value; +}; + +/** + * CLI for the publisher: reads the selection `select-channels.ts` wrote, + * fetches and verifies each channel's bytes, composes them into `--out`, and + * copies `channels.json` to `--channels-out` for the small provenance + * artifact. Exits non-zero on any problem, so nothing is uploaded. + */ +const main = async (): Promise => { + const selection = parseSelection(JSON.parse(readFileSync(argument('--selection'), 'utf8')) as unknown); + const work = resolve(argument('--work')); + const outDir = argument('--out'); + const channelsOut = argument('--channels-out'); + + if (!selection.deploy) { + throw new Error(`[compose-site] the selection deploys nothing (${selection.reason}); refusing to compose`); + } + + const client = createGitHubClient(process.env); + const cache: SnapshotCache = { dir: join(work, 'snapshots') }; + rmSync(work, { recursive: true, force: true }); + mkdirSync(work, { recursive: true }); + + const fetchPlan = async (plan: ChannelPlan): Promise => { + if (plan.action === 'absent') { + return null; + } + + const snapshot = await fetchChannelSnapshot(client, plan, join(work, plan.channel), cache); + console.log(`[compose-site] ${plan.channel}: ${plan.action} ${snapshot.manifest.sourceSha} via ${snapshot.transport}`); + + return { dir: snapshot.dir, source: snapshot.source }; + }; + + const stable = await fetchPlan(selection.channels.stable); + const next = await fetchPlan(selection.channels.next); + + if (stable === null) { + throw new Error('[compose-site] Stable is always published; the selection has none'); + } + + const report = composeSite({ outDir, stable, next }); + + if (report.budget.level === 'warn') { + console.log(`::warning::Composed Pages tree: ${report.budget.message}`); + } + + if (report.problems.length > 0 || report.channels === null) { + throw new Error(`[compose-site] the composed tree is refused:\n ${report.problems.join('\n ')}`); + } + + mkdirSync(channelsOut, { recursive: true }); + copyFileSync(join(outDir, CHANNELS_FILE), join(channelsOut, CHANNELS_FILE)); + console.log(`[compose-site] composed ${report.bytes} bytes, pairing ${report.channels.pairing}; ${report.budget.message}`); +}; + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/scripts/pages/github-api.ts b/scripts/pages/github-api.ts new file mode 100644 index 000000000..4df4bf256 --- /dev/null +++ b/scripts/pages/github-api.ts @@ -0,0 +1,90 @@ +/** + * The few GitHub REST calls the Pages publisher makes, with the workflow + * token. The token is only ever sent to the API host: an artifact download + * redirects to blob storage, and that request goes out without it. Errors + * name the status and the API path, never a header. + * + * Dependency-free and erasable TypeScript, run with plain `node`. + */ + +export interface GitHubClient { + /** `owner/name` of the repository the workflow runs in. */ + readonly repository: string; + /** GET an API path; `null` for a 404 when `allowMissing` is set. */ + getJson(path: string, options?: { allowMissing?: boolean }): Promise; + /** GET an API path that answers with a redirect to a download, and return the bytes. */ + download(path: string): Promise; +} + +const REPOSITORY = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; + +export const createGitHubClient = (env: Readonly>): GitHubClient => { + const token = env['GITHUB_TOKEN'] ?? ''; + const repository = env['GITHUB_REPOSITORY'] ?? ''; + const apiUrl = (env['GITHUB_API_URL'] ?? 'https://api.github.com').replace(/\/+$/, ''); + + if (token === '') { + throw new Error('[github-api] GITHUB_TOKEN is not set'); + } + + if (!REPOSITORY.test(repository)) { + throw new Error(`[github-api] GITHUB_REPOSITORY ${JSON.stringify(repository)} is not owner/name`); + } + + if (!apiUrl.startsWith('https://')) { + throw new Error('[github-api] GITHUB_API_URL must be an https URL'); + } + + const headers = { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + }; + + const request = (path: string, redirect: RequestRedirect): Promise => { + if (!path.startsWith('/')) { + throw new Error(`[github-api] API path must start with a slash: ${path}`); + } + + return fetch(`${apiUrl}${path}`, { headers, redirect }); + }; + + return { + repository, + + async getJson(path: string, options: { allowMissing?: boolean } = {}): Promise { + const response = await request(path, 'follow'); + + if (response.status === 404 && options.allowMissing) { + return null; + } + + if (!response.ok) { + throw new Error(`[github-api] GET ${path} answered ${response.status}`); + } + + return (await response.json()) as T; + }, + + async download(path: string): Promise { + const response = await request(path, 'manual'); + let body: Response = response; + + if (response.status >= 300 && response.status < 400) { + const location = response.headers.get('location'); + + if (location === null || !location.startsWith('https://')) { + throw new Error(`[github-api] GET ${path} redirected without an https location`); + } + + body = await fetch(location, { redirect: 'follow' }); + } + + if (!body.ok) { + throw new Error(`[github-api] download of ${path} answered ${body.status}`); + } + + return Buffer.from(await body.arrayBuffer()); + }, + }; +}; diff --git a/scripts/pages/select-channels.ts b/scripts/pages/select-channels.ts new file mode 100644 index 000000000..1501b6bf7 --- /dev/null +++ b/scripts/pages/select-channels.ts @@ -0,0 +1,590 @@ +import { appendFileSync, writeFileSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; + +import { readZipFile } from './archive-safety.ts'; +import { parseCandidateArtifactName } from './channel-artifact.ts'; +import { type ChannelEntry, CHANNELS_FILE, type ChannelsDocument, type ChannelSource, parseChannelsDocument } from './channels-document.ts'; +import { createGitHubClient, type GitHubClient } from './github-api.ts'; +import { SITE_CHANNEL_BRANCH, SITE_CHANNELS, type SiteChannel } from './site-channel.ts'; + +/** + * Decides what the Pages publisher deploys, from GitHub's own metadata. + * + * The run that triggered the publisher is only a wake-up call. Every start + * re-decides both channels: each gets the newest qualified candidate on its + * branch's current history, or keeps the snapshot that is already published. + * A queued or re-run event can therefore neither roll a channel back nor + * drop the other channel's update. + * + * Dependency-free and erasable TypeScript, run by the publisher with plain + * `node` from the trusted default-branch checkout. + */ + +export const CI_WORKFLOW_PATH = '.github/workflows/ci.yml'; +export const RELEASE_WORKFLOW_PATH = '.github/workflows/release.yml'; + +/** The small artifact a successful publisher run leaves its `channels.json` in. */ +export const channelsArtifactName = (runId: number): string => `pages-channels-${runId}`; + +export const PAGES_ARTIFACT_NAME = 'github-pages'; + +const STABLE_RELEASE_TAG = /^v\d+\.\d+\.\d+$/; +const DIGEST = /^sha256:[0-9a-f]{64}$/; + +export interface WorkflowRunMeta { + id: number; + attempt: number; + workflowId: number; + workflowPath: string; + event: string; + status: string; + conclusion: string | null; + /** Branch, or tag for a tag push. */ + headBranch: string | null; + headSha: string; + /** `owner/name` the run belongs to. */ + repository: string; + /** `owner/name` its commit came from; differs for a fork. */ + headRepository: string | null; +} + +export interface ArtifactMeta { + id: number; + name: string; + expired: boolean; + digest: string | null; + sizeInBytes: number; + runId: number; + runHeadSha: string | null; +} + +export interface CandidateRun { + run: WorkflowRunMeta; + artifacts: readonly ArtifactMeta[]; +} + +export interface SelectionPolicy { + repository: string; + ciWorkflowId: number; + releaseWorkflowId: number; +} + +export interface PreviousPublication { + /** The successful publisher run that deployed `channels`. */ + runId: number; + channels: ChannelsDocument; + /** That run's Pages artifact, the composed snapshot. */ + pagesArtifact: ArtifactMeta | null; +} + +export interface SelectionInput { + policy: SelectionPolicy; + trigger: WorkflowRunMeta; + branchHeads: Record; + /** + * Per channel, how many commits its branch head is ahead of a commit, or + * `null` when the commit is not on the branch's current history. A commit + * missing from the map counts as not on it. + */ + history: Record>>; + ciRuns: readonly CandidateRun[]; + releaseRuns: readonly CandidateRun[]; + previous: PreviousPublication | null; +} + +export type Transport = + | { kind: 'artifact'; runId: number; artifactId: number; artifactName: string; digest: string } + | { kind: 'snapshot'; runId: number; artifactId: number; digest: string; path: '' | 'next/' } + | { kind: 'release'; workflowId: number; runId: number; runAttempt: number; artifactId: number; artifactName: string; digest: string }; + +export type ChannelPlan = + | { channel: SiteChannel; action: 'update'; sourceSha: string; source: ChannelSource; reason: string } + | { channel: SiteChannel; action: 'keep'; published: ChannelEntry; transports: Transport[]; reason: string } + | { channel: SiteChannel; action: 'absent'; reason: string }; + +export interface Selection { + deploy: boolean; + reason: string; + /** Reasons nothing may be deployed. */ + problems: string[]; + /** Candidates passed over, for the job log. */ + notes: string[]; + channels: Record; +} + +const sameRepository = (run: WorkflowRunMeta, policy: SelectionPolicy): boolean => + run.repository === policy.repository && run.headRepository === policy.repository; + +const completedSuccessfully = (run: WorkflowRunMeta): boolean => run.status === 'completed' && run.conclusion === 'success'; + +const isTrustedCiPush = (run: WorkflowRunMeta, policy: SelectionPolicy): boolean => + sameRepository(run, policy) && + completedSuccessfully(run) && + run.workflowId === policy.ciWorkflowId && + run.workflowPath === CI_WORKFLOW_PATH && + run.event === 'push' && + SITE_CHANNELS.some(channel => SITE_CHANNEL_BRANCH[channel] === run.headBranch); + +const isTrustedRelease = (run: WorkflowRunMeta, policy: SelectionPolicy): boolean => + sameRepository(run, policy) && + completedSuccessfully(run) && + run.workflowId === policy.releaseWorkflowId && + run.workflowPath === RELEASE_WORKFLOW_PATH && + run.event === 'push' && + STABLE_RELEASE_TAG.test(run.headBranch ?? ''); + +interface Qualified { + kind: 'ci' | 'release'; + run: WorkflowRunMeta; + artifact: ArtifactMeta; + attempt: number; + distance: number; +} + +/** + * The run's own candidate for `channel`, or a reason it has none. A run that + * carries any site artifact contradicting its own metadata is disqualified. + */ +const candidateOf = (entry: CandidateRun, channel: SiteChannel): { artifact: ArtifactMeta; attempt: number } | { note: string } | null => { + const { run } = entry; + let best: { artifact: ArtifactMeta; attempt: number } | null = null; + + for (const artifact of entry.artifacts) { + const parsed = parseCandidateArtifactName(artifact.name); + + if (parsed === null) { + continue; + } + + if ( + parsed.channel !== channel || + parsed.sha !== run.headSha || + parsed.attempt > run.attempt || + artifact.runId !== run.id || + (artifact.runHeadSha !== null && artifact.runHeadSha !== run.headSha) + ) { + return { note: `run ${run.id} carries ${artifact.name}, which contradicts its ${channel} metadata at ${run.headSha}` }; + } + + if (artifact.expired || artifact.digest === null || !DIGEST.test(artifact.digest)) { + continue; + } + + if (best === null || parsed.attempt > best.attempt) { + best = { artifact, attempt: parsed.attempt }; + } + } + + return best; +}; + +const sourceOf = (candidate: Qualified): ChannelSource => ({ + kind: candidate.kind, + workflowId: candidate.run.workflowId, + runId: candidate.run.id, + runAttempt: candidate.run.attempt, + artifactId: candidate.artifact.id, + artifactName: candidate.artifact.name, + digest: candidate.artifact.digest!, +}); + +const rank = (a: Qualified, b: Qualified): number => + a.distance - b.distance || (a.kind === b.kind ? 0 : a.kind === 'ci' ? -1 : 1) || b.attempt - a.attempt || b.run.id - a.run.id; + +const qualify = (input: SelectionInput, channel: SiteChannel, notes: string[]): Qualified[] => { + const qualified: Qualified[] = []; + + const consider = (kind: 'ci' | 'release', entry: CandidateRun): void => { + const found = candidateOf(entry, channel); + + if (found === null) { + return; + } + + if ('note' in found) { + notes.push(found.note); + + return; + } + + const distance = input.history[channel][entry.run.headSha]; + + if (distance === null || distance === undefined) { + notes.push(`${channel}: ${found.artifact.name} is not on the current ${SITE_CHANNEL_BRANCH[channel]} history`); + + return; + } + + qualified.push({ kind, run: entry.run, artifact: found.artifact, attempt: found.attempt, distance }); + }; + + for (const entry of input.ciRuns) { + if (isTrustedCiPush(entry.run, input.policy) && entry.run.headBranch === SITE_CHANNEL_BRANCH[channel]) { + consider('ci', entry); + } + } + + if (channel === 'stable') { + for (const entry of input.releaseRuns) { + if (isTrustedRelease(entry.run, input.policy)) { + consider('release', entry); + } + } + } + + return qualified.sort(rank); +}; + +const transportsFor = (input: SelectionInput, published: ChannelEntry, qualified: readonly Qualified[]): Transport[] => { + const transports: Transport[] = []; + const original = [...input.ciRuns, ...input.releaseRuns] + .filter(entry => entry.run.id === published.source.runId && sameRepository(entry.run, input.policy)) + .flatMap(entry => entry.artifacts) + .find(artifact => artifact.id === published.source.artifactId && !artifact.expired && artifact.digest === published.source.digest); + + if (original !== undefined) { + transports.push({ + kind: 'artifact', + runId: original.runId, + artifactId: original.id, + artifactName: original.name, + digest: original.digest!, + }); + } + + const pages = input.previous?.pagesArtifact ?? null; + + if (pages !== null && !pages.expired && pages.digest !== null && DIGEST.test(pages.digest)) { + transports.push({ kind: 'snapshot', runId: input.previous!.runId, artifactId: pages.id, digest: pages.digest, path: published.path }); + } + + for (const candidate of qualified) { + if (candidate.kind === 'release' && candidate.run.headSha === published.sourceSha && candidate.artifact.id !== original?.id) { + const source = sourceOf(candidate); + transports.push({ ...source, kind: 'release' }); + } + } + + return transports; +}; + +const planChannel = (input: SelectionInput, channel: SiteChannel, notes: string[]): ChannelPlan => { + const qualified = qualify(input, channel, notes); + const newest = qualified[0] ?? null; + const published = input.previous?.channels.channels[channel] ?? null; + + if (published === null) { + if (newest === null) { + return { channel, action: 'absent', reason: `no qualified ${channel} candidate and none published` }; + } + + return { channel, action: 'update', sourceSha: newest.run.headSha, source: sourceOf(newest), reason: `first ${channel} publication` }; + } + + const publishedDistance = input.history[channel][published.sourceSha]; + const publishedOnHistory = publishedDistance !== null && publishedDistance !== undefined; + + if (newest !== null && (!publishedOnHistory || newest.distance < publishedDistance)) { + return { + channel, + action: 'update', + sourceSha: newest.run.headSha, + source: sourceOf(newest), + reason: publishedOnHistory + ? `${newest.run.headSha} is ${publishedDistance - newest.distance} commit(s) ahead of the published ${published.sourceSha}` + : `the published ${published.sourceSha} is no longer on the ${SITE_CHANNEL_BRANCH[channel]} history`, + }; + } + + return { + channel, + action: 'keep', + published, + transports: transportsFor(input, published, qualified), + reason: newest === null ? 'no qualified candidate' : `no candidate newer than the published ${published.sourceSha}`, + }; +}; + +/** The publisher's decision for both channels; pure, see `SelectionInput`. */ +export const selectChannels = (input: SelectionInput): Selection => { + const notes: string[] = []; + const trigger = input.trigger; + + if (!isTrustedCiPush(trigger, input.policy) && !isTrustedRelease(trigger, input.policy)) { + return { + deploy: false, + reason: `untrusted trigger: run ${trigger.id} (${trigger.event} on ${String(trigger.headBranch)}, ${String(trigger.conclusion)}) is not a successful same-repository CI push to a channel branch or a stable Release tag`, + problems: [], + notes, + channels: { + stable: { channel: 'stable', action: 'absent', reason: 'not evaluated' }, + next: { channel: 'next', action: 'absent', reason: 'not evaluated' }, + }, + }; + } + + const problems: string[] = []; + + for (const channel of SITE_CHANNELS) { + if (input.branchHeads[channel] === null) { + problems.push(`the ${SITE_CHANNEL_BRANCH[channel]} branch head could not be resolved`); + } + } + + const channels = { stable: planChannel(input, 'stable', notes), next: planChannel(input, 'next', notes) }; + + if (channels.stable.action === 'absent') { + problems.push('no qualified stable snapshot exists; Stable is always required'); + } + + const changed = input.previous === null || SITE_CHANNELS.some(channel => channels[channel].action === 'update'); + + if (problems.length === 0 && !changed) { + return { deploy: false, reason: 'unchanged pairing: both channels are already published', problems, notes, channels }; + } + + for (const channel of SITE_CHANNELS) { + const plan = channels[channel]; + + if (plan.action === 'keep' && plan.transports.length === 0) { + problems.push(`${channel}: no trusted source of the published snapshot ${plan.published.sourceSha} is left`); + } + } + + if (problems.length > 0) { + return { deploy: false, reason: 'refused', problems, notes, channels }; + } + + return { deploy: true, reason: input.previous === null ? 'first publication' : 'channel update', problems, notes, channels }; +}; + +/** Re-reads a selection the same job wrote; throws on anything malformed. */ +export const parseSelection = (value: unknown): Selection => { + const raw = (typeof value === 'object' && value !== null ? value : {}) as Partial; + const channels = raw.channels as Partial> | undefined; + + if (typeof raw.deploy !== 'boolean' || typeof raw.reason !== 'string' || channels === undefined) { + throw new Error('[select-channels] the selection file is malformed'); + } + + for (const channel of SITE_CHANNELS) { + const plan = channels[channel]; + + if (plan === undefined || plan.channel !== channel || !['update', 'keep', 'absent'].includes(plan.action)) { + throw new Error(`[select-channels] the selection has no valid ${channel} plan`); + } + } + + return raw as Selection; +}; + +interface ApiRun { + id: number; + run_attempt: number; + workflow_id: number; + path: string; + event: string; + status: string; + conclusion: string | null; + head_branch: string | null; + head_sha: string; + repository: { full_name: string }; + head_repository: { full_name: string } | null; +} + +interface ApiArtifact { + id: number; + name: string; + expired: boolean; + digest?: string | null; + size_in_bytes: number; + workflow_run?: { id?: number; head_sha?: string } | null; +} + +const toRun = (run: ApiRun): WorkflowRunMeta => ({ + id: run.id, + attempt: run.run_attempt, + workflowId: run.workflow_id, + // A run's `path` may carry an `@ref` suffix for reusable or dynamic workflows. + workflowPath: run.path.replace(/@.*$/, ''), + event: run.event, + status: run.status, + conclusion: run.conclusion, + headBranch: run.head_branch, + headSha: run.head_sha, + repository: run.repository.full_name, + headRepository: run.head_repository?.full_name ?? null, +}); + +const toArtifact = (artifact: ApiArtifact, runId: number): ArtifactMeta => ({ + id: artifact.id, + name: artifact.name, + expired: artifact.expired, + digest: artifact.digest ?? null, + sizeInBytes: artifact.size_in_bytes, + runId: artifact.workflow_run?.id ?? runId, + runHeadSha: artifact.workflow_run?.head_sha ?? null, +}); + +const listArtifacts = async (client: GitHubClient, runId: number): Promise => { + const page = await client.getJson<{ artifacts: ApiArtifact[] }>(`/repos/${client.repository}/actions/runs/${runId}/artifacts?per_page=100`); + + return (page?.artifacts ?? []).map(artifact => toArtifact(artifact, runId)); +}; + +const listRuns = async (client: GitHubClient, workflowId: number, query: string): Promise => { + const page = await client.getJson<{ workflow_runs: ApiRun[] }>(`/repos/${client.repository}/actions/workflows/${workflowId}/runs?${query}`); + + return (page?.workflow_runs ?? []).map(toRun); +}; + +const withArtifacts = async (client: GitHubClient, runs: readonly WorkflowRunMeta[]): Promise => + Promise.all(runs.map(async run => ({ run, artifacts: await listArtifacts(client, run.id) }))); + +const findPrevious = async (client: GitHubClient, publisherWorkflowId: number, currentRunId: number): Promise => { + for (const run of await listRuns(client, publisherWorkflowId, 'status=success&per_page=30')) { + if (run.id === currentRunId) { + continue; + } + + const artifacts = await listArtifacts(client, run.id); + const record = artifacts.find(artifact => artifact.name === channelsArtifactName(run.id) && !artifact.expired); + + if (record === undefined) { + continue; + } + + const zip = await client.download(`/repos/${client.repository}/actions/artifacts/${record.id}/zip`); + const channels = parseChannelsDocument(JSON.parse(readZipFile(zip, CHANNELS_FILE, { maxUnpackedBytes: 1_000_000 }).toString('utf8')) as unknown); + + return { runId: run.id, channels, pagesArtifact: artifacts.find(artifact => artifact.name === PAGES_ARTIFACT_NAME) ?? null }; + } + + return null; +}; + +const distanceTo = async (client: GitHubClient, sha: string, head: string): Promise => { + const comparison = await client.getJson<{ status: string; ahead_by: number }>(`/repos/${client.repository}/compare/${sha}...${head}?per_page=1`, { + allowMissing: true, + }); + + if (comparison === null) { + return null; + } + + if (comparison.status === 'identical') { + return 0; + } + + return comparison.status === 'ahead' ? comparison.ahead_by : null; +}; + +const requireEnv = (name: string): string => { + const value = process.env[name] ?? ''; + + if (value === '') { + throw new Error(`[select-channels] ${name} is not set`); + } + + return value; +}; + +/** + * CLI for the publisher job: gathers the metadata `selectChannels` needs from + * the GitHub API, writes the selection to `--out`, and `deploy=true|false` + * to `$GITHUB_OUTPUT`. + */ +const main = async (): Promise => { + const outIndex = process.argv.indexOf('--out'); + const out = outIndex === -1 ? undefined : process.argv[outIndex + 1]; + + if (out === undefined) { + throw new Error('[select-channels] usage: select-channels.ts --out '); + } + + const client = createGitHubClient(process.env); + const repo = `/repos/${client.repository}`; + const triggerId = Number(requireEnv('TRIGGER_RUN_ID')); + const currentId = Number(requireEnv('GITHUB_RUN_ID')); + const ci = await client.getJson<{ id: number }>(`${repo}/actions/workflows/ci.yml`); + const release = await client.getJson<{ id: number }>(`${repo}/actions/workflows/release.yml`); + const current = await client.getJson(`${repo}/actions/runs/${currentId}`); + const trigger = await client.getJson(`${repo}/actions/runs/${triggerId}`); + + if (ci === null || release === null || current === null || trigger === null) { + throw new Error('[select-channels] the workflow metadata is incomplete'); + } + + const policy: SelectionPolicy = { repository: client.repository, ciWorkflowId: ci.id, releaseWorkflowId: release.id }; + const branchHeads = {} as Record; + + for (const channel of SITE_CHANNELS) { + const branch = await client.getJson<{ commit: { sha: string } }>(`${repo}/branches/${SITE_CHANNEL_BRANCH[channel]}`, { + allowMissing: true, + }); + branchHeads[channel] = branch?.commit.sha ?? null; + } + + const ciRuns: CandidateRun[] = []; + + for (const channel of SITE_CHANNELS) { + const runs = await listRuns(client, ci.id, `branch=${SITE_CHANNEL_BRANCH[channel]}&event=push&status=success&per_page=25`); + ciRuns.push(...(await withArtifacts(client, runs))); + } + + const releaseRuns = await withArtifacts(client, await listRuns(client, release.id, 'event=push&status=success&per_page=10')); + const previous = await findPrevious(client, current.workflow_id, currentId); + const history = { stable: {}, next: {} } as Record>; + + for (const channel of SITE_CHANNELS) { + const head = branchHeads[channel]; + + if (head === null) { + continue; + } + + const shas = new Set([ + ...ciRuns.filter(entry => entry.run.headBranch === SITE_CHANNEL_BRANCH[channel]).map(entry => entry.run.headSha), + ...(channel === 'stable' ? releaseRuns.map(entry => entry.run.headSha) : []), + ...(previous?.channels.channels[channel] ? [previous.channels.channels[channel].sourceSha] : []), + ]); + + for (const sha of shas) { + history[channel][sha] = /^[0-9a-f]{40}$/.test(sha) ? await distanceTo(client, sha, head) : null; + } + } + + const selection = selectChannels({ policy, trigger: toRun(trigger), branchHeads, history, ciRuns, releaseRuns, previous }); + writeFileSync(out, `${JSON.stringify(selection, null, 2)}\n`); + + for (const channel of SITE_CHANNELS) { + const plan = selection.channels[channel]; + const detail = plan.action === 'update' ? plan.sourceSha : plan.action === 'keep' ? plan.published.sourceSha : ''; + console.log(`[select-channels] ${channel}: ${plan.action} ${detail} (${plan.reason})`); + } + + for (const note of selection.notes) { + console.log(`[select-channels] note: ${note}`); + } + + for (const problem of selection.problems) { + console.log(`::error::${problem}`); + } + + console.log(`[select-channels] deploy=${selection.deploy}: ${selection.reason}`); + + if (process.env['GITHUB_OUTPUT']) { + appendFileSync(process.env['GITHUB_OUTPUT'], `deploy=${selection.deploy}\n`); + } + + if (selection.problems.length > 0) { + process.exitCode = 1; + } +}; + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/scripts/pages/site-budget.ts b/scripts/pages/site-budget.ts new file mode 100644 index 000000000..929e6d4b4 --- /dev/null +++ b/scripts/pages/site-budget.ts @@ -0,0 +1,38 @@ +/** + * The project's budget for the published Pages tree, measured in unpacked + * bytes on disk. GitHub supports published sites up to 1 GB; these thresholds + * leave headroom below that and are a project decision, not a GitHub limit. + * A compressed artifact size says nothing about either. + */ + +export const SITE_SIZE_WARN_BYTES = 750_000_000; + +export const SITE_SIZE_REFUSE_BYTES = 900_000_000; + +export interface SiteSizeThresholds { + warn: number; + refuse: number; +} + +export interface SiteSizeVerdict { + level: 'ok' | 'warn' | 'refuse'; + bytes: number; + message: string; +} + +export const evaluateSiteSize = ( + bytes: number, + thresholds: SiteSizeThresholds = { warn: SITE_SIZE_WARN_BYTES, refuse: SITE_SIZE_REFUSE_BYTES }, +): SiteSizeVerdict => { + const amount = `${bytes.toLocaleString('en-US')} unpacked bytes`; + + if (bytes >= thresholds.refuse) { + return { level: 'refuse', bytes, message: `${amount} reaches the ${thresholds.refuse.toLocaleString('en-US')}-byte refusal budget` }; + } + + if (bytes >= thresholds.warn) { + return { level: 'warn', bytes, message: `${amount} reaches the ${thresholds.warn.toLocaleString('en-US')}-byte warning budget` }; + } + + return { level: 'ok', bytes, message: `${amount}, below the ${thresholds.warn.toLocaleString('en-US')}-byte warning budget` }; +}; diff --git a/scripts/pages/site-channel.ts b/scripts/pages/site-channel.ts new file mode 100644 index 000000000..0166e9c55 --- /dev/null +++ b/scripts/pages/site-channel.ts @@ -0,0 +1,133 @@ +import { pathToFileURL } from 'node:url'; + +/** + * The site channel build contract. + * + * One Pages project site carries two channels: Stable at `/ExoJS/` and Next at + * `/ExoJS/next/`. A build selects its channel with `EXOJS_SITE_CHANNEL` and may + * name its base with `EXOJS_SITE_BASE`; the base ends up in every URL the + * build emits, so only the canonical pairs are accepted. A custom base exists + * for local experiments only (`EXOJS_SITE_LOCAL_TEST=1`), is refused on CI and + * is recorded in `site-build.json` so the publisher can refuse it as well. + * + * Dependency-free and erasable TypeScript: the Astro config, the site build + * and the Pages publisher (which runs with plain `node` and no install) all + * read this one table. + */ + +export type SiteChannel = 'stable' | 'next'; + +export const SITE_CHANNELS: readonly SiteChannel[] = ['stable', 'next']; + +export const SITE_CHANNEL_BASE: Readonly> = { stable: '/ExoJS/', next: '/ExoJS/next/' }; + +/** The branch whose pushes publish each channel. */ +export const SITE_CHANNEL_BRANCH: Readonly> = { stable: 'main', next: 'next' }; + +export interface SiteChannelConfig { + readonly channel: SiteChannel; + readonly basePath: string; + /** A non-canonical base accepted for a local experiment; never publishable. */ + readonly localTest: boolean; +} + +type Env = Readonly>; + +const SEGMENT = /^[A-Za-z0-9_-][A-Za-z0-9._-]*$/; + +/** + * A base path that can be interpolated into a URL, an HTML attribute or a + * shell word without escaping: `/`, or `/seg/.../` of letters, digits, `.`, + * `_` and `-`, with no empty, `.` or `..` segment. + */ +export const isSafeBasePath = (value: string): boolean => { + if (value === '/') { + return true; + } + + if (!value.startsWith('/') || !value.endsWith('/')) { + return false; + } + + return value + .slice(1, -1) + .split('/') + .every(segment => SEGMENT.test(segment) && segment !== '.' && segment !== '..'); +}; + +const isChannel = (value: string): value is SiteChannel => (SITE_CHANNELS as readonly string[]).includes(value); + +const channelForBase = (base: string): SiteChannel | null => SITE_CHANNELS.find(channel => SITE_CHANNEL_BASE[channel] === base) ?? null; + +/** + * Resolves the channel a site build targets from its environment. Unset keeps + * the Stable site at `/ExoJS/`. Throws on an unknown channel, a contradictory + * pair, or a non-canonical base outside explicit local test mode. + */ +export const resolveSiteChannel = (env: Env): SiteChannelConfig => { + const rawChannel = env['EXOJS_SITE_CHANNEL'] ?? ''; + const rawBase = env['EXOJS_SITE_BASE'] ?? ''; + const localTest = env['EXOJS_SITE_LOCAL_TEST'] === '1'; + + if (rawChannel !== '' && !isChannel(rawChannel)) { + throw new Error(`EXOJS_SITE_CHANNEL must be one of ${SITE_CHANNELS.join(', ')}; got ${JSON.stringify(rawChannel)}.`); + } + + if (rawBase === '') { + const channel = rawChannel === '' ? 'stable' : rawChannel; + + return { channel, basePath: SITE_CHANNEL_BASE[channel], localTest: false }; + } + + if (!isSafeBasePath(rawBase)) { + throw new Error(`EXOJS_SITE_BASE must look like /segment/.../ with safe segments; got ${JSON.stringify(rawBase)}.`); + } + + const canonicalChannel = channelForBase(rawBase); + + if (canonicalChannel !== null) { + if (rawChannel !== '' && rawChannel !== canonicalChannel) { + throw new Error(`EXOJS_SITE_BASE ${rawBase} is the ${canonicalChannel} base and contradicts EXOJS_SITE_CHANNEL=${rawChannel}.`); + } + + return { channel: canonicalChannel, basePath: rawBase, localTest: false }; + } + + if (!localTest) { + throw new Error( + `EXOJS_SITE_BASE ${rawBase} is not a published base (${SITE_CHANNELS.map(channel => SITE_CHANNEL_BASE[channel]).join(', ')}); ` + + 'set EXOJS_SITE_LOCAL_TEST=1 to build it for a local experiment.', + ); + } + + if (env['GITHUB_ACTIONS'] === 'true' || env['CI'] === 'true') { + throw new Error('EXOJS_SITE_LOCAL_TEST is refused on CI: a CI site build must target a published base.'); + } + + if (rawChannel === '') { + throw new Error('EXOJS_SITE_LOCAL_TEST needs an explicit EXOJS_SITE_CHANNEL for its custom base.'); + } + + return { channel: rawChannel, basePath: rawBase, localTest: true }; +}; + +/** The channel a push to `branch` publishes, or `null` for any other ref. */ +export const channelForBranch = (branch: string): SiteChannel | null => SITE_CHANNELS.find(channel => SITE_CHANNEL_BRANCH[channel] === branch) ?? null; + +/** Where an explicit channel switch leads; a local test build has no counterpart. */ +export const counterpartBasePath = (config: SiteChannelConfig): string | null => + config.localTest ? null : SITE_CHANNEL_BASE[config.channel === 'stable' ? 'next' : 'stable']; + +/** + * `$GITHUB_OUTPUT` lines for building `ref`: a long-lived branch builds its own + * channel, every other ref the unconfigured default. + */ +export const siteChannelOutputs = (ref: string): string[] => { + const channel = channelForBranch(ref) ?? 'stable'; + + return [`channel=${channel}`, `base=${SITE_CHANNEL_BASE[channel]}`]; +}; + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + process.stdout.write(`${siteChannelOutputs(process.env['SITE_REF'] ?? '').join('\n')}\n`); +} diff --git a/scripts/pages/site-manifest.ts b/scripts/pages/site-manifest.ts new file mode 100644 index 000000000..06135ff0b --- /dev/null +++ b/scripts/pages/site-manifest.ts @@ -0,0 +1,359 @@ +import { createHash } from 'node:crypto'; +import { lstatSync, readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { isSafeBasePath, SITE_CHANNEL_BASE, SITE_CHANNELS, type SiteChannel } from './site-channel.ts'; + +/** + * `site-build.json`: the identity and the exact content of one built channel + * tree. It sits at the tree root and is not part of its own file list. + * + * Dependency-free and erasable TypeScript, so the Pages publisher can run it + * with plain `node`. + */ + +export const SITE_BUILD_FILE = 'site-build.json'; + +export const SITE_BUILD_SCHEMA_VERSION = 1; + +/** + * Which demo media profile the build ships. `standard` is the + * `@codexo/exojs-demo-assets` standard profile served under + * `vendor/exojs-demo-assets/media/`; the extended media are not part of a site. + */ +export const SITE_ASSET_PROFILES = ['standard'] as const; + +export type SiteAssetProfile = (typeof SITE_ASSET_PROFILES)[number]; + +export interface SiteBuildIdentity { + channel: SiteChannel; + basePath: string; + /** The commit the tree was built from. */ + sourceSha: string; + /** The root package version the tree was built from, unreleased or not. */ + packageVersion: string; + assetProfile: SiteAssetProfile; + /** ISO 8601 UTC build time. */ + createdAt: string; + /** Built for a non-canonical base; the publisher refuses it. */ + localTest: boolean; +} + +export interface SiteBuildFileEntry { + path: string; + size: number; + sha256: string; +} + +export interface SiteBuildManifest extends SiteBuildIdentity { + schemaVersion: typeof SITE_BUILD_SCHEMA_VERSION; + fileCount: number; + totalBytes: number; + /** SHA-256 over the sorted file list; see `computeContentHash`. */ + contentHash: string; + files: SiteBuildFileEntry[]; +} + +const SHA1_HEX = /^[0-9a-f]{40}$/; +const SHA256_HEX = /^[0-9a-f]{64}$/; +const SEMVER = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/; +const ISO_UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z$/; +// Control characters, the Windows separator and the drive/stream separator. +// eslint-disable-next-line no-control-regex +const FORBIDDEN_PATH_CHARACTERS = /[\u0000-\u001F\u007F\\:]/; + +/** + * Why `path` cannot name a file in a site tree, or `null` when it can. A site + * path is relative, POSIX-separated and has no empty, `.` or `..` segment, + * so it can neither escape the tree nor alias another entry. + */ +export const validateSitePath = (path: string): string | null => { + if (path === '') { + return 'empty path'; + } + + if (path.startsWith('/')) { + return `absolute path ${JSON.stringify(path)}`; + } + + if (FORBIDDEN_PATH_CHARACTERS.test(path)) { + return `path with a control character, backslash or colon ${JSON.stringify(path)}`; + } + + for (const segment of path.split('/')) { + if (segment === '' || segment === '.' || segment === '..') { + return `path with an empty, '.' or '..' segment ${JSON.stringify(path)}`; + } + } + + return null; +}; + +const compareCodeUnits = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0); + +const sha256 = (data: Buffer | string): string => createHash('sha256').update(data).digest('hex'); + +/** + * Every regular file below `root`, sorted by code unit, with size and SHA-256. + * Throws on a symbolic link or any other non-regular entry, and on a name that + * is not a valid site path. `exclude` receives root-relative paths. + */ +export const hashTreeFiles = (root: string, exclude: (path: string) => boolean = () => false): SiteBuildFileEntry[] => { + const files: SiteBuildFileEntry[] = []; + + const visit = (directory: string, prefix: string): void => { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const rel = prefix === '' ? entry.name : `${prefix}/${entry.name}`; + const problem = validateSitePath(rel); + + if (problem !== null) { + throw new Error(`[site-manifest] ${problem}`); + } + + const abs = join(directory, entry.name); + const stats = lstatSync(abs); + + if (stats.isSymbolicLink()) { + throw new Error(`[site-manifest] symlink in the site tree: ${rel}`); + } + + if (stats.isDirectory()) { + visit(abs, rel); + continue; + } + + if (!stats.isFile()) { + throw new Error(`[site-manifest] special file in the site tree: ${rel}`); + } + + if (exclude(rel)) { + continue; + } + + files.push({ path: rel, size: stats.size, sha256: sha256(readFileSync(abs)) }); + } + }; + + visit(root, ''); + + return files.sort((a, b) => compareCodeUnits(a.path, b.path)); +}; + +/** SHA-256 over one ` \n` line per file, in list order. */ +export const computeContentHash = (files: readonly SiteBuildFileEntry[]): string => + sha256(files.map(file => `${file.sha256} ${file.size} ${file.path}\n`).join('')); + +const isRootManifest = (path: string): boolean => path === SITE_BUILD_FILE; + +const identityProblems = (identity: SiteBuildIdentity): string[] => { + const problems: string[] = []; + + if (!(SITE_CHANNELS as readonly string[]).includes(identity.channel)) { + problems.push(`channel ${JSON.stringify(identity.channel)} is not a site channel`); + } + + if (typeof identity.basePath !== 'string' || !isSafeBasePath(identity.basePath)) { + problems.push(`basePath ${JSON.stringify(identity.basePath)} is not a safe base path`); + } else if (typeof identity.localTest !== 'boolean') { + problems.push('localTest must be a boolean'); + } else if (!identity.localTest && SITE_CHANNEL_BASE[identity.channel] !== identity.basePath) { + problems.push(`basePath ${identity.basePath} is not the ${identity.channel} base ${SITE_CHANNEL_BASE[identity.channel]}`); + } + + if (typeof identity.sourceSha !== 'string' || !SHA1_HEX.test(identity.sourceSha)) { + problems.push(`sourceSha ${JSON.stringify(identity.sourceSha)} is not a full commit SHA`); + } + + if (typeof identity.packageVersion !== 'string' || !SEMVER.test(identity.packageVersion)) { + problems.push(`packageVersion ${JSON.stringify(identity.packageVersion)} is not a semantic version`); + } + + if (!(SITE_ASSET_PROFILES as readonly string[]).includes(identity.assetProfile)) { + problems.push(`assetProfile ${JSON.stringify(identity.assetProfile)} is not one of ${SITE_ASSET_PROFILES.join(', ')}`); + } + + if (typeof identity.createdAt !== 'string' || !ISO_UTC.test(identity.createdAt) || Number.isNaN(Date.parse(identity.createdAt))) { + problems.push(`createdAt ${JSON.stringify(identity.createdAt)} is not an ISO 8601 UTC time`); + } + + return problems; +}; + +/** Throws when `identity` could not describe a build of the declared channel. */ +export const assertSiteBuildIdentity = (identity: SiteBuildIdentity): void => { + const problems = identityProblems(identity); + + if (problems.length > 0) { + throw new Error(`[site-manifest] invalid site build identity:\n ${problems.join('\n ')}`); + } +}; + +/** The manifest of the tree at `root`, excluding a `site-build.json` at its root. */ +export const createSiteBuildManifest = (root: string, identity: SiteBuildIdentity): SiteBuildManifest => { + assertSiteBuildIdentity(identity); + const files = hashTreeFiles(root, isRootManifest); + + return { + schemaVersion: SITE_BUILD_SCHEMA_VERSION, + channel: identity.channel, + basePath: identity.basePath, + sourceSha: identity.sourceSha, + packageVersion: identity.packageVersion, + assetProfile: identity.assetProfile, + createdAt: identity.createdAt, + localTest: identity.localTest, + fileCount: files.length, + totalBytes: files.reduce((sum, file) => sum + file.size, 0), + contentHash: computeContentHash(files), + files, + }; +}; + +/** Serialized form, stable for a given manifest. */ +export const serializeSiteBuildManifest = (manifest: SiteBuildManifest): string => `${JSON.stringify(manifest, null, 2)}\n`; + +/** + * Validates an untrusted, already JSON-parsed manifest. Throws listing every + * problem; the returned value is a fresh object with exactly the known fields. + */ +export const parseSiteBuildManifest = (value: unknown): SiteBuildManifest => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error(`[site-manifest] ${SITE_BUILD_FILE} is not a JSON object`); + } + + const raw = value as Record; + const problems: string[] = []; + + if (raw['schemaVersion'] !== SITE_BUILD_SCHEMA_VERSION) { + problems.push(`schemaVersion ${JSON.stringify(raw['schemaVersion'])} is not ${SITE_BUILD_SCHEMA_VERSION}`); + } + + const identity = { + channel: raw['channel'], + basePath: raw['basePath'], + sourceSha: raw['sourceSha'], + packageVersion: raw['packageVersion'], + assetProfile: raw['assetProfile'], + createdAt: raw['createdAt'], + localTest: raw['localTest'], + } as SiteBuildIdentity; + + problems.push(...identityProblems(identity)); + + const files: SiteBuildFileEntry[] = []; + + if (!Array.isArray(raw['files'])) { + problems.push('files is not an array'); + } else { + let previous: string | null = null; + + for (const [index, entry] of (raw['files'] as unknown[]).entries()) { + const file = (typeof entry === 'object' && entry !== null ? entry : {}) as Record; + const path = file['path']; + const size = file['size']; + const hash = file['sha256']; + + if (typeof path !== 'string' || validateSitePath(path) !== null) { + problems.push(`files[${index}].path: ${typeof path === 'string' ? validateSitePath(path) : 'not a string'}`); + continue; + } + + if (path === SITE_BUILD_FILE) { + problems.push(`files[${index}] lists ${SITE_BUILD_FILE} itself`); + } + + if (previous !== null && compareCodeUnits(previous, path) >= 0) { + problems.push(`files[${index}] ${path} is not sorted after ${previous} or is a duplicate`); + } + + if (typeof size !== 'number' || !Number.isSafeInteger(size) || size < 0) { + problems.push(`files[${index}].size is not a non-negative integer`); + } + + if (typeof hash !== 'string' || !SHA256_HEX.test(hash)) { + problems.push(`files[${index}].sha256 is not a SHA-256 hex digest`); + } + + previous = path; + files.push({ path, size: size as number, sha256: hash as string }); + } + } + + if (raw['fileCount'] !== files.length) { + problems.push(`fileCount ${JSON.stringify(raw['fileCount'])} does not match the ${files.length} listed files`); + } + + const totalBytes = files.reduce((sum, file) => sum + file.size, 0); + + if (raw['totalBytes'] !== totalBytes) { + problems.push(`totalBytes ${JSON.stringify(raw['totalBytes'])} does not match the listed sizes (${totalBytes})`); + } + + const contentHash = computeContentHash(files); + + if (raw['contentHash'] !== contentHash) { + problems.push(`contentHash ${JSON.stringify(raw['contentHash'])} does not match the file list (${contentHash})`); + } + + if (problems.length > 0) { + throw new Error(`[site-manifest] invalid ${SITE_BUILD_FILE}:\n ${problems.join('\n ')}`); + } + + return { + schemaVersion: SITE_BUILD_SCHEMA_VERSION, + ...identity, + fileCount: files.length, + totalBytes, + contentHash, + files, + }; +}; + +/** Reads and validates `/site-build.json`. */ +export const readSiteBuildManifest = (root: string): SiteBuildManifest => + parseSiteBuildManifest(JSON.parse(readFileSync(join(root, SITE_BUILD_FILE), 'utf8')) as unknown); + +export interface VerifySiteTreeOptions { + /** Root-relative paths that may exist beside the manifest's files. */ + allowExtra?: (path: string) => boolean; +} + +/** + * Every difference between the tree at `root` and `manifest`: a missing, + * unexpected or altered file, or an unsafe entry. Empty when they match. + */ +export const verifySiteTree = (root: string, manifest: SiteBuildManifest, options: VerifySiteTreeOptions = {}): string[] => { + const allowExtra = options.allowExtra ?? (() => false); + let actual: SiteBuildFileEntry[]; + + try { + actual = hashTreeFiles(root, path => isRootManifest(path) || allowExtra(path)); + } catch (error) { + return [error instanceof Error ? error.message : String(error)]; + } + + const problems: string[] = []; + const actualByPath = new Map(actual.map(file => [file.path, file])); + + for (const expected of manifest.files) { + const found = actualByPath.get(expected.path); + + if (found === undefined) { + problems.push(`missing file ${expected.path}`); + } else if (found.size !== expected.size || found.sha256 !== expected.sha256) { + problems.push(`altered file ${expected.path}`); + } + + actualByPath.delete(expected.path); + } + + for (const path of actualByPath.keys()) { + problems.push(`unexpected file ${path}`); + } + + if (computeContentHash(manifest.files) !== manifest.contentHash) { + problems.push('contentHash does not match the manifest file list'); + } + + return problems; +}; diff --git a/scripts/pages/stamp-stable-site.ts b/scripts/pages/stamp-stable-site.ts new file mode 100644 index 000000000..4295da624 --- /dev/null +++ b/scripts/pages/stamp-stable-site.ts @@ -0,0 +1,89 @@ +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { SITE_CHANNEL_BASE } from './site-channel.ts'; +import { createSiteBuildManifest, serializeSiteBuildManifest, SITE_BUILD_FILE, type SiteBuildIdentity } from './site-manifest.ts'; + +/** + * Records the identity and content of a finished Stable site tree in its + * `site-build.json`, so the Pages publisher can verify it like any other + * channel candidate. + * + * The Stable site build on this branch has a fixed `/ExoJS/` base and writes + * no manifest of its own; a channel-aware site build records it during the + * build instead, and makes this step unnecessary. The demo media of this + * site are recorded as `standard`, the only asset profile the manifest + * schema accepts. + * + * Dependency-free and erasable TypeScript, run by CI with plain `node`. + */ + +export interface StampOptions { + dir: string; + sourceSha: string; + packageVersion: string; + createdAt: string; +} + +/** Writes the Stable `site-build.json` into `dir`. Throws when the tree is not a Stable build. */ +export const stampStableSite = (options: StampOptions): SiteBuildIdentity => { + const base = SITE_CHANNEL_BASE.stable; + const entry = join(options.dir, 'index.html'); + + if (existsSync(join(options.dir, SITE_BUILD_FILE))) { + throw new Error(`[stamp-stable-site] ${SITE_BUILD_FILE} already exists in ${options.dir}`); + } + + if (!existsSync(entry)) { + throw new Error(`[stamp-stable-site] ${options.dir} has no index.html`); + } + + if (!readFileSync(entry, 'utf8').includes(`"${base}`)) { + throw new Error(`[stamp-stable-site] index.html references no asset under the Stable base ${base}`); + } + + const identity: SiteBuildIdentity = { + channel: 'stable', + basePath: base, + sourceSha: options.sourceSha, + packageVersion: options.packageVersion, + assetProfile: 'standard', + createdAt: options.createdAt, + localTest: false, + }; + const manifest = createSiteBuildManifest(options.dir, identity); + writeFileSync(join(options.dir, SITE_BUILD_FILE), serializeSiteBuildManifest(manifest)); + console.log( + `[stamp-stable-site] ${base} ${manifest.sourceSha.slice(0, 12)} ${manifest.packageVersion}: ` + + `${manifest.fileCount} files, ${manifest.totalBytes} bytes, content ${manifest.contentHash}`, + ); + + return identity; +}; + +const argument = (name: string): string => { + const index = process.argv.indexOf(name); + const value = index === -1 ? undefined : process.argv[index + 1]; + + if (value === undefined || value.startsWith('--')) { + throw new Error(`[stamp-stable-site] usage: stamp-stable-site.ts --dir --sha ; missing ${name}`); + } + + return value; +}; + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + const rootPackage = JSON.parse(readFileSync('package.json', 'utf8')) as { version?: string }; + stampStableSite({ + dir: argument('--dir'), + sourceSha: argument('--sha'), + packageVersion: rootPackage.version ?? '', + createdAt: new Date().toISOString(), + }); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/test/ci/pages-deployment.test.ts b/test/ci/pages-deployment.test.ts index fb82553bb..b3739febb 100644 --- a/test/ci/pages-deployment.test.ts +++ b/test/ci/pages-deployment.test.ts @@ -6,10 +6,13 @@ import { describe, expect, it } from 'vitest'; /** * Locks the build-once/deploy-once contract for GitHub Pages. * - * The site is built and validated exactly once, by the `site-build` job in - * `ci.yml`, which publishes `site/dist/**` as an artifact named after - * the commit it was built from. `deploy-pages.yml` then downloads exactly that - * artifact - from exactly the workflow run that triggered it - and deploys it. + * Each channel is built and validated exactly once, by the `site` job in + * `ci.yml` of the branch that publishes it: this branch builds Stable, and + * records the finished tree in its `site-build.json` after the build. The + * tree goes up as `site-stable--`. `deploy-pages.yml` is the one publisher: it re-decides + * both channels from GitHub metadata, verifies the chosen bytes with scripts + * from the trusted default branch, composes Stable at the root and Next under + * `next/`, and deploys that one tree. * * The defect this prevents: the Pages workflow used to check the sources out * and rebuild the library and the site itself, so what went live was a second, @@ -24,9 +27,7 @@ const ci = readWorkflow('ci.yml'); const release = readWorkflow('release.yml'); const deployPages = readWorkflow('deploy-pages.yml'); -/** The artifact name is derived from the commit on both sides of the handover. */ -const SITE_ARTIFACT_UPLOAD = 'site-dist-${{ github.sha }}'; -const SITE_ARTIFACT_DOWNLOAD = 'site-dist-${{ github.event.workflow_run.head_sha }}'; +const CANDIDATE_UPLOAD = 'site-stable-${{ github.sha }}-${{ github.run_attempt }}'; /** Everything between `:` and the next top-level job key. */ const jobBlock = (workflow: string, job: string) => { @@ -34,66 +35,136 @@ const jobBlock = (workflow: string, job: string) => { expect(start, `job '${job}' not found`).toBeGreaterThan(-1); const rest = workflow.slice(start + 1); const next = rest.search(/\n {2}[a-z][\w-]*:\n/); + return next === -1 ? rest : rest.slice(0, next); }; -describe('site artifact production (ci.yml)', () => { - const siteBuild = jobBlock(ci, 'site'); +/** The script bodies of every `run:` step, single-line and block form. */ +const runScripts = (workflow: string): string[] => + [...workflow.matchAll(/^( *)(?:- )?run: ?(?:[|>]-?\n((?:\1 {2,}.*\n?)+)|(.*))$/gm)].map(match => (match[2] ?? match[3] ?? '').trim()); + +describe('site candidate production (ci.yml)', () => { + const site = jobBlock(ci, 'site'); - it('uploads the built site as a commit-identified artifact', () => { - expect(siteBuild).toContain('actions/upload-artifact@v7'); - expect(siteBuild).toContain(`name: ${SITE_ARTIFACT_UPLOAD}`); - expect(siteBuild).toContain('path: site/dist/'); + it('records the built tree as the Stable channel of this commit', () => { + const buildIndex = site.indexOf('pnpm site:build'); + const stampIndex = site.indexOf('node scripts/pages/stamp-stable-site.ts --dir site/dist --sha "$GITHUB_SHA"'); + expect(buildIndex).toBeGreaterThan(-1); + expect(stampIndex).toBeGreaterThan(buildIndex); + expect(site).toContain('node scripts/pages/channel-artifact.ts verify --dir site/dist --channel stable --sha "$GITHUB_SHA"'); + }); + + it('uploads the built tree as a channel candidate, kept 90 days for a push', () => { + expect(site).toContain('actions/upload-artifact@v7'); + expect(site).toContain(`name: ${CANDIDATE_UPLOAD}`); + expect(site).toContain('path: site/dist/'); + expect(site).toContain("retention-days: ${{ github.event_name == 'push' && 90 || 1 }}"); + expect(site).toContain(`candidate: ${CANDIDATE_UPLOAD}`); }); it('fails the build rather than publishing an empty site artifact', () => { - expect(siteBuild).toContain('if-no-files-found: error'); + expect(site).toContain('if-no-files-found: error'); }); - it('uploads only after the site typecheck gate and the site build have passed', () => { - const gateIndex = siteBuild.indexOf('pnpm gates site'); - const buildIndex = siteBuild.indexOf('pnpm site:build'); - const uploadIndex = siteBuild.indexOf('actions/upload-artifact@v7'); + it('uploads only after the site gate, the build and the candidate verification have passed', () => { + const gateIndex = site.indexOf('pnpm gates site'); + const buildIndex = site.indexOf('pnpm site:build'); + const verifyIndex = site.indexOf('node scripts/pages/channel-artifact.ts verify --dir site/dist'); + const uploadIndex = site.indexOf('actions/upload-artifact@v7\n with:\n name: site-'); expect(gateIndex).toBeGreaterThan(-1); expect(buildIndex).toBeGreaterThan(gateIndex); - expect(uploadIndex).toBeGreaterThan(buildIndex); + expect(verifyIndex).toBeGreaterThan(buildIndex); + expect(uploadIndex).toBeGreaterThan(verifyIndex); + }); + + it('smokes exactly that candidate', () => { + const smoke = jobBlock(ci, 'smoke'); + expect(smoke).toContain('name: ${{ needs.site.outputs.candidate }}'); + expect(smoke).toContain('path: site/dist'); }); }); describe('site artifact production at a release (release.yml)', () => { - it('uploads the site under the tagged commit so Deploy Pages finds it', () => { + // A release on this branch still uploads its site under the old name, which + // the publisher does not qualify: Stable comes from the CI push instead. + it('uploads the site under the tagged commit', () => { const prepare = jobBlock(release, 'prepare'); expect(prepare).toContain('name: site-dist-${{ needs.trust.outputs.sha }}'); expect(prepare).toContain('path: site/dist/'); }); }); -describe('Pages deployment (deploy-pages.yml)', () => { +describe('Pages publisher (deploy-pages.yml)', () => { + const compose = jobBlock(deployPages, 'compose'); + const deploy = jobBlock(deployPages, 'deploy'); + it('builds no source code of its own', () => { - for (const forbidden of ['pnpm install', 'pnpm build', 'pnpm site:build', 'pnpm --filter', 'pnpm/action-setup', 'actions/setup-node', 'actions/checkout']) { - expect(deployPages, `deploy-pages.yml must not run '${forbidden}'`).not.toContain(forbidden); + for (const forbidden of ['pnpm', './.github/actions/setup', 'npm install', 'npx ', 'actions/download-artifact']) { + expect(deployPages, `deploy-pages.yml must not use '${forbidden}'`).not.toContain(forbidden); + } + }); + + it('runs only its own scripts from the trusted default-branch checkout, never a downloaded file', () => { + expect(compose).toContain('ref: ${{ github.event.repository.default_branch }}'); + expect(compose).toContain('persist-credentials: false'); + expect(compose).toContain('/scripts/pages/'); + expect(deployPages.match(/actions\/checkout@/g)).toHaveLength(1); + + const scripts = runScripts(deployPages); + expect(scripts.length).toBeGreaterThan(0); + + for (const script of scripts) { + expect(script, 'every step runs a publisher script with node').toMatch(/^node scripts\/pages\/[a-z-]+\.ts\b/); + // Event text must never be interpolated into a shell script. + expect(script).not.toContain('${{'); + } + }); + + it('is woken by a finished CI or Release run and has no other trigger', () => { + const triggers = deployPages.slice(deployPages.indexOf('\non:\n'), deployPages.indexOf('\npermissions:')); + expect(triggers).toContain("workflows: ['Release', 'CI']"); + expect(triggers).toContain('types: [completed]'); + + for (const other of ['workflow_dispatch', 'pull_request', 'push:', 'schedule']) { + expect(triggers).not.toContain(other); } }); - it('deploys the artifact of the run that triggered it', () => { - expect(deployPages).toContain('actions/download-artifact@v8'); - expect(deployPages).toContain(`name: ${SITE_ARTIFACT_DOWNLOAD}`); - expect(deployPages).toContain('run-id: ${{ github.event.workflow_run.id }}'); - // Cross-run artifact reads need an explicit token and `actions: read`. - expect(deployPages).toContain('actions: read'); - expect(deployPages).toContain('github-token: ${{ github.token }}'); + it('acts only on a successful same-repository push run, and re-decides from metadata', () => { + for (const condition of [ + "github.event.workflow_run.conclusion == 'success'", + "github.event.workflow_run.event == 'push'", + 'github.event.workflow_run.repository.full_name == github.repository', + 'github.event.workflow_run.head_repository.full_name == github.repository', + ]) { + expect(compose).toContain(condition); + } + + expect(compose).toContain('TRIGGER_RUN_ID: ${{ github.event.workflow_run.id }}'); + expect(compose).toContain('node scripts/pages/select-channels.ts'); + expect(compose).toContain('node scripts/pages/compose-site.ts'); + }); + + it('serializes the whole site without cancelling a running deployment', () => { + expect(deployPages).toContain('concurrency:\n group: pages-combined\n cancel-in-progress: false'); }); - it('refuses to deploy when it has no triggering run to read the artifact from', () => { - expect(deployPages).toContain('workflow_run.id'); - expect(deployPages).toMatch(/github\.event_name == 'workflow_run'/); - // A manual dispatch has no triggering run, so it must not be a trigger. - const triggers = deployPages.slice(deployPages.indexOf('\non:\n'), deployPages.indexOf('\npermissions:\n')); - expect(triggers).not.toContain('workflow_dispatch'); + it('grants each job only what it needs', () => { + expect(deployPages).toContain('\npermissions: {}\n'); + expect(compose).toContain( + 'permissions:\n contents: read\n # Listing runs and reading artifacts of other workflow runs.\n actions: read\n outputs:', + ); + expect(compose).not.toMatch(/pages: write|id-token: write/); + expect(deploy).toContain('permissions:\n pages: write\n id-token: write\n environment:'); + expect(deploy).not.toMatch(/contents:|actions:/); }); - it('still hands the deployed tree to the Pages actions', () => { - expect(deployPages).toContain('actions/upload-pages-artifact@v5'); - expect(deployPages).toContain('actions/deploy-pages@v5'); + it('uploads exactly one Pages artifact and keeps it with the deployed pairing', () => { + expect(deployPages.match(/actions\/upload-pages-artifact@v5/g)).toHaveLength(1); + expect(deployPages.match(/actions\/deploy-pages@v5/g)).toHaveLength(1); + expect(compose).toContain('retention-days: 90'); + expect(compose).toContain('name: pages-channels-${{ github.run_id }}'); + expect(deploy).toContain("if: needs.compose.outputs.deploy == 'true'"); + expect(deploy).toContain('name: github-pages'); }); }); diff --git a/test/site/pages-archive-safety.test.ts b/test/site/pages-archive-safety.test.ts new file mode 100644 index 000000000..57f9ca4d6 --- /dev/null +++ b/test/site/pages-archive-safety.test.ts @@ -0,0 +1,157 @@ +import { existsSync, linkSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { checkArchiveEntries, checkTreeSafety, extractTar, extractZip, readTarEntries, readZipEntries, readZipFile } from '../../scripts/pages/archive-safety'; +import { evaluateSiteSize, SITE_SIZE_REFUSE_BYTES, SITE_SIZE_WARN_BYTES } from '../../scripts/pages/site-budget'; +import { createTar, createZip } from '../support/archive-fixtures'; + +/** + * Candidate artifacts are downloaded by a workflow that can deploy, so every + * archive is inspected before a single byte is written, and the composed tree + * is inspected again before it is uploaded. + */ + +let work: string; + +beforeEach(() => { + work = mkdtempSync(join(tmpdir(), 'exo-archive-')); +}); + +afterEach(() => rmSync(work, { recursive: true, force: true })); + +const limits = { maxUnpackedBytes: 1_000_000 }; + +describe('zip archives', () => { + it('reads and extracts a well-formed archive', () => { + const zip = createZip([{ name: 'en/' }, { name: 'en/index.html', data: '

hi

' }, { name: 'index.html', data: 'root', store: true }]); + expect(readZipEntries(zip).map(entry => [entry.name, entry.type])).toEqual([ + ['en', 'directory'], + ['en/index.html', 'file'], + ['index.html', 'file'], + ]); + extractZip(zip, work, limits); + expect(readFileSync(join(work, 'en/index.html'), 'utf8')).toBe('

hi

'); + expect(readFileSync(join(work, 'index.html'), 'utf8')).toBe('root'); + }); + + it('reads one named file without extracting anything', () => { + const zip = createZip([{ name: 'artifact.tar', data: 'tar bytes' }]); + expect(readZipFile(zip, 'artifact.tar', limits).toString('utf8')).toBe('tar bytes'); + expect(() => readZipFile(zip, 'other.tar', limits)).toThrow(/other\.tar/); + }); + + it.each([ + ['an absolute path', '/etc/cron.d/x'], + ['a parent traversal', '../outside.html'], + ['a nested traversal', 'en/../../outside.html'], + ['a Windows drive', 'C:/Windows/x.dll'], + ['a backslash', 'en\\..\\x.html'], + ])('refuses %s before writing anything', (_label, name) => { + const zip = createZip([ + { name: 'ok.html', data: 'ok' }, + { name, data: 'evil' }, + ]); + expect(() => extractZip(zip, work, limits)).toThrow(/path/i); + expect(existsSync(join(work, 'ok.html'))).toBe(false); + }); + + it('refuses symbolic links and special files', () => { + expect(() => extractZip(createZip([{ name: 'link', data: '/etc/passwd', mode: 0o120777 }]), work, limits)).toThrow(/symlink/); + expect(() => extractZip(createZip([{ name: 'fifo', mode: 0o010644 }]), work, limits)).toThrow(/special/); + }); + + it('refuses duplicate entries and encrypted entries', () => { + expect(() => + extractZip( + createZip([ + { name: 'a.html', data: '1' }, + { name: 'a.html', data: '2' }, + ]), + work, + limits, + ), + ).toThrow(/duplicate/); + expect(() => extractZip(createZip([{ name: 'a.html', data: '1', encrypted: true }]), work, limits)).toThrow(/encrypt/); + }); + + it('refuses an archive whose declared unpacked size exceeds the budget', () => { + const zip = createZip([{ name: 'big.bin', data: Buffer.alloc(2048) }]); + expect(() => extractZip(zip, work, { maxUnpackedBytes: 1024 })).toThrow(/budget/); + }); + + it('refuses an entry that inflates past its declared size', () => { + const bomb = createZip([{ name: 'bomb.bin', data: Buffer.alloc(500_000), declaredSize: 10 }]); + expect(checkArchiveEntries(readZipEntries(bomb), limits)).toEqual([]); + expect(() => extractZip(bomb, work, limits)).toThrow(/size/); + }); + + it('refuses a truncated or foreign file', () => { + expect(() => readZipEntries(Buffer.from('not a zip at all'))).toThrow(/zip/i); + const zip = createZip([{ name: 'a.html', data: 'x' }]); + expect(() => readZipEntries(zip.subarray(0, zip.length - 4))).toThrow(); + }); +}); + +describe('tar archives', () => { + it('reads and extracts the shape the Pages upload action writes', () => { + const tar = createTar([{ name: './' }, { name: './en/' }, { name: './en/index.html', data: 'hello' }, { name: './.nojekyll' }]); + expect(readTarEntries(tar).map(entry => [entry.name, entry.type])).toEqual([ + ['en', 'directory'], + ['en/index.html', 'file'], + ['.nojekyll', 'file'], + ]); + extractTar(tar, work, limits); + expect(readFileSync(join(work, 'en/index.html'), 'utf8')).toBe('hello'); + }); + + it('honours a pax path and still validates it', () => { + const long = `${'deep/'.repeat(30)}file.html`; + expect(readTarEntries(createTar([{ name: long, data: 'x', pax: true }]))[0]?.name).toBe(long); + expect(() => extractTar(createTar([{ name: '../escape.html', data: 'x', pax: true }]), work, limits)).toThrow(/path/i); + }); + + it('refuses symlinks, hardlinks and special files', () => { + expect(() => extractTar(createTar([{ name: 'a', type: '2', linkName: '/etc/passwd' }]), work, limits)).toThrow(/symlink/); + expect(() => extractTar(createTar([{ name: 'b', type: '1', linkName: 'a' }]), work, limits)).toThrow(/hardlink/); + expect(() => extractTar(createTar([{ name: 'c', type: '3' }]), work, limits)).toThrow(/special/); + }); + + it('refuses an archive past the unpacked budget', () => { + expect(() => extractTar(createTar([{ name: 'big.bin', data: Buffer.alloc(4096) }]), work, { maxUnpackedBytes: 1000 })).toThrow(/budget/); + }); +}); + +describe('checkTreeSafety', () => { + it('passes a tree of plain files and directories', () => { + mkdirSync(join(work, 'en')); + writeFileSync(join(work, 'en/index.html'), 'x'); + expect(checkTreeSafety(work)).toEqual([]); + }); + + it('reports a hardlinked file', () => { + writeFileSync(join(work, 'a.html'), 'x'); + linkSync(join(work, 'a.html'), join(work, 'b.html')); + expect(checkTreeSafety(work).join('\n')).toMatch(/hardlink/); + }); + + it('reports a linked directory', () => { + mkdirSync(join(work, 'target')); + // A junction needs no privilege on Windows and is a plain symlink elsewhere. + symlinkSync(join(work, 'target'), join(work, 'link'), 'junction'); + expect(checkTreeSafety(work).join('\n')).toMatch(/symlink/); + }); +}); + +describe('evaluateSiteSize', () => { + it('measures against the project budget in unpacked bytes', () => { + expect(SITE_SIZE_WARN_BYTES).toBe(750_000_000); + expect(SITE_SIZE_REFUSE_BYTES).toBe(900_000_000); + expect(evaluateSiteSize(749_999_999).level).toBe('ok'); + expect(evaluateSiteSize(750_000_000).level).toBe('warn'); + expect(evaluateSiteSize(899_999_999).level).toBe('warn'); + expect(evaluateSiteSize(900_000_000).level).toBe('refuse'); + }); +}); diff --git a/test/site/pages-channel-artifact.test.ts b/test/site/pages-channel-artifact.test.ts new file mode 100644 index 000000000..874aad729 --- /dev/null +++ b/test/site/pages-channel-artifact.test.ts @@ -0,0 +1,106 @@ +import { createHash } from 'node:crypto'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { candidateArtifactName, openChannelArchive, parseCandidateArtifactName, verifyChannelTree, verifyDigest } from '../../scripts/pages/channel-artifact'; +import { createSiteBuildManifest, SITE_BUILD_FILE, type SiteBuildIdentity } from '../../scripts/pages/site-manifest'; +import { createZip } from '../support/archive-fixtures'; + +/** + * A channel candidate is the zip of one built tree plus its `site-build.json`. + * Its name, its bytes and its manifest have to agree with the run metadata + * GitHub reports before the publisher uses it. + */ + +const SHA = 'c'.repeat(40); + +let work: string; + +const identity: SiteBuildIdentity = { + channel: 'next', + basePath: '/ExoJS/next/', + sourceSha: SHA, + packageVersion: '0.19.1', + assetProfile: 'standard', + createdAt: '2026-10-21T12:00:00.000Z', + localTest: false, +}; + +const buildTree = (overrides: Partial = {}): string => { + const dir = join(work, 'tree'); + mkdirSync(join(dir, 'en'), { recursive: true }); + writeFileSync(join(dir, 'index.html'), 'x'); + writeFileSync(join(dir, 'en/index.html'), 'y'); + writeFileSync(join(dir, SITE_BUILD_FILE), JSON.stringify(createSiteBuildManifest(dir, { ...identity, ...overrides }))); + + return dir; +}; + +beforeEach(() => { + work = mkdtempSync(join(tmpdir(), 'exo-channel-artifact-')); +}); + +afterEach(() => rmSync(work, { recursive: true, force: true })); + +describe('candidate artifact names', () => { + it('round-trips site---', () => { + expect(candidateArtifactName('next', SHA, 2)).toBe(`site-next-${SHA}-2`); + expect(parseCandidateArtifactName(`site-stable-${SHA}-1`)).toEqual({ channel: 'stable', sha: SHA, attempt: 1 }); + }); + + it('rejects anything else', () => { + for (const name of [`site-dist-${SHA}`, `site-beta-${SHA}-1`, `site-next-${SHA.slice(1)}-1`, `site-next-${SHA}-0`, `site-next-${SHA}-1x`, 'github-pages']) { + expect(parseCandidateArtifactName(name), `${name}`).toBeNull(); + } + }); +}); + +describe('verifyDigest', () => { + it('compares the archive bytes against the digest GitHub reported', () => { + const bytes = Buffer.from('zip bytes'); + const digest = `sha256:${createHash('sha256').update(bytes).digest('hex')}`; + expect(verifyDigest(bytes, digest)).toBeNull(); + expect(verifyDigest(Buffer.from('other'), digest)).toMatch(/digest/); + expect(verifyDigest(bytes, 'sha1:abc')).toMatch(/digest/); + }); +}); + +describe('verifyChannelTree', () => { + it('accepts a tree that matches the expected channel and commit', () => { + expect(verifyChannelTree(buildTree(), { channel: 'next', sourceSha: SHA }).problems).toEqual([]); + }); + + it.each<[string, Partial, RegExp]>([ + ['another commit', { sourceSha: 'd'.repeat(40) }, /sourceSha/], + ['another channel', { channel: 'stable', basePath: '/ExoJS/' }, /channel/], + ['a local test base', { basePath: '/sandbox/', localTest: true }, /local test/], + ])('refuses %s', (_label, overrides, pattern) => { + expect(verifyChannelTree(buildTree(overrides), { channel: 'next', sourceSha: SHA }).problems.join('\n')).toMatch(pattern); + }); + + it('refuses a tree without a manifest or with a tampered file', () => { + const dir = buildTree(); + writeFileSync(join(dir, 'en/index.html'), 'tampered'); + expect(verifyChannelTree(dir, { channel: 'next', sourceSha: SHA }).problems.join('\n')).toMatch(/en\/index\.html/); + rmSync(join(dir, SITE_BUILD_FILE)); + expect(verifyChannelTree(dir, { channel: 'next', sourceSha: SHA }).problems.join('\n')).toMatch(SITE_BUILD_FILE); + }); +}); + +describe('openChannelArchive', () => { + it('extracts a verified candidate zip and checks its manifest', () => { + const tree = buildTree(); + const zip = createZip([ + { name: 'index.html', data: 'x' }, + { name: 'en/index.html', data: 'y' }, + { name: SITE_BUILD_FILE, data: JSON.stringify(createSiteBuildManifest(tree, identity)) }, + ]); + const digest = `sha256:${createHash('sha256').update(zip).digest('hex')}`; + const out = join(work, 'out'); + expect(openChannelArchive(zip, { digest, channel: 'next', sourceSha: SHA, outDir: out }).manifest.contentHash).toHaveLength(64); + expect(() => openChannelArchive(zip, { digest: `sha256:${'0'.repeat(64)}`, channel: 'next', sourceSha: SHA, outDir: join(work, 'o2') })).toThrow(/digest/); + }); +}); diff --git a/test/site/pages-compose-site.test.ts b/test/site/pages-compose-site.test.ts new file mode 100644 index 000000000..95ebf3e6c --- /dev/null +++ b/test/site/pages-compose-site.test.ts @@ -0,0 +1,253 @@ +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { + buildChannelsDocument, + type ChannelSource, + composeSite, + extractComposedChannel, + findCrossChannelReferences, + parseChannelsDocument, +} from '../../scripts/pages/compose-site'; +import { createSiteBuildManifest, hashTreeFiles, SITE_BUILD_FILE, type SiteBuildIdentity, verifySiteTree } from '../../scripts/pages/site-manifest'; + +/** + * The composer joins two independently qualified channel trees into the one + * tree Pages serves: Stable at the root, Next under `next/`. It adds only + * `channels.json` and `.nojekyll`, and it proves that neither channel's bytes + * changed on the way. + */ + +let work: string; + +const sha = (seed: string): string => seed.repeat(40); + +const identity = (channel: 'stable' | 'next', seed: string): SiteBuildIdentity => ({ + channel, + basePath: channel === 'stable' ? '/ExoJS/' : '/ExoJS/next/', + sourceSha: sha(seed), + packageVersion: channel === 'stable' ? '0.19.1' : '0.19.1', + assetProfile: 'standard', + createdAt: '2026-10-21T12:00:00.000Z', + localTest: false, +}); + +const source = (channel: string, seed: string): ChannelSource => ({ + kind: 'ci', + workflowId: 11, + runId: 100, + runAttempt: 1, + artifactId: 200, + artifactName: `site-${channel}-${sha(seed)}-1`, + digest: `sha256:${'d'.repeat(64)}`, +}); + +const buildChannel = (channel: 'stable' | 'next', seed: string, files: Record): string => { + const dir = join(work, `${channel}-${seed}`); + + for (const [rel, content] of Object.entries(files)) { + mkdirSync(join(dir, rel, '..'), { recursive: true }); + writeFileSync(join(dir, rel), content); + } + + const manifest = createSiteBuildManifest(dir, identity(channel, seed)); + writeFileSync(join(dir, SITE_BUILD_FILE), JSON.stringify(manifest)); + + return dir; +}; + +const stableFiles = { + 'index.html': '', + 'en/index.html': 'Guide', + 'preview.html': 'preview', + 'pagefind/pagefind.js': 'pf', +}; + +const nextFiles = { + 'index.html': '', + 'en/index.html': 'GuideStable', + 'preview.html': 'preview next', + 'pagefind/pagefind.js': 'pf next', +}; + +beforeEach(() => { + work = mkdtempSync(join(tmpdir(), 'exo-compose-')); +}); + +afterEach(() => rmSync(work, { recursive: true, force: true })); + +const compose = (out: string, stableDir: string, nextDir: string | null) => + composeSite({ + outDir: out, + stable: { dir: stableDir, source: source('stable', 'a') }, + next: nextDir === null ? null : { dir: nextDir, source: source('next', '1') }, + }); + +describe('composeSite (W02)', () => { + it('puts Stable at the root and Next under next/, adding only channels.json and .nojekyll', () => { + const out = join(work, 'out'); + const report = compose(out, buildChannel('stable', 'a', stableFiles), buildChannel('next', '1', nextFiles)); + + expect(report.problems).toEqual([]); + expect(readdirSync(out).sort()).toEqual(['.nojekyll', 'channels.json', 'en', 'index.html', 'next', 'pagefind', 'preview.html', SITE_BUILD_FILE]); + expect(readFileSync(join(out, 'next/preview.html'), 'utf8')).toBe('preview next'); + expect(readFileSync(join(out, '.nojekyll'), 'utf8')).toBe(''); + }); + + it('leaves both channels byte-identical to their manifests', () => { + const stableDir = buildChannel('stable', 'a', stableFiles); + const nextDir = buildChannel('next', '1', nextFiles); + const out = join(work, 'out'); + compose(out, stableDir, nextDir); + + const stableManifest = createSiteBuildManifest(stableDir, identity('stable', 'a')); + const nextManifest = createSiteBuildManifest(nextDir, identity('next', '1')); + const composerFiles = new Set(['channels.json', '.nojekyll']); + expect(verifySiteTree(out, stableManifest, { allowExtra: path => path.startsWith('next/') || composerFiles.has(path) })).toEqual([]); + expect(verifySiteTree(join(out, 'next'), nextManifest)).toEqual([]); + }); + + it('records both channels in channels.json', () => { + const out = join(work, 'out'); + compose(out, buildChannel('stable', 'a', stableFiles), buildChannel('next', '1', nextFiles)); + const channels = parseChannelsDocument(JSON.parse(readFileSync(join(out, 'channels.json'), 'utf8'))); + + expect(channels.channels.stable).toMatchObject({ basePath: '/ExoJS/', path: '', sourceSha: sha('a'), packageVersion: '0.19.1' }); + expect(channels.channels.next).toMatchObject({ basePath: '/ExoJS/next/', path: 'next/', sourceSha: sha('1'), source: { runId: 100 } }); + }); + + it('composes Stable alone while Next is absent, without a next/ directory', () => { + const out = join(work, 'out'); + const report = compose(out, buildChannel('stable', 'a', stableFiles), null); + expect(report.problems).toEqual([]); + expect(existsSync(join(out, 'next'))).toBe(false); + expect(parseChannelsDocument(JSON.parse(readFileSync(join(out, 'channels.json'), 'utf8'))).channels.next).toBeNull(); + }); + + it('is idempotent for the same pairing', () => { + const stableDir = buildChannel('stable', 'a', stableFiles); + const nextDir = buildChannel('next', '1', nextFiles); + compose(join(work, 'one'), stableDir, nextDir); + compose(join(work, 'two'), stableDir, nextDir); + expect(hashTreeFiles(join(work, 'one'))).toEqual(hashTreeFiles(join(work, 'two'))); + }); + + it('copies only the files a manifest lists', () => { + const stableDir = buildChannel('stable', 'a', stableFiles); + writeFileSync(join(stableDir, 'smuggled.js'), 'x'); + const report = compose(join(work, 'out'), stableDir, null); + expect(report.problems.join('\n')).toMatch(/smuggled\.js/); + }); +}); + +describe('composeSite refuses inconsistent channels (W07)', () => { + it('refuses swapped channels', () => { + const report = compose(join(work, 'out'), buildChannel('next', '1', nextFiles), buildChannel('stable', 'a', stableFiles)); + expect(report.problems.join('\n')).toMatch(/channel/); + }); + + it('refuses a Stable tree that carries its own next/, channels.json or .nojekyll', () => { + for (const [index, extra] of ['next/index.html', 'channels.json', '.nojekyll'].entries()) { + const seed = ['b', 'c', 'e'][index]!; + const report = compose(join(work, `out-${seed}`), buildChannel('stable', seed, { ...stableFiles, [extra]: 'x' }), null); + expect(report.problems.join('\n'), `${extra}`).toMatch(/reserved/); + } + }); + + it('refuses a local test build', () => { + const dir = join(work, 'local'); + mkdirSync(dir); + writeFileSync(join(dir, 'index.html'), 'x'); + writeFileSync(join(dir, SITE_BUILD_FILE), JSON.stringify(createSiteBuildManifest(dir, { ...identity('stable', 'a'), localTest: true }))); + expect(compose(join(work, 'out'), dir, null).problems.join('\n')).toMatch(/local test/); + }); + + it('refuses an output directory that is not empty', () => { + const out = join(work, 'out'); + mkdirSync(out); + writeFileSync(join(out, 'old.html'), 'x'); + expect(() => compose(out, buildChannel('stable', 'a', stableFiles), null)).toThrow(/empty/); + }); +}); + +describe('composeSite size budget (W12)', () => { + it('warns from 750,000,000 and refuses from 900,000,000 unpacked bytes', () => { + const stableDir = buildChannel('stable', 'a', stableFiles); + const warn = composeSite({ + outDir: join(work, 'warn'), + stable: { dir: stableDir, source: source('stable', 'a') }, + next: null, + budget: { warn: 10, refuse: 10_000 }, + }); + expect(warn.budget.level).toBe('warn'); + expect(warn.problems).toEqual([]); + const refuse = composeSite({ + outDir: join(work, 'refuse'), + stable: { dir: stableDir, source: source('stable', 'a') }, + next: null, + budget: { warn: 10, refuse: 20 }, + }); + expect(refuse.budget.level).toBe('refuse'); + expect(refuse.problems.join('\n')).toMatch(/budget/); + }); +}); + +describe('extractComposedChannel', () => { + it('recovers each channel from a composed snapshot with its hash intact', () => { + const stableDir = buildChannel('stable', 'a', stableFiles); + const nextDir = buildChannel('next', '1', nextFiles); + const out = join(work, 'out'); + compose(out, stableDir, nextDir); + + const stableBack = join(work, 'stable-back'); + const nextBack = join(work, 'next-back'); + const stableManifest = extractComposedChannel(out, '', stableBack); + const nextManifest = extractComposedChannel(out, 'next/', nextBack); + + expect(stableManifest.contentHash).toBe(createSiteBuildManifest(stableDir, identity('stable', 'a')).contentHash); + expect(nextManifest.channel).toBe('next'); + expect(existsSync(join(stableBack, 'next'))).toBe(false); + expect(existsSync(join(stableBack, 'channels.json'))).toBe(false); + expect(hashTreeFiles(nextBack)).toEqual(hashTreeFiles(nextDir)); + }); +}); + +describe('buildChannelsDocument and parseChannelsDocument', () => { + it('derives the pairing from both channel identities only', () => { + const stableDir = buildChannel('stable', 'a', stableFiles); + const nextDir = buildChannel('next', '1', nextFiles); + const stable = createSiteBuildManifest(stableDir, identity('stable', 'a')); + const next = createSiteBuildManifest(nextDir, identity('next', '1')); + const first = buildChannelsDocument({ + stable: { manifest: stable, source: source('stable', 'a') }, + next: { manifest: next, source: source('next', '1') }, + }); + const again = buildChannelsDocument({ + stable: { manifest: stable, source: source('stable', 'a') }, + next: { manifest: next, source: source('next', '1') }, + }); + const stableOnly = buildChannelsDocument({ stable: { manifest: stable, source: source('stable', 'a') }, next: null }); + expect(first.pairing).toBe(again.pairing); + expect(stableOnly.pairing).not.toBe(first.pairing); + expect(parseChannelsDocument(JSON.parse(JSON.stringify(first)))).toEqual(first); + }); + + it('rejects a document without Stable or with a wrong path', () => { + expect(() => parseChannelsDocument({ schemaVersion: 1, pairing: 'x', channels: { stable: null, next: null } })).toThrow(/stable/); + expect(() => parseChannelsDocument(null)).toThrow(); + }); +}); + +describe('findCrossChannelReferences (W01)', () => { + it('flags a Next page that links deep into Stable, but not the explicit switch to the Stable root', () => { + const dir = buildChannel('next', '2', { + 'ok.html': 'xStable', + 'bad.html': '', + }); + const hits = findCrossChannelReferences(dir, '/ExoJS/next/', '/ExoJS/'); + expect(hits.map(hit => hit.file)).toEqual(['bad.html', 'bad.html']); + }); +}); diff --git a/test/site/pages-select-channels.test.ts b/test/site/pages-select-channels.test.ts new file mode 100644 index 000000000..87b802232 --- /dev/null +++ b/test/site/pages-select-channels.test.ts @@ -0,0 +1,431 @@ +import { describe, expect, it } from 'vitest'; + +import type { ChannelEntry, ChannelsDocument } from '../../scripts/pages/compose-site'; +import { type ArtifactMeta, type CandidateRun, selectChannels, type SelectionInput, type WorkflowRunMeta } from '../../scripts/pages/select-channels'; + +/** + * The publisher re-decides both channels every time it starts. The trigger + * only wakes it up: what is deployed is the newest qualified snapshot of each + * channel on its branch's current history, or the snapshot that is already + * live, never something older and never something untrusted. + */ + +const REPO = 'Exoridus/ExoJS'; +const CI_ID = 11; +const RELEASE_ID = 22; +const sha = (seed: string): string => seed.repeat(40).slice(0, 40); + +let nextId = 1000; + +const run = (overrides: Partial = {}): WorkflowRunMeta => ({ + id: nextId++, + attempt: 1, + workflowId: CI_ID, + workflowPath: '.github/workflows/ci.yml', + event: 'push', + status: 'completed', + conclusion: 'success', + headBranch: 'next', + headSha: sha('1'), + repository: REPO, + headRepository: REPO, + ...overrides, +}); + +const artifact = (owner: WorkflowRunMeta, overrides: Partial = {}): ArtifactMeta => ({ + id: owner.id * 10, + name: `site-${owner.headBranch === 'main' || owner.workflowId === RELEASE_ID ? 'stable' : 'next'}-${owner.headSha}-${owner.attempt}`, + expired: false, + digest: `sha256:${'d'.repeat(64)}`, + sizeInBytes: 1000, + runId: owner.id, + runHeadSha: owner.headSha, + ...overrides, +}); + +const candidate = (overrides: Partial = {}, artifactOverrides?: Partial | null): CandidateRun => { + const owner = run(overrides); + + return { run: owner, artifacts: artifactOverrides === null ? [] : [artifact(owner, artifactOverrides)] }; +}; + +const entryFor = (source: CandidateRun, channel: 'stable' | 'next'): ChannelEntry => { + const art = source.artifacts[0]!; + + return { + channel, + path: channel === 'stable' ? '' : 'next/', + basePath: channel === 'stable' ? '/ExoJS/' : '/ExoJS/next/', + sourceSha: source.run.headSha, + packageVersion: '0.19.1', + assetProfile: 'standard', + createdAt: '2026-10-20T10:00:00.000Z', + contentHash: `${channel === 'stable' ? 'a' : 'b'}${source.run.headSha.slice(0, 1)}`.repeat(32), + fileCount: 10, + totalBytes: 1000, + source: { + kind: source.run.workflowId === RELEASE_ID ? 'release' : 'ci', + workflowId: source.run.workflowId, + runId: source.run.id, + runAttempt: source.run.attempt, + artifactId: art.id, + artifactName: art.name, + digest: art.digest!, + }, + }; +}; + +const doc = (stable: ChannelEntry, next: ChannelEntry | null): ChannelsDocument => ({ + schemaVersion: 1, + pairing: 'p'.repeat(64), + channels: { stable, next }, +}); + +const PAGES_ARTIFACT: ArtifactMeta = { + id: 9_999, + name: 'github-pages', + expired: false, + digest: `sha256:${'e'.repeat(64)}`, + sizeInBytes: 1000, + runId: 900, + runHeadSha: null, +}; + +/** A history in which `shas` sit on the branch, newest first. */ +const history = (...shas: string[]): Record => Object.fromEntries(shas.map((value, index) => [value, index])); + +const base = (overrides: Partial = {}): SelectionInput => ({ + policy: { repository: REPO, ciWorkflowId: CI_ID, releaseWorkflowId: RELEASE_ID }, + trigger: run(), + branchHeads: { stable: sha('a'), next: sha('1') }, + history: { stable: history(sha('a')), next: history(sha('1')) }, + ciRuns: [], + releaseRuns: [], + previous: null, + ...overrides, +}); + +describe('bootstrap (W10)', () => { + it('publishes Stable alone while Next has never qualified', () => { + const stable = candidate({ headBranch: 'main', headSha: sha('a') }); + const result = selectChannels(base({ trigger: stable.run, ciRuns: [stable] })); + expect(result.deploy).toBe(true); + expect(result.channels.stable).toMatchObject({ action: 'update', sourceSha: sha('a') }); + expect(result.channels.next).toMatchObject({ action: 'absent' }); + }); + + it('refuses to deploy without any qualified Stable snapshot', () => { + const next = candidate({ headBranch: 'next', headSha: sha('1') }); + const result = selectChannels(base({ trigger: next.run, ciRuns: [next] })); + expect(result.deploy).toBe(false); + expect(result.problems.join('\n')).toMatch(/stable/i); + }); +}); + +describe('one channel moves (W03/W04)', () => { + const stableRun = candidate({ headBranch: 'main', headSha: sha('a') }); + const oldNext = candidate({ headBranch: 'next', headSha: sha('1') }); + const previous = { runId: 900, channels: doc(entryFor(stableRun, 'stable'), entryFor(oldNext, 'next')), pagesArtifact: PAGES_ARTIFACT }; + + it('updates only Next when only Next produced a newer candidate', () => { + const newNext = candidate({ headBranch: 'next', headSha: sha('2') }); + const result = selectChannels( + base({ + trigger: newNext.run, + ciRuns: [newNext, oldNext, stableRun], + branchHeads: { stable: sha('a'), next: sha('2') }, + history: { stable: history(sha('a')), next: history(sha('2'), sha('1')) }, + previous, + }), + ); + expect(result.deploy).toBe(true); + expect(result.channels.next).toMatchObject({ action: 'update', sourceSha: sha('2') }); + expect(result.channels.stable).toMatchObject({ action: 'keep' }); + }); + + it('updates only Stable when only main produced a newer candidate', () => { + const newStable = candidate({ headBranch: 'main', headSha: sha('b') }); + const result = selectChannels( + base({ + trigger: newStable.run, + ciRuns: [newStable, stableRun, oldNext], + branchHeads: { stable: sha('b'), next: sha('1') }, + history: { stable: history(sha('b'), sha('a')), next: history(sha('1')) }, + previous, + }), + ); + expect(result.channels.stable).toMatchObject({ action: 'update', sourceSha: sha('b') }); + expect(result.channels.next).toMatchObject({ action: 'keep' }); + }); + + it('keeps a channel from its original artifact first, then from the last published snapshot', () => { + const newNext = candidate({ headBranch: 'next', headSha: sha('2') }); + const input = base({ + trigger: newNext.run, + ciRuns: [newNext, stableRun], + branchHeads: { stable: sha('a'), next: sha('2') }, + history: { stable: history(sha('a')), next: history(sha('2'), sha('1')) }, + previous, + }); + const keep = selectChannels(input).channels.stable; + expect(keep.action).toBe('keep'); + expect(keep.action === 'keep' && keep.transports.map(transport => transport.kind)).toEqual(['artifact', 'snapshot']); + }); +}); + +describe('a green run without a site candidate (W05)', () => { + it('changes nothing and claims no new SHA', () => { + const stableRun = candidate({ headBranch: 'main', headSha: sha('a') }); + const nextRun = candidate({ headBranch: 'next', headSha: sha('1') }); + const bare = candidate({ headBranch: 'next', headSha: sha('2') }, null); + const result = selectChannels( + base({ + trigger: bare.run, + ciRuns: [bare, nextRun, stableRun], + branchHeads: { stable: sha('a'), next: sha('2') }, + history: { stable: history(sha('a')), next: history(sha('2'), sha('1')) }, + previous: { runId: 900, channels: doc(entryFor(stableRun, 'stable'), entryFor(nextRun, 'next')), pagesArtifact: PAGES_ARTIFACT }, + }), + ); + expect(result.deploy).toBe(false); + expect(result.reason).toMatch(/unchanged/); + expect(result.channels.next).toMatchObject({ action: 'keep', published: { sourceSha: sha('1') } }); + }); +}); + +describe('untrusted runs (W06)', () => { + const stable = candidate({ headBranch: 'main', headSha: sha('a') }); + + it.each<[string, Partial]>([ + ['a pull request', { event: 'pull_request' }], + ['a manual dispatch', { event: 'workflow_dispatch' }], + ['a fork', { headRepository: 'mallory/ExoJS' }], + ['another repository', { repository: 'mallory/ExoJS' }], + ['a failed run', { conclusion: 'failure' }], + ['a cancelled run', { conclusion: 'cancelled' }], + ['an unlisted workflow', { workflowId: 33, workflowPath: '.github/workflows/other.yml' }], + ['a feature branch', { headBranch: 'feature/x' }], + ])('deploys nothing when triggered by %s', (_label, overrides) => { + const result = selectChannels(base({ trigger: run({ headBranch: 'main', headSha: sha('a'), ...overrides }), ciRuns: [stable] })); + expect(result.deploy).toBe(false); + expect(result.reason).toMatch(/trigger/); + }); + + it.each<[string, Partial]>([ + ['a pull request', { event: 'pull_request' }], + ['a dispatch', { event: 'workflow_dispatch' }], + ['a fork', { headRepository: 'mallory/ExoJS' }], + ['a failed run', { conclusion: 'failure' }], + ['a still running run', { status: 'in_progress', conclusion: null }], + ])('never selects a candidate from %s', (_label, overrides) => { + const trusted = candidate({ headBranch: 'main', headSha: sha('a') }); + const hostile = candidate({ headBranch: 'main', headSha: sha('b'), ...overrides }); + const result = selectChannels( + base({ + trigger: trusted.run, + ciRuns: [hostile, trusted], + branchHeads: { stable: sha('b'), next: sha('1') }, + history: { stable: history(sha('b'), sha('a')), next: history(sha('1')) }, + }), + ); + expect(result.channels.stable).toMatchObject({ action: 'update', sourceSha: sha('a') }); + }); + + it('accepts a Release run only for a version tag push on main history', () => { + const release = candidate({ + workflowId: RELEASE_ID, + workflowPath: '.github/workflows/release.yml', + headBranch: 'v0.19.2', + headSha: sha('b'), + }); + const dispatched = candidate({ + workflowId: RELEASE_ID, + workflowPath: '.github/workflows/release.yml', + event: 'workflow_dispatch', + headBranch: 'main', + headSha: sha('c'), + }); + const input = base({ + trigger: release.run, + releaseRuns: [dispatched, release], + branchHeads: { stable: sha('c'), next: sha('1') }, + history: { stable: history(sha('c'), sha('b')), next: history(sha('1')) }, + }); + expect(selectChannels(input).channels.stable).toMatchObject({ action: 'update', sourceSha: sha('b'), source: { kind: 'release' } }); + expect(selectChannels({ ...input, trigger: dispatched.run }).deploy).toBe(false); + }); +}); + +describe('contradictory metadata (W07)', () => { + it.each<[string, Partial]>([ + ['another commit in the name', { name: `site-stable-${sha('f')}-1` }], + ['the other channel in the name', { name: `site-next-${sha('a')}-1` }], + ['a later attempt than the run has', { name: `site-stable-${sha('a')}-2` }], + ['another run', { runId: 1 }], + ['another head SHA', { runHeadSha: sha('f') }], + ['no digest', { digest: null }], + ['a malformed digest', { digest: 'md5:abc' }], + ['an expired artifact', { expired: true }], + ])('refuses an artifact with %s', (_label, overrides) => { + const stable = candidate({ headBranch: 'main', headSha: sha('a') }, overrides); + const result = selectChannels(base({ trigger: stable.run, ciRuns: [stable] })); + expect(result.channels.stable.action).not.toBe('update'); + expect(result.deploy).toBe(false); + }); + + it('refuses a candidate whose commit is not on the branch history', () => { + const stable = candidate({ headBranch: 'main', headSha: sha('a') }); + const result = selectChannels(base({ trigger: stable.run, ciRuns: [stable], history: { stable: { [sha('a')]: null }, next: {} } })); + expect(result.deploy).toBe(false); + }); + + it('accepts an earlier attempt of a re-run whose site job was not repeated', () => { + const stable = candidate({ headBranch: 'main', headSha: sha('a'), attempt: 2 }, { name: `site-stable-${sha('a')}-1` }); + expect(selectChannels(base({ trigger: stable.run, ciRuns: [stable] })).channels.stable).toMatchObject({ action: 'update' }); + }); +}); + +describe('expired artifacts (W08)', () => { + const stableRun = candidate({ headBranch: 'main', headSha: sha('a') }); + const nextRun = candidate({ headBranch: 'next', headSha: sha('1') }); + const published = doc(entryFor(stableRun, 'stable'), entryFor(nextRun, 'next')); + + it('keeps an unchanged channel from the last published snapshot when its candidate expired', () => { + const newNext = candidate({ headBranch: 'next', headSha: sha('2') }); + const expiredStable = { run: stableRun.run, artifacts: [{ ...stableRun.artifacts[0]!, expired: true }] }; + const result = selectChannels( + base({ + trigger: newNext.run, + ciRuns: [newNext, nextRun, expiredStable], + branchHeads: { stable: sha('a'), next: sha('2') }, + history: { stable: history(sha('a')), next: history(sha('2'), sha('1')) }, + previous: { runId: 900, channels: published, pagesArtifact: PAGES_ARTIFACT }, + }), + ); + expect(result.deploy).toBe(true); + expect(result.channels.stable.action === 'keep' && result.channels.stable.transports).toEqual([ + { kind: 'snapshot', runId: 900, artifactId: PAGES_ARTIFACT.id, digest: PAGES_ARTIFACT.digest, path: '' }, + ]); + }); + + it('falls back to the Release artifact of the same commit for Stable', () => { + const release = candidate({ + workflowId: RELEASE_ID, + workflowPath: '.github/workflows/release.yml', + headBranch: 'v0.19.1', + headSha: sha('a'), + }); + const newNext = candidate({ headBranch: 'next', headSha: sha('2') }); + const result = selectChannels( + base({ + trigger: newNext.run, + ciRuns: [newNext, nextRun], + releaseRuns: [release], + branchHeads: { stable: sha('a'), next: sha('2') }, + history: { stable: history(sha('a')), next: history(sha('2'), sha('1')) }, + previous: { runId: 900, channels: published, pagesArtifact: { ...PAGES_ARTIFACT, expired: true } }, + }), + ); + expect(result.channels.stable.action === 'keep' && result.channels.stable.transports.map(transport => transport.kind)).toEqual(['release']); + }); + + it('deploys nothing when no trusted source of a kept channel is left', () => { + const newNext = candidate({ headBranch: 'next', headSha: sha('2') }); + const result = selectChannels( + base({ + trigger: newNext.run, + ciRuns: [newNext], + branchHeads: { stable: sha('a'), next: sha('2') }, + history: { stable: history(sha('a')), next: history(sha('2'), sha('1')) }, + previous: { runId: 900, channels: published, pagesArtifact: null }, + }), + ); + expect(result.deploy).toBe(false); + expect(result.problems.join('\n')).toMatch(/stable/i); + }); +}); + +describe('ordering (W09)', () => { + const stableRun = candidate({ headBranch: 'main', headSha: sha('a') }); + + it('picks the commit nearest the branch head, not the run that finished last', () => { + const newer = candidate({ headBranch: 'next', headSha: sha('3') }); + const olderRerun = candidate({ headBranch: 'next', headSha: sha('2'), attempt: 2 }, { name: `site-next-${sha('2')}-2` }); + const result = selectChannels( + base({ + trigger: olderRerun.run, + ciRuns: [olderRerun, newer, stableRun], + branchHeads: { stable: sha('a'), next: sha('3') }, + history: { stable: history(sha('a')), next: history(sha('3'), sha('2')) }, + }), + ); + expect(result.channels.next).toMatchObject({ action: 'update', sourceSha: sha('3') }); + }); + + it('never rolls a published channel back to an older candidate', () => { + const published = candidate({ headBranch: 'next', headSha: sha('3') }); + const oldRerun = candidate({ headBranch: 'next', headSha: sha('2'), attempt: 3 }, { name: `site-next-${sha('2')}-3` }); + const result = selectChannels( + base({ + trigger: oldRerun.run, + ciRuns: [oldRerun, stableRun], + branchHeads: { stable: sha('a'), next: sha('4') }, + history: { stable: history(sha('a')), next: history(sha('4'), sha('3'), sha('2')) }, + previous: { runId: 900, channels: doc(entryFor(stableRun, 'stable'), entryFor(published, 'next')), pagesArtifact: PAGES_ARTIFACT }, + }), + ); + expect(result.channels.next).toMatchObject({ action: 'keep', published: { sourceSha: sha('3') } }); + expect(result.deploy).toBe(false); + }); + + it('picks up both channels when main and next finished in either order', () => { + const newStable = candidate({ headBranch: 'main', headSha: sha('b') }); + const newNext = candidate({ headBranch: 'next', headSha: sha('2') }); + const oldNext = candidate({ headBranch: 'next', headSha: sha('1') }); + const input = base({ + ciRuns: [newNext, newStable, oldNext, stableRun], + branchHeads: { stable: sha('b'), next: sha('2') }, + history: { stable: history(sha('b'), sha('a')), next: history(sha('2'), sha('1')) }, + previous: { runId: 900, channels: doc(entryFor(stableRun, 'stable'), entryFor(oldNext, 'next')), pagesArtifact: PAGES_ARTIFACT }, + }); + + for (const trigger of [newStable.run, newNext.run]) { + const result = selectChannels({ ...input, trigger }); + expect(result.channels.stable).toMatchObject({ action: 'update', sourceSha: sha('b') }); + expect(result.channels.next).toMatchObject({ action: 'update', sourceSha: sha('2') }); + } + }); + + it('replaces a published snapshot that left the branch history with the current one', () => { + const rewritten = candidate({ headBranch: 'next', headSha: sha('5') }); + const orphan = candidate({ headBranch: 'next', headSha: sha('4') }); + const result = selectChannels( + base({ + trigger: rewritten.run, + ciRuns: [rewritten, stableRun], + branchHeads: { stable: sha('a'), next: sha('5') }, + history: { stable: history(sha('a')), next: { [sha('5')]: 0, [sha('4')]: null } }, + previous: { runId: 900, channels: doc(entryFor(stableRun, 'stable'), entryFor(orphan, 'next')), pagesArtifact: PAGES_ARTIFACT }, + }), + ); + expect(result.channels.next).toMatchObject({ action: 'update', sourceSha: sha('5') }); + }); +}); + +describe('Next after its first publication', () => { + it('stays published when its newest candidate failed', () => { + const stableRun = candidate({ headBranch: 'main', headSha: sha('a') }); + const goodNext = candidate({ headBranch: 'next', headSha: sha('1') }); + const failed = candidate({ headBranch: 'next', headSha: sha('2'), conclusion: 'failure' }); + const result = selectChannels( + base({ + trigger: stableRun.run, + ciRuns: [failed, goodNext, stableRun], + branchHeads: { stable: sha('a'), next: sha('2') }, + history: { stable: history(sha('a')), next: history(sha('2'), sha('1')) }, + previous: { runId: 900, channels: doc(entryFor(stableRun, 'stable'), entryFor(goodNext, 'next')), pagesArtifact: PAGES_ARTIFACT }, + }), + ); + expect(result.channels.next).toMatchObject({ action: 'keep', published: { sourceSha: sha('1') } }); + }); +}); diff --git a/test/site/pages-site-channel.test.ts b/test/site/pages-site-channel.test.ts new file mode 100644 index 000000000..ce072f603 --- /dev/null +++ b/test/site/pages-site-channel.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from 'vitest'; + +import { + channelForBranch, + counterpartBasePath, + isSafeBasePath, + resolveSiteChannel, + SITE_CHANNEL_BASE, + siteChannelOutputs, +} from '../../scripts/pages/site-channel'; + +/** + * The channel build contract: `EXOJS_SITE_CHANNEL` and `EXOJS_SITE_BASE` + * decide the base path every generated URL carries, so an invalid or + * contradictory pair must never reach a build. + */ + +describe('resolveSiteChannel', () => { + it('keeps the stable site at /ExoJS/ when nothing is configured', () => { + expect(resolveSiteChannel({})).toEqual({ channel: 'stable', basePath: '/ExoJS/', localTest: false }); + }); + + it('derives the canonical base from the channel', () => { + expect(resolveSiteChannel({ EXOJS_SITE_CHANNEL: 'next' })).toEqual({ channel: 'next', basePath: '/ExoJS/next/', localTest: false }); + expect(resolveSiteChannel({ EXOJS_SITE_CHANNEL: 'stable' }).basePath).toBe('/ExoJS/'); + }); + + it('accepts the canonical pairs and infers the channel from a canonical base', () => { + expect(resolveSiteChannel({ EXOJS_SITE_CHANNEL: 'next', EXOJS_SITE_BASE: '/ExoJS/next/' }).channel).toBe('next'); + expect(resolveSiteChannel({ EXOJS_SITE_BASE: '/ExoJS/next/' }).channel).toBe('next'); + expect(resolveSiteChannel({ EXOJS_SITE_BASE: '/ExoJS/' }).channel).toBe('stable'); + }); + + it('treats empty values as unset', () => { + expect(resolveSiteChannel({ EXOJS_SITE_CHANNEL: '', EXOJS_SITE_BASE: '' }).channel).toBe('stable'); + }); + + it('rejects an unknown channel', () => { + expect(() => resolveSiteChannel({ EXOJS_SITE_CHANNEL: 'beta' })).toThrow(/EXOJS_SITE_CHANNEL/); + expect(() => resolveSiteChannel({ EXOJS_SITE_CHANNEL: 'Next' })).toThrow(/EXOJS_SITE_CHANNEL/); + }); + + it('rejects a contradictory channel/base pair', () => { + expect(() => resolveSiteChannel({ EXOJS_SITE_CHANNEL: 'stable', EXOJS_SITE_BASE: '/ExoJS/next/' })).toThrow(/contradicts/); + expect(() => resolveSiteChannel({ EXOJS_SITE_CHANNEL: 'next', EXOJS_SITE_BASE: '/ExoJS/' })).toThrow(/contradicts/); + }); + + it('rejects a non-canonical base outside local test mode', () => { + expect(() => resolveSiteChannel({ EXOJS_SITE_BASE: '/preview/' })).toThrow(/EXOJS_SITE_LOCAL_TEST/); + expect(() => resolveSiteChannel({ EXOJS_SITE_CHANNEL: 'next', EXOJS_SITE_BASE: '/ExoJS/next' })).toThrow(); + }); + + it('allows a safe custom base only in explicit local test mode, and never on CI', () => { + expect(resolveSiteChannel({ EXOJS_SITE_CHANNEL: 'next', EXOJS_SITE_BASE: '/sandbox/next/', EXOJS_SITE_LOCAL_TEST: '1' })).toEqual({ + channel: 'next', + basePath: '/sandbox/next/', + localTest: true, + }); + expect(() => resolveSiteChannel({ EXOJS_SITE_CHANNEL: 'next', EXOJS_SITE_BASE: '/sandbox/', EXOJS_SITE_LOCAL_TEST: '1', GITHUB_ACTIONS: 'true' })).toThrow( + /CI/, + ); + expect(() => resolveSiteChannel({ EXOJS_SITE_BASE: '/sandbox/', EXOJS_SITE_LOCAL_TEST: '1' })).toThrow(/EXOJS_SITE_CHANNEL/); + }); + + it('rejects an unsafe base even in local test mode', () => { + for (const base of ['/a/../b/', '/a//b/', 'relative/', '/a b/', '/a/"x/', "/a/'/", '/a//', '/a/$(id)/', '//evil.example/', '/a/./']) { + expect(() => resolveSiteChannel({ EXOJS_SITE_CHANNEL: 'stable', EXOJS_SITE_BASE: base, EXOJS_SITE_LOCAL_TEST: '1' }), `${base}`).toThrow(); + } + }); +}); + +describe('isSafeBasePath', () => { + it('accepts the canonical bases and the root', () => { + expect(isSafeBasePath('/')).toBe(true); + expect(isSafeBasePath(SITE_CHANNEL_BASE.stable)).toBe(true); + expect(isSafeBasePath(SITE_CHANNEL_BASE.next)).toBe(true); + }); + + it('refuses a base without both slashes', () => { + expect(isSafeBasePath('/ExoJS')).toBe(false); + expect(isSafeBasePath('ExoJS/')).toBe(false); + expect(isSafeBasePath('')).toBe(false); + }); +}); + +describe('channel relations', () => { + it('maps the long-lived branches to their channels and nothing else', () => { + expect(channelForBranch('main')).toBe('stable'); + expect(channelForBranch('next')).toBe('next'); + expect(channelForBranch('feature/x')).toBeNull(); + expect(channelForBranch('refs/heads/main')).toBeNull(); + }); + + it('names the other channel as the explicit switch target, but not for a local test base', () => { + expect(counterpartBasePath({ channel: 'next', basePath: '/ExoJS/next/', localTest: false })).toBe('/ExoJS/'); + expect(counterpartBasePath({ channel: 'stable', basePath: '/ExoJS/', localTest: false })).toBe('/ExoJS/next/'); + expect(counterpartBasePath({ channel: 'next', basePath: '/sandbox/', localTest: true })).toBeNull(); + }); +}); + +describe('siteChannelOutputs', () => { + it('builds a long-lived branch for its channel', () => { + expect(siteChannelOutputs('next')).toEqual(['channel=next', 'base=/ExoJS/next/']); + expect(siteChannelOutputs('main')).toEqual(['channel=stable', 'base=/ExoJS/']); + }); + + it('builds any other ref as stable, the unconfigured default', () => { + expect(siteChannelOutputs('feature/x')).toEqual(['channel=stable', 'base=/ExoJS/']); + expect(siteChannelOutputs('')).toEqual(['channel=stable', 'base=/ExoJS/']); + }); +}); diff --git a/test/site/pages-site-manifest.test.ts b/test/site/pages-site-manifest.test.ts new file mode 100644 index 000000000..a06f0612f --- /dev/null +++ b/test/site/pages-site-manifest.test.ts @@ -0,0 +1,179 @@ +import { createHash } from 'node:crypto'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { + computeContentHash, + createSiteBuildManifest, + hashTreeFiles, + parseSiteBuildManifest, + SITE_BUILD_FILE, + SITE_BUILD_SCHEMA_VERSION, + type SiteBuildIdentity, + validateSitePath, + verifySiteTree, +} from '../../scripts/pages/site-manifest'; + +/** + * `site-build.json` identifies one built channel tree: which commit, version, + * channel and base it was built for, and exactly which bytes it consists of. + * The publisher trusts a tree only when it matches this manifest, so the + * format, the path rules and the hash are pinned here. + */ + +let tree: string; + +const write = (rel: string, content: string): void => { + const abs = join(tree, rel); + mkdirSync(join(abs, '..'), { recursive: true }); + writeFileSync(abs, content); +}; + +const identity: SiteBuildIdentity = { + channel: 'next', + basePath: '/ExoJS/next/', + sourceSha: 'a'.repeat(40), + packageVersion: '0.19.1', + assetProfile: 'standard', + createdAt: '2026-10-21T12:00:00.000Z', + localTest: false, +}; + +beforeEach(() => { + tree = mkdtempSync(join(tmpdir(), 'exo-site-manifest-')); + write('index.html', ''); + write('en/index.html', 'en'); + write('vendor/exojs/esm/index.js', 'export {};'); + write('B.txt', 'upper'); + write('a.txt', 'lower'); +}); + +afterEach(() => rmSync(tree, { recursive: true, force: true })); + +describe('validateSitePath', () => { + it('accepts plain relative POSIX paths', () => { + for (const path of ['index.html', 'en/guide/index.html', '_astro/x.CbA9.js', 'pagefind/fragment/en_1.pf_fragment', '.nojekyll']) { + expect(validateSitePath(path), `${path}`).toBeNull(); + } + }); + + it('rejects absolute, traversing, Windows-shaped and control-character paths', () => { + for (const path of ['', '/etc/passwd', '../outside', 'a/../../b', 'a/./b', 'a//b', 'a/', 'C:/x', 'c:x', 'a\\b', 'a\u0000b', 'a\nb', 'a:b', '.', '..']) { + expect(validateSitePath(path), `${JSON.stringify(path)}`).not.toBeNull(); + } + }); +}); + +describe('hashTreeFiles and computeContentHash', () => { + it('lists every file once, sorted by code unit, with size and SHA-256', () => { + const files = hashTreeFiles(tree); + expect(files.map(file => file.path)).toEqual(['B.txt', 'a.txt', 'en/index.html', 'index.html', 'vendor/exojs/esm/index.js']); + expect(files.find(file => file.path === 'a.txt')).toEqual({ + path: 'a.txt', + size: 5, + sha256: createHash('sha256').update('lower').digest('hex'), + }); + }); + + it('is deterministic and changes with any byte or path', () => { + const first = computeContentHash(hashTreeFiles(tree)); + expect(computeContentHash(hashTreeFiles(tree))).toBe(first); + write('a.txt', 'Lower'); + const edited = computeContentHash(hashTreeFiles(tree)); + expect(edited).not.toBe(first); + rmSync(join(tree, 'a.txt')); + write('c.txt', 'Lower'); + expect(computeContentHash(hashTreeFiles(tree))).not.toBe(edited); + }); +}); + +describe('createSiteBuildManifest', () => { + it('records the identity, the file list and its hash, excluding itself', () => { + write(SITE_BUILD_FILE, '{"stale":true}'); + const manifest = createSiteBuildManifest(tree, identity); + + expect(manifest).toMatchObject({ schemaVersion: SITE_BUILD_SCHEMA_VERSION, ...identity, fileCount: 5 }); + expect(manifest.files.some(file => file.path === SITE_BUILD_FILE)).toBe(false); + expect(manifest.totalBytes).toBe(manifest.files.reduce((sum, file) => sum + file.size, 0)); + expect(manifest.contentHash).toBe(computeContentHash(manifest.files)); + }); + + it('only excludes the manifest at the tree root', () => { + write(`nested/${SITE_BUILD_FILE}`, '{}'); + expect(createSiteBuildManifest(tree, identity).files.some(file => file.path === `nested/${SITE_BUILD_FILE}`)).toBe(true); + }); + + it('refuses an identity that is not publishable as declared', () => { + expect(() => createSiteBuildManifest(tree, { ...identity, sourceSha: 'abc' })).toThrow(/sourceSha/); + expect(() => createSiteBuildManifest(tree, { ...identity, packageVersion: '0.20' })).toThrow(/packageVersion/); + expect(() => createSiteBuildManifest(tree, { ...identity, basePath: '/ExoJS/' })).toThrow(/basePath/); + expect(() => createSiteBuildManifest(tree, { ...identity, createdAt: 'yesterday' })).toThrow(/createdAt/); + expect(() => createSiteBuildManifest(tree, { ...identity, assetProfile: 'tiny' as 'standard' })).toThrow(/assetProfile/); + }); + + it('keeps a local test base and flags it, so the publisher can refuse it', () => { + const manifest = createSiteBuildManifest(tree, { ...identity, basePath: '/sandbox/', localTest: true }); + expect(manifest).toMatchObject({ basePath: '/sandbox/', localTest: true }); + }); +}); + +describe('parseSiteBuildManifest', () => { + const valid = (): Record => JSON.parse(JSON.stringify(createSiteBuildManifest(tree, identity))) as Record; + + it('round-trips a written manifest', () => { + const manifest = createSiteBuildManifest(tree, identity); + expect(parseSiteBuildManifest(JSON.parse(JSON.stringify(manifest)))).toEqual(manifest); + }); + + it('rejects an unknown schema version, a missing field and a forged count or hash', () => { + expect(() => parseSiteBuildManifest({ ...valid(), schemaVersion: 2 })).toThrow(/schemaVersion/); + const { channel: _channel, ...withoutChannel } = valid(); + expect(() => parseSiteBuildManifest(withoutChannel)).toThrow(/channel/); + expect(() => parseSiteBuildManifest({ ...valid(), fileCount: 99 })).toThrow(/fileCount/); + expect(() => parseSiteBuildManifest({ ...valid(), contentHash: 'f'.repeat(64) })).toThrow(/contentHash/); + }); + + it('rejects unsafe, duplicate or unsorted file paths', () => { + const files = valid()['files'] as Array>; + expect(() => parseSiteBuildManifest({ ...valid(), files: [{ ...files[0], path: '../x' }, ...files.slice(1)] })).toThrow(/path/); + expect(() => parseSiteBuildManifest({ ...valid(), files: [files[0], ...files] })).toThrow(/sorted|duplicate/); + expect(() => parseSiteBuildManifest({ ...valid(), files: [...files].reverse() })).toThrow(/sorted|duplicate/); + }); + + it('rejects a non-object and a list that names the manifest itself', () => { + expect(() => parseSiteBuildManifest(null)).toThrow(); + expect(() => parseSiteBuildManifest('{}')).toThrow(); + const files = valid()['files'] as Array>; + const selfListed = [...files, { path: SITE_BUILD_FILE, size: 1, sha256: '0'.repeat(64) }].sort((a, b) => (String(a['path']) < String(b['path']) ? -1 : 1)); + expect(() => parseSiteBuildManifest({ ...valid(), files: selfListed, fileCount: selfListed.length })).toThrow(SITE_BUILD_FILE); + }); +}); + +describe('verifySiteTree', () => { + it('passes the tree the manifest was made from', () => { + const manifest = createSiteBuildManifest(tree, identity); + write(SITE_BUILD_FILE, JSON.stringify(manifest)); + expect(verifySiteTree(tree, manifest)).toEqual([]); + }); + + it('reports a changed, a missing and an unexpected file', () => { + const manifest = createSiteBuildManifest(tree, identity); + write('a.txt', 'tampered'); + rmSync(join(tree, 'B.txt')); + write('extra.js', 'alert(1)'); + const problems = verifySiteTree(tree, manifest).join('\n'); + expect(problems).toMatch(/a\.txt/); + expect(problems).toMatch(/missing.*B\.txt/); + expect(problems).toMatch(/unexpected.*extra\.js/); + }); + + it('tolerates only the extra paths the caller names', () => { + const manifest = createSiteBuildManifest(tree, identity); + write('next/index.html', 'next'); + expect(verifySiteTree(tree, manifest, { allowExtra: path => path.startsWith('next/') })).toEqual([]); + expect(verifySiteTree(tree, manifest)).not.toEqual([]); + }); +}); diff --git a/test/site/pages-stamp-stable-site.test.ts b/test/site/pages-stamp-stable-site.test.ts new file mode 100644 index 000000000..889246cf2 --- /dev/null +++ b/test/site/pages-stamp-stable-site.test.ts @@ -0,0 +1,71 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { verifyChannelTree } from '../../scripts/pages/channel-artifact'; +import { SITE_BUILD_FILE } from '../../scripts/pages/site-manifest'; +import { stampStableSite } from '../../scripts/pages/stamp-stable-site'; + +/** + * The Stable site build writes no manifest of its own; the stamp records the + * finished tree so it passes the same candidate verification as a channel + * build, and refuses anything that is not a Stable tree. + */ + +const SHA = 'a'.repeat(40); + +let work: string; + +const options = (dir: string) => ({ dir, sourceSha: SHA, packageVersion: '0.19.1', createdAt: '2026-10-22T12:00:00.000Z' }); + +const buildTree = (entry = ''): string => { + const dir = join(work, 'tree'); + mkdirSync(join(dir, 'en'), { recursive: true }); + writeFileSync(join(dir, 'index.html'), entry); + writeFileSync(join(dir, 'en', 'index.html'), '

home

'); + + return dir; +}; + +beforeEach(() => { + work = mkdtempSync(join(tmpdir(), 'exojs-stamp-')); +}); + +afterEach(() => { + rmSync(work, { recursive: true, force: true }); +}); + +describe('stampStableSite', () => { + it('records a tree that passes the Stable candidate verification', () => { + const dir = buildTree(); + + stampStableSite(options(dir)); + const { problems, manifest } = verifyChannelTree(dir, { channel: 'stable', sourceSha: SHA }); + + expect(problems).toEqual([]); + expect(manifest?.basePath).toBe('/ExoJS/'); + expect(manifest?.files.map(file => file.path)).toEqual(['en/index.html', 'index.html']); + }); + + it('refuses a tree that already carries a manifest', () => { + const dir = buildTree(); + writeFileSync(join(dir, SITE_BUILD_FILE), '{}'); + + expect(() => stampStableSite(options(dir))).toThrow(/already exists/); + }); + + it('refuses a tree without an entry page', () => { + const dir = join(work, 'empty'); + mkdirSync(dir); + + expect(() => stampStableSite(options(dir))).toThrow(/no index\.html/); + }); + + it('refuses a tree rendered for another base', () => { + const dir = buildTree(''); + + expect(() => stampStableSite(options(dir))).toThrow(/Stable base/); + }); +}); diff --git a/test/support/archive-fixtures.ts b/test/support/archive-fixtures.ts new file mode 100644 index 000000000..db440cd83 --- /dev/null +++ b/test/support/archive-fixtures.ts @@ -0,0 +1,168 @@ +import { crc32, deflateRawSync } from 'node:zlib'; + +/** + * Hand-built archives for the Pages archive-safety suites. The writers emit + * exactly what they are told, including entries no well-behaved archiver + * would produce, so the readers can be tested against hostile input. + */ + +export interface ZipFixtureEntry { + name: string; + data?: Buffer | string; + /** Unix mode including the file-type bits; defaults to a regular file or a directory by name. */ + mode?: number; + /** Stored instead of deflated. */ + store?: boolean; + /** Overrides the uncompressed size written to the headers. */ + declaredSize?: number; + /** Sets the encryption flag. */ + encrypted?: boolean; +} + +const u16 = (value: number): Buffer => { + const buffer = Buffer.alloc(2); + buffer.writeUInt16LE(value); + + return buffer; +}; + +const u32 = (value: number): Buffer => { + const buffer = Buffer.alloc(4); + buffer.writeUInt32LE(value >>> 0); + + return buffer; +}; + +export const createZip = (entries: readonly ZipFixtureEntry[]): Buffer => { + const locals: Buffer[] = []; + const centrals: Buffer[] = []; + let offset = 0; + + for (const entry of entries) { + const name = Buffer.from(entry.name, 'utf8'); + const raw = Buffer.isBuffer(entry.data) ? entry.data : Buffer.from(entry.data ?? '', 'utf8'); + const isDirectory = entry.name.endsWith('/'); + const method = entry.store || isDirectory ? 0 : 8; + const payload = method === 8 ? deflateRawSync(raw) : raw; + const size = entry.declaredSize ?? raw.length; + const mode = entry.mode ?? (isDirectory ? 0o040755 : 0o100644); + const flags = 0x0800 | (entry.encrypted ? 0x0001 : 0); + const crc = crc32(raw); + + const local = Buffer.concat([ + u32(0x04034b50), + u16(20), + u16(flags), + u16(method), + u16(0), + u16(0), + u32(crc), + u32(payload.length), + u32(size), + u16(name.length), + u16(0), + name, + payload, + ]); + const central = Buffer.concat([ + u32(0x02014b50), + u16((3 << 8) | 20), + u16(20), + u16(flags), + u16(method), + u16(0), + u16(0), + u32(crc), + u32(payload.length), + u32(size), + u16(name.length), + u16(0), + u16(0), + u16(0), + u16(0), + u32(mode << 16), + u32(offset), + name, + ]); + + locals.push(local); + centrals.push(central); + offset += local.length; + } + + const centralDirectory = Buffer.concat(centrals); + const end = Buffer.concat([u32(0x06054b50), u16(0), u16(0), u16(entries.length), u16(entries.length), u32(centralDirectory.length), u32(offset), u16(0)]); + + return Buffer.concat([...locals, centralDirectory, end]); +}; + +export interface TarFixtureEntry { + name: string; + data?: Buffer | string; + /** Tar type flag: `0` file, `5` directory, `2` symlink, `1` hardlink, `3`-`7` special. */ + type?: string; + linkName?: string; + /** Emits the path through a pax extended header instead of the ustar name field. */ + pax?: boolean; +} + +const octal = (value: number, width: number): string => `${value.toString(8).padStart(width - 1, '0')}\0`; + +const tarHeader = (name: string, size: number, type: string, linkName = ''): Buffer => { + const header = Buffer.alloc(512); + header.write(name.slice(0, 100), 0, 'utf8'); + header.write(octal(type === '5' ? 0o755 : 0o644, 8), 100, 'ascii'); + header.write(octal(0, 8), 108, 'ascii'); + header.write(octal(0, 8), 116, 'ascii'); + header.write(octal(size, 12), 124, 'ascii'); + header.write(octal(0, 12), 136, 'ascii'); + header.write(' ', 148, 'ascii'); + header.write(type, 156, 'ascii'); + header.write(linkName.slice(0, 100), 157, 'utf8'); + header.write('ustar\0', 257, 'ascii'); + header.write('00', 263, 'ascii'); + + let sum = 0; + + for (const byte of header) { + sum += byte; + } + + header.write(`${sum.toString(8).padStart(6, '0')}\0 `, 148, 'ascii'); + + return header; +}; + +const padded = (data: Buffer): Buffer => Buffer.concat([data, Buffer.alloc((512 - (data.length % 512)) % 512)]); + +export const createTar = (entries: readonly TarFixtureEntry[]): Buffer => { + const blocks: Buffer[] = []; + + for (const entry of entries) { + const data = Buffer.isBuffer(entry.data) ? entry.data : Buffer.from(entry.data ?? '', 'utf8'); + const type = entry.type ?? (entry.name.endsWith('/') ? '5' : '0'); + + if (entry.pax) { + const body = ` path=${entry.name}\n`; + let length = body.length; + + while (`${length}${body}`.length !== length) { + length = `${length}${body}`.length; + } + + const record = Buffer.from(`${length}${body}`, 'utf8'); + blocks.push(tarHeader('PaxHeader', record.length, 'x'), padded(record)); + blocks.push(tarHeader('placeholder', type === '0' ? data.length : 0, type, entry.linkName)); + } else { + blocks.push(tarHeader(entry.name, type === '0' ? data.length : 0, type, entry.linkName)); + } + + if (type === '0') { + blocks.push(padded(data)); + } + } + + blocks.push(Buffer.alloc(1024)); + + return Buffer.concat(blocks); +};