From 865db06f70ed5970a8c897f22d89f4941a5cef67 Mon Sep 17 00:00:00 2001 From: Michael Gilardi <11632652+mgilardi@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:20:43 -0700 Subject: [PATCH 1/2] Turn import jobs on in the example site The example left imports off, so the import panel had never run in a real admin. It now has a curators-only upload collection for source files, two seeded jobs over made-up files (one waiting, one that fails on purpose), and smoke checks that visitors and members can't read a source file, a job, run one or upload, and that running the waiting job gives the committed fixture's checksum. Uploaded files go in a throwaway folder that is deleted when the site stops; the site now waits for the server to exit before taking it away. Not yet clicked through in a browser: that follows in this branch. Co-Authored-By: Claude --- apps/payload-example/.gitignore | 1 + .../app/(payload)/admin/importMap.js | 2 + apps/payload-example/payload.config.ts | 9 ++- apps/payload-example/scripts/run.mjs | 17 +++- apps/payload-example/scripts/smoke.mjs | 77 ++++++++++++++++++- apps/payload-example/src/seed.ts | 44 ++++++++++- apps/payload-example/src/sources.ts | 22 ++++++ 7 files changed, 163 insertions(+), 9 deletions(-) create mode 100644 apps/payload-example/src/sources.ts diff --git a/apps/payload-example/.gitignore b/apps/payload-example/.gitignore index 69b299b..26e1654 100644 --- a/apps/payload-example/.gitignore +++ b/apps/payload-example/.gitignore @@ -1,3 +1,4 @@ .next/ next-env.d.ts tsconfig.tsbuildinfo +.uploads/ diff --git a/apps/payload-example/app/(payload)/admin/importMap.js b/apps/payload-example/app/(payload)/admin/importMap.js index ba2a071..2355697 100644 --- a/apps/payload-example/app/(payload)/admin/importMap.js +++ b/apps/payload-example/app/(payload)/admin/importMap.js @@ -1,5 +1,6 @@ import { SnapshotPanel as SnapshotPanel_577f3b5b503ff777fe77b01de8cacfa5 } from '@captiveorigins/payload-plugin-onetaxonomy/client' import { ViewPanel as ViewPanel_577f3b5b503ff777fe77b01de8cacfa5 } from '@captiveorigins/payload-plugin-onetaxonomy/client' +import { ImportPanel as ImportPanel_577f3b5b503ff777fe77b01de8cacfa5 } from '@captiveorigins/payload-plugin-onetaxonomy/client' import { ProposalPanel as ProposalPanel_577f3b5b503ff777fe77b01de8cacfa5 } from '@captiveorigins/payload-plugin-onetaxonomy/client' import { CollectionCards as CollectionCards_f9c02e79a4aed9a3924487c0cd4cafb1 } from '@payloadcms/next/rsc' @@ -7,6 +8,7 @@ import { CollectionCards as CollectionCards_f9c02e79a4aed9a3924487c0cd4cafb1 } f export const importMap = { "@captiveorigins/payload-plugin-onetaxonomy/client#SnapshotPanel": SnapshotPanel_577f3b5b503ff777fe77b01de8cacfa5, "@captiveorigins/payload-plugin-onetaxonomy/client#ViewPanel": ViewPanel_577f3b5b503ff777fe77b01de8cacfa5, + "@captiveorigins/payload-plugin-onetaxonomy/client#ImportPanel": ImportPanel_577f3b5b503ff777fe77b01de8cacfa5, "@captiveorigins/payload-plugin-onetaxonomy/client#ProposalPanel": ProposalPanel_577f3b5b503ff777fe77b01de8cacfa5, "@payloadcms/next/rsc#CollectionCards": CollectionCards_f9c02e79a4aed9a3924487c0cd4cafb1 } diff --git a/apps/payload-example/payload.config.ts b/apps/payload-example/payload.config.ts index b876c08..7fd13e0 100644 --- a/apps/payload-example/payload.config.ts +++ b/apps/payload-example/payload.config.ts @@ -6,6 +6,7 @@ import { mongooseAdapter } from '@payloadcms/db-mongodb' import { buildConfig } from 'payload' import { seed } from './src/seed' +import { sources } from './src/sources' import { isCurator, users } from './src/users' const dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -19,7 +20,9 @@ export default buildConfig({ secret: process.env.PAYLOAD_SECRET || '', // Payload's default avatar asks gravatar.com for a picture by a hash of the signed-in address. This example asks nobody. admin: { user: users.slug, avatar: 'default', importMap: { baseDir: dirname } }, - collections: [users], + // Uploaded source files are kept on disk. `pnpm dev` names a throwaway folder and deletes it when it stops; with a + // database of your own they go in `.uploads` here, which git ignores. + collections: [users, sources(process.env.EXAMPLE_UPLOADS || path.join(dirname, '.uploads'))], // `ensureIndexes` builds the indexes before the first request, so the seed's first writes don't race them on an empty database. db: mongooseAdapter({ url: process.env.DATABASE_URI || '', ensureIndexes: true }), plugins: [ @@ -31,6 +34,10 @@ export default buildConfig({ // `propose` is left at its default: any signed-in person. }, locale: { default: 'en' }, + // Import jobs read a curator's uploaded file and store what they make as a staged snapshot. A new job waits + // until a curator runs it from its page; the plugin's default runs it as soon as it is made. There is no + // `fetch`, so a job can't name a URL. + imports: { uploads: { collection: 'sources' }, runOnCreate: false }, proposals: { // `personId` is left at its default, the account's id: that is all a proposal stores and all an accepted one // seals. The handle is looked up here each time a proposal is shown. A member can't read other accounts, so diff --git a/apps/payload-example/scripts/run.mjs b/apps/payload-example/scripts/run.mjs index 08f73b0..8f10a77 100644 --- a/apps/payload-example/scripts/run.mjs +++ b/apps/payload-example/scripts/run.mjs @@ -1,10 +1,13 @@ #!/usr/bin/env node // Runs the example site: `node scripts/run.mjs dev|build|start`. // Unless DATABASE_URI names a database, it starts a throwaway in-memory MongoDB first (data is gone when it stops) -// and seeds two accounts with a password made up for this run. The site listens on this machine only. +// and seeds two accounts with a password made up for this run. Uploaded files then go in a throwaway folder that is +// deleted when it stops. The site listens on this machine only. import { spawn } from 'node:child_process' import { randomBytes } from 'node:crypto' +import { mkdtempSync, rmSync } from 'node:fs' import { createRequire } from 'node:module' +import os from 'node:os' import path from 'node:path' import { fileURLToPath } from 'node:url' @@ -22,6 +25,7 @@ env.EXAMPLE_URL ||= `http://localhost:${port}` env.PAYLOAD_SECRET ||= randomBytes(32).toString('hex') let mongo = null +let uploads = null if (command === 'build') { // Building connects to nothing: every page here is rendered per request. env.DATABASE_URI ||= 'mongodb://127.0.0.1:27017/not-used-while-building' @@ -29,9 +33,12 @@ if (command === 'build') { const { MongoMemoryReplSet } = await import('mongodb-memory-server') mongo = await MongoMemoryReplSet.create({ replSet: { count: 1, storageEngine: 'wiredTiger' } }) env.DATABASE_URI = mongo.getUri('onetaxonomy-example') + // Uploaded files live as long as the database they belong to. A folder you named yourself is left alone. + if (!env.EXAMPLE_UPLOADS) env.EXAMPLE_UPLOADS = uploads = mkdtempSync(path.join(os.tmpdir(), 'onetaxonomy-example-uploads-')) const made = !env.EXAMPLE_PASSWORD env.EXAMPLE_PASSWORD ||= randomBytes(12).toString('base64url') console.log(`\nIn-memory database started. Nothing is kept after this stops.`) + if (uploads) console.log(`Uploads: ${uploads} (deleted when this stops)`) console.log(`Site: ${env.EXAMPLE_URL}`) console.log(`Admin: ${env.EXAMPLE_URL}/admin`) console.log(`Curator: curator@example.org`) @@ -47,8 +54,14 @@ let stopping = false async function stop(code) { if (stopping) return stopping = true - if (child.exitCode === null && child.signalCode === null) child.kill('SIGTERM') + if (child.exitCode === null && child.signalCode === null) { + // Wait for the site to go (five seconds at most) before taking its database and its files away. + const gone = new Promise((resolve) => child.once('exit', resolve)) + child.kill('SIGTERM') + await Promise.race([gone, new Promise((resolve) => setTimeout(resolve, 5000))]) + } if (mongo) await mongo.stop().catch(() => {}) + if (uploads) rmSync(uploads, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }) process.exit(code) } child.on('exit', (code, signal) => void stop(code ?? (signal ? 1 : 0))) diff --git a/apps/payload-example/scripts/smoke.mjs b/apps/payload-example/scripts/smoke.mjs index e1c4e57..b95cfaa 100644 --- a/apps/payload-example/scripts/smoke.mjs +++ b/apps/payload-example/scripts/smoke.mjs @@ -9,6 +9,7 @@ // It makes one more account and one proposal, so it always starts its own site and never checks one you have open. import { spawn } from 'node:child_process' import { randomBytes } from 'node:crypto' +import { existsSync, readFileSync } from 'node:fs' import { createServer } from 'node:net' import path from 'node:path' import { fileURLToPath } from 'node:url' @@ -40,8 +41,9 @@ const finished = (child) => new Promise((resolve) => (child.exitCode !== null || const port = await freePort() const site = `http://localhost:${port}` const password = randomBytes(12).toString('base64url') -// DATABASE_URI is cleared so the checks never run against a database you named for something else. -const env = { ...process.env, PORT: String(port), EXAMPLE_URL: site, EXAMPLE_PASSWORD: password, DATABASE_URI: '' } +// DATABASE_URI and EXAMPLE_UPLOADS are cleared so the checks never run against a database or a folder you named for +// something else. +const env = { ...process.env, PORT: String(port), EXAMPLE_URL: site, EXAMPLE_PASSWORD: password, DATABASE_URI: '', EXAMPLE_UPLOADS: '' } if (!flags.has('--dev') && !flags.has('--skip-build')) { const started = Date.now() @@ -55,7 +57,14 @@ if (!flags.has('--dev') && !flags.has('--skip-build')) { const server = run([flags.has('--dev') ? 'dev' : 'start'], env, true) let log = '' -for (const stream of [server.stdout, server.stderr]) stream.on('data', (chunk) => (log = (log + chunk).slice(-20_000))) +// The throwaway folder the site keeps uploaded files in, as it says when it starts. +let uploads = null +for (const stream of [server.stdout, server.stderr]) + stream.on('data', (chunk) => { + log = (log + chunk).slice(-20_000) + uploads ??= log.match(/^Uploads: (.+) \(deleted when this stops\)$/m)?.[1] ?? null + }) +const fixture = (name) => readFileSync(path.join(app, '../../fixtures', name)) let failed = 0 let passed = 0 @@ -89,6 +98,15 @@ async function call(method, pathname, { as, origin, from = 'same-origin', body } } const get = (pathname, as) => call('GET', pathname, { as }) +/** Uploads a file the way Payload's admin does: a form with the file and the document's other fields as JSON. */ +async function upload(name, bytes, type, as) { + const form = new FormData() + form.set('file', new Blob([bytes], { type }), name) + form.set('_payload', '{}') + const res = await fetch(`${site}/api/sources?depth=0`, { method: 'POST', redirect: 'manual', headers: { ...(as ? { Cookie: as } : {}), Origin: site }, body: form, signal: AbortSignal.timeout(120_000) }) + return { status: res.status, json: await res.json().catch(() => null) } +} + async function signIn(email) { const { status, res } = await call('POST', '/api/users/login', { body: { email, password } }) const cookie = res.headers.getSetCookie().find((c) => c.startsWith('payload-token=')) @@ -236,9 +254,61 @@ async function checks() { check('the curator publishes version 2, and the answer holds the fields asked for and not the snapshot\'s content', publishing.status === 200 && answered?.status === 'published' && Boolean(answered?.publishedAt) && !('snapshot' in answered) && publishing.text.length < 2000, { status: publishing.status, keys: answered && Object.keys(answered), length: publishing.text.length }) const publicSummary = await get(`/api/taxonomy-snapshots/${next?.doc}/summary`) const publicFrom = publicSummary.json?.derivedFrom?.proposal + await imports({ member, curator }) check('a visitor now reads version 2\'s summary, with the ids and no names', publicSummary.status === 200 && publicFrom?.author === String(me?.id) && !('authorName' in publicFrom) && !('acceptedByName' in publicFrom) && !publicSummary.text.includes('Morgan') && !publicSummary.text.includes('Cara') && !publicSummary.text.includes('@example.org'), publicFrom ?? { status: publicSummary.status }) } +/** Imports: source files and jobs are the curator's, a job waits until it is run, and the same file gives the same snapshot. */ +async function imports({ member, curator }) { + const run = (id, as) => call('POST', `/api/taxonomy-imports/${id}/run`, { as, origin: site }) + const snapshots = async () => ((await get('/api/taxonomy-snapshots?depth=0&limit=100', curator)).json?.docs ?? []).length + const jobs = (await get('/api/taxonomy-imports?depth=0', curator)).json?.docs ?? [] + const waiting = jobs.find((j) => j.status === 'queued') + const broken = jobs.find((j) => j.status === 'failed') + check('the curator sees the two seeded import jobs: one waiting, and one that failed and says why', jobs.length === 2 && Boolean(waiting) && waiting.attempts === 0 && Boolean(broken) && !broken.snapshot && /Missing required column/.test(broken.diagnostics?.[0]?.message ?? ''), jobs.map((j) => [j.title, j.status, j.message])) + const files = (await get('/api/sources?depth=0', curator)).json?.docs ?? [] + const first = files.find((f) => f.filename === 'checklist.csv') + // Payload gives a file's address with the site in front; `get` wants the path. + const address = first?.url ? first.url.replace(site, '') : '/api/sources/file/checklist.csv' + check('the curator reads the seeded source file back, byte for byte', address.startsWith('/api/sources/file/') && (await get(address, curator)).text === String(fixture('checklist.csv')), files.map((f) => [f.filename, f.url])) + for (const [who, as] of [['a visitor', undefined], ['the member', member]]) { + const seen = { files: (await get('/api/sources?depth=0', as)).status, file: (await get(address, as)).status, jobs: (await get('/api/taxonomy-imports?depth=0', as)).status, job: (await get(`/api/taxonomy-imports/${waiting?.id}`, as)).status, run: (await run(waiting?.id, as)).status, upload: (await upload('mine.csv', 'id,name\n', 'text/csv', as)).status } + check(`${who} is refused the source files, a file by its address, the import jobs, running one, and uploading`, Object.values(seen).every((status) => status === 403 || status === 404), seen) + } + check('nobody but the curator ran the waiting job or added a file', (await get(`/api/taxonomy-imports/${waiting?.id}?depth=0`, curator)).json?.status === 'queued' && ((await get('/api/sources?depth=0', curator)).json?.docs ?? []).length === files.length) + const page = await get(`/admin/collections/taxonomy-imports/${waiting?.id}`, curator) + check('the curator\'s import job page in the admin carries the plugin\'s panel', page.status === 200 && page.text.includes('onetaxonomy-panel'), { status: page.status }) + + // Running the waiting job gives the committed fixture, checksum and all, staged. + const release1 = JSON.parse(fixture('checklist.json')) + const before = await snapshots() + const ran = await run(waiting?.id, curator) + const made = (await get(`/api/taxonomy-snapshots/${ran.json?.snapshot}?depth=0`, curator)).json + check('the curator runs the waiting job: a staged snapshot with the checksum of fixtures/checklist.json', ran.status === 200 && ran.json?.status === 'done' && ran.json?.checksum === release1.checksum && made?.status === 'staged' && made?.snapshotId === 'fixture-checklist' && (await snapshots()) === before + 1, ran.json ?? { status: ran.status }) + check('the staged import is "not found" for a visitor', (await get(`/api/taxonomy-snapshots/${ran.json?.snapshot}`)).status === 404) + const again = await run(waiting?.id, curator) + check('running the finished job again stores nothing new', again.status === 200 && again.json?.status === 'done' && again.json?.snapshot === ran.json?.snapshot && (await snapshots()) === before + 1, again.json ?? { status: again.status }) + const settings = (await get(`/api/taxonomy-imports/${waiting?.id}?depth=0`, curator)).json?.settings + const same = await call('POST', '/api/taxonomy-imports?depth=0', { as: curator, origin: site, body: { title: 'The same file again', format: 'csv', file: first?.id, settings } }) + const found = await run(same.json?.doc?.id, curator) + check('a new job over the same file and settings waits, then finds the snapshot already stored', same.status === 201 && same.json?.doc?.status === 'queued' && found.json?.status === 'already-imported' && found.json?.snapshot === ran.json?.snapshot && (await snapshots()) === before + 1, found.json ?? same.json?.errors ?? { status: same.status }) + const failedAgain = await run(broken?.id, curator) + check('running the failed job again fails the same way and stores nothing', failedAgain.status === 200 && failedAgain.json?.status === 'failed' && /Missing required column/.test(failedAgain.json?.message ?? '') && (await snapshots()) === before + 1, failedAgain.json ?? { status: failedAgain.status }) + + // The whole way round over HTTP: upload release 2, make its job, run it. + const release2 = JSON.parse(fixture('checklist-v2.json')) + const sent = await upload('checklist-v2.csv', fixture('checklist-v2.csv'), 'text/csv', curator) + check('the curator uploads a file, and it lands in the site\'s throwaway folder', sent.status === 201 && Boolean(uploads) && existsSync(path.join(uploads ?? '', sent.json?.doc?.filename ?? 'none')), sent.json?.errors ?? { status: sent.status, uploads }) + const v2 = { ...settings, snapshot: { ...settings?.snapshot, version: '2' }, source: { ...settings?.source, externalVersion: '2' }, options: { ...settings?.options, classification: { ...settings?.options?.classification, version: '2' } } } + const job2 = await call('POST', '/api/taxonomy-imports?depth=0', { as: curator, origin: site, body: { title: 'Checklist, release 2', format: 'csv', file: sent.json?.doc?.id, settings: v2 } }) + const ran2 = await run(job2.json?.doc?.id, curator) + check('a job over the uploaded file gives the checksum of fixtures/checklist-v2.json', job2.status === 201 && ran2.json?.status === 'done' && ran2.json?.checksum === release2.checksum && (await snapshots()) === before + 2, ran2.json ?? job2.json?.errors ?? { status: job2.status }) + const html = await upload('page.html', '', 'text/html', curator) + check('an HTML file is refused as a source file, even from the curator', html.status >= 400 && html.status < 500, { status: html.status }) + const edited = await call('PATCH', `/api/taxonomy-imports/${job2.json?.doc?.id}?depth=0`, { as: curator, origin: site, body: { status: 'queued' } }) + check('a finished job\'s results can\'t be edited, even by the curator', edited.status >= 400 && edited.status < 500 && (await get(`/api/taxonomy-imports/${job2.json?.doc?.id}?depth=0`, curator)).json?.status === 'done', { status: edited.status }) +} + let up = false try { for (let i = 0; i < 180 && !up && server.exitCode === null; i += 1) { @@ -253,6 +323,7 @@ try { server.kill('SIGTERM') await finished(server) } +if (up) check('the folder of uploaded files is gone once the site has stopped', Boolean(uploads) && !existsSync(uploads), { uploads }) if (failed && up) console.log(`\nThe site's last output:\n${log.split('\n').slice(-25).join('\n')}`) console.log(`\n${passed} passed, ${failed} failed.`) process.exit(failed ? 1 : 0) diff --git a/apps/payload-example/src/seed.ts b/apps/payload-example/src/seed.ts index 3fc5603..71a02ae 100644 --- a/apps/payload-example/src/seed.ts +++ b/apps/payload-example/src/seed.ts @@ -1,17 +1,38 @@ +import { readFile } from 'node:fs/promises' +import path from 'node:path' + import { createView, indexDataset, sealSnapshot, type DatasetSnapshot } from '@captiveorigins/onetaxonomy-core' -import type { Payload, TypedUser } from 'payload' +import { runImport } from '@captiveorigins/payload-plugin-onetaxonomy' +import { createLocalReq, type Payload, type TypedUser } from 'payload' import amniotes from '../../../fixtures/amniotes.json' import structures from '../../../fixtures/structures.json' const SNAPSHOTS = 'taxonomy-snapshots' const VIEWS = 'taxonomy-views' +const IMPORTS = 'taxonomy-imports' +const SOURCES = 'sources' const copy = (v: T): T => JSON.parse(JSON.stringify(v)) as T +/** + * What the checklist job reads `fixtures/checklist.csv` with. These are the settings `fixtures/checklist.json` was + * made with, so running the job gives that snapshot, checksum and all. + */ +const CHECKLIST = { + snapshot: { id: 'fixture-checklist', version: '1', createdAt: '2026-11-02T00:00:00.000Z' }, + source: { id: 'fixture-checklist', title: 'OneTaxonomy illustrative checklist (fictional taxa)', citation: 'Illustrative teaching data with invented names. Not a scientific source.', license: 'CC0-1.0', externalVersion: '1', illustrative: true }, + options: { + idPrefix: 'fx', + classification: { id: 'fx:checklist', title: 'Illustrative checklist', version: '1' }, + columns: { taxonID: 'taxonID', name: 'scientificName', parentID: 'parentNameUsageID', acceptedID: 'acceptedNameUsageID', status: 'taxonomicStatus', rank: 'taxonRank', authorship: 'scientificNameAuthorship', vernacular: [{ column: 'vernacular_en', locale: 'en' }, { column: 'vernacular_de', locale: 'de' }] }, + }, +} + /** * Fills an empty site, once: two accounts (with the password in `EXAMPLE_PASSWORD`, which `pnpm dev` makes up), one - * published snapshot with two published views, and one staged snapshot that only a curator sees. The snapshots are - * the repository's illustrative fixtures, not real data. + * published snapshot with two published views, one staged snapshot that only a curator sees, and two import jobs + * over made-up files: one waiting to be run, and one that failed, with its reason. The snapshots and files are the + * repository's illustrative fixtures, not real data. * * Publishing is recorded against a person, so the seed publishes as the curator. On a database with no curator and * no `EXAMPLE_PASSWORD` it adds nothing. @@ -47,4 +68,21 @@ export async function seed(payload: Payload): Promise { // Staged: stored and checked, but not public until a curator publishes it. await payload.create({ collection: SNAPSHOTS as never, data: { snapshot: sealSnapshot(copy(structures) as unknown as DatasetSnapshot) } as never, user }) + + // Import jobs. The site starts in this app's folder, so the fixtures are two folders up. + const csv = await readFile(path.resolve(process.cwd(), '../../fixtures/checklist.csv')).catch(() => null) + if (!csv) { + payload.logger.info('OneTaxonomy example: fixtures/checklist.csv wasn\'t found from here, so no import jobs were added.') + return + } + const upload = async (name: string, data: Buffer) => ((await payload.create({ collection: SOURCES as never, data: {} as never, file: { data, mimetype: 'text/csv', name, size: data.length }, user })) as { id: string | number }).id + // Waiting: a curator runs it from its page in the admin. + await payload.create({ collection: IMPORTS as never, data: { title: 'Checklist, release 1', format: 'csv', file: await upload('checklist.csv', csv), settings: CHECKLIST } as never, user }) + // Failed: the file lacks the columns the settings name. Nothing is stored, and the job says why. + const broken = (await payload.create({ + collection: IMPORTS as never, + data: { title: 'A checklist with columns missing (fails on purpose)', format: 'csv', file: await upload('columns-missing.csv', Buffer.from('id,name\n1,Alphus primus\n')), settings: { ...CHECKLIST, snapshot: { ...CHECKLIST.snapshot, id: 'fixture-columns-missing' } } } as never, + user, + })) as { id: string | number } + await runImport(await createLocalReq({ user }, payload), String(broken.id)) } diff --git a/apps/payload-example/src/sources.ts b/apps/payload-example/src/sources.ts new file mode 100644 index 0000000..02683ec --- /dev/null +++ b/apps/payload-example/src/sources.ts @@ -0,0 +1,22 @@ +import type { CollectionConfig } from 'payload' + +import { isCurator } from './users' + +const curatorOnly = ({ req }: { req: { user?: unknown } }) => isCurator(req.user) + +/** + * The files import jobs read: checklists and tree files a curator uploads. Curators only, to read as well as to add, + * so nobody else can fetch a source file by its address. An import job reads its file as the person running it, so + * these rules decide that too. + * + * No list of allowed file types: tree files have no dependable one. Payload refuses HTML, scripts and programs on its + * own, and serves what it keeps with a content security policy. + */ +export const sources = (staticDir: string): CollectionConfig => ({ + slug: 'sources', + labels: { singular: 'Source file', plural: 'Source files' }, + admin: { group: 'Taxonomy', description: 'Files for import jobs to read. Upload one here, then name it in a new taxonomy import.' }, + access: { read: curatorOnly, create: curatorOnly, update: curatorOnly, delete: curatorOnly }, + upload: { staticDir }, + fields: [], +}) From b51287cec59067de91da8b101638865d367cf751 Mon Sep 17 00:00:00 2001 From: Michael Gilardi <11632652+mgilardi@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:52:58 -0700 Subject: [PATCH 2/2] Click through the admin in a browser, and fix what it found The example site now has import jobs, so the import panel could run in a real admin for the first time. scripts/click-through.mjs does a curator's work on all four panels in a headless Chrome already on the machine, with a throwaway profile, and fails on any request off the site apart from Payload's code editor. It found three things jsdom can't see: - The panels' paragraphs and lists sat flush against each other, because the panel's margin reset outranked its spacing rule. The reset now carries no weight, and the script measures the gaps. - The form under a panel repeated it. The fields a panel already shows are now hidden in the form, not disabled, so they stay in its data: the import panel puts a run's result there so a later Save doesn't send the old one. - The made-up-data tag touched the source's title. The docs said every button asks first. Running an import never did, and they now say so. Tested: 633 unit tests, 74 smoke checks and the brand check locally; the click-through passes 40 of 40 on the built site. Six deliberate breaks each failed their own check. Not in CI: the click-through, which needs Chrome and the network. Not tested: a screen reader, Safari, Firefox, a phone, Postgres or SQLite. Co-Authored-By: Claude --- CHANGELOG.md | 4 +- apps/payload-example/README.md | 35 +- apps/payload-example/package.json | 1 + .../payload-example/scripts/click-through.mjs | 475 ++++++++++++++++++ docs/architecture/adr/0012-payload-plugin.md | 6 +- docs/test-audit.md | 29 ++ packages/payload/README.md | 8 +- .../payload/src/client/ImportPanelView.tsx | 2 +- .../payload/src/client/SnapshotPanelView.tsx | 2 +- packages/payload/src/client/panels.css | 15 +- packages/payload/src/collections.ts | 15 +- packages/payload/src/imports.ts | 19 +- packages/payload/test/imports.test.ts | 16 + packages/payload/test/snapshots.test.ts | 9 + 14 files changed, 602 insertions(+), 34 deletions(-) create mode 100644 apps/payload-example/scripts/click-through.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 10fd36b..44be04a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,10 @@ ## Unreleased +- Example site: import jobs, and a script that clicks through the admin. `apps/payload-example` now turns the plugin's `imports` option on over a new `sources` upload collection that only a curator can read, add to, change or delete, so nobody else can fetch a source file by its address. The seed adds a job waiting to be run and one that failed on purpose, both over made-up files; a job doesn't run when it is made. Uploaded files go in a throwaway folder that is deleted when the site stops (`EXAMPLE_UPLOADS` names a folder to keep instead). The smoke test goes from 58 to 74 checks. New `pnpm click-through` builds the site, opens a Chrome already on the machine with no window and a throwaway profile, signs in as the made-up curator and does a curator's work on all four panels, 40 checks; it installs nothing, needs the network for Payload's code editor, fails on any other request off the site, and isn't in CI. +- Payload plugin: three things that script found in the real admin (changes the unreleased plugin). The panels' paragraphs and lists sat flush against each other, because the panel's margin reset outranked its spacing rule; jsdom doesn't lay out, so no test saw it. The reset now carries no weight and the import panel's status and message are spaced too. The form under a panel repeated it: a snapshot's label, id, version, checksum and counts, and an import job's message, checksums, diagnostics, warnings and progress (three of them in Payload's code editor) are now hidden in the form with `admin.hidden`, still in its data, the API and the list's columns. And the "Made-up example data" tag touched the source's title. The docs said every button asks first; running an import never did, and they now say so. - Payload plugin: a snapshot's content is read only when it is asked for by name (changes the unreleased plugin). A stored snapshot can be 15 MB, and Payload read all of it for every list, every relationship it filled in and every admin page. Now a read, create, update or delete that names no `select` answers with every field except `snapshot`; `select: { snapshot: true }`, REST `?select[snapshot]=true` or GraphQL's `select: true` gives the content, and the export endpoint is as before. A caller's own `select` is left exactly as asked. In the admin the content field is on the new-snapshot form only: a stored snapshot's page is sent without the content and without the JSON editor, and Payload's own Save there sends the status alone. Hosts' code that read `doc.snapshot` from a plain read, or from the document an after-read or after-change hook is given, has to select it. The lock on sealed content doesn't depend on this. Fixed on the way: the admin panels showed the old status after Payload's own Save; they now read from the site again whenever Payload reports the document saved. Tested: 10 new tests (8 on a real Payload with an in-memory MongoDB, one of them over HTTP, and 2 in jsdom), 3 new checks over HTTP in the example site (58 in all), and five deliberate breaks, each caught. Also run in headless Chrome on the example site, outside CI: a stored snapshot's page has no content and no editor, a sealed snapshot pasted into the new-snapshot form is stored with the same checksum, and Payload's Save sends one small request. Not explained: in that run, after a snapshot was made through the form and its page loaded again, Payload's read-only Counts group showed no fields under its heading; four shorter runs of the same steps drew them. Not tested: Postgres, SQLite, a browser with a window. See [ADR 0012](docs/architecture/adr/0012-payload-plugin.md), point 17. -- Payload plugin, part 5 of M7: admin panels (still not on npm). Each of the plugin's four collections gets a panel at the top of its edit page, on unless `admin: { panels: false }`. The snapshot panel says the status in words, what the snapshot holds (counts, a line per classification and phylogeny, sources with their licenses) and, for one made by accepting a proposal, which proposal and the two sealed person ids; a manager gets Publish and Withdraw. The view panel draws the saved view (the first 200 nodes) with the same tree as an outline for the keyboard and the lines readers are told, and holds Publish, with the reason, until the form is saved and the snapshot is published. The import panel shows the steps of a run, its warnings and diagnostics, looks again every 3 seconds while a job runs, and offers to run a queued, failed or interrupted job. The proposal panel says what a proposal would change, and a manager accepts it (naming the new version) or declines it. Every button asks first in the page and says what can't be undone; accepting lists what gets sealed for good and asks the manager to check it for personal details. The panels have no powers of their own: they call the host's site from the browser as the signed-in person, so the host's `access` callbacks decide, and a refusal reads "Nothing was changed." with the site's reason. New: the `@captiveorigins/payload-plugin-onetaxonomy/client` entry (found through Payload's import map, so run `payload generate:importmap` after adding the plugin), the `admin.panels` option, and one endpoint, `GET /api//:id/summary` (and `summarizeSnapshot`), which answers counts and capped lists, never names or trees, with the same 404 as the other endpoints. For a snapshot made from a proposal it gives the name the host shows for each sealed id to signed-in requesters only, looked up per request and never stored. `@payloadcms/ui` and `react` are new optional peer dependencies, and the plugin now depends on `@captiveorigins/onetaxonomy-react` for the drawing. Colours come from Payload's theme variables; a test reads the installed `@payloadcms/ui` stylesheets and checks text at 4.5:1 and control edges at 3:1 in light and dark, and a status is never colour alone. The example site shows the panels to its curator and turns Payload's Gravatar avatar off. Tested: 94 new tests (74 render the panels in jsdom with Payload's hooks and `fetch` replaced by stand-ins, 11 on the stylesheet, 9 on the summary against a real Payload with an in-memory MongoDB), and 8 new checks over HTTP in the example site (55 in all). The snapshot, view and proposal panels were also clicked through once in headless Chrome on the example site, in light and dark; that run is not in CI. Not tested: the import panel in a real admin, a screen reader, Safari, Firefox, a phone, or a custom Payload theme. Found on the way and written in the README: Payload's own admin loads its JSON editor from cdn.jsdelivr.net and, unless `admin.avatar` is `'default'`, asks gravatar.com with a hash of the signed-in person's email address. See [ADR 0012](docs/architecture/adr/0012-payload-plugin.md), point 16. +- Payload plugin, part 5 of M7: admin panels (still not on npm). Each of the plugin's four collections gets a panel at the top of its edit page, on unless `admin: { panels: false }`. The snapshot panel says the status in words, what the snapshot holds (counts, a line per classification and phylogeny, sources with their licenses) and, for one made by accepting a proposal, which proposal and the two sealed person ids; a manager gets Publish and Withdraw. The view panel draws the saved view (the first 200 nodes) with the same tree as an outline for the keyboard and the lines readers are told, and holds Publish, with the reason, until the form is saved and the snapshot is published. The import panel shows the steps of a run, its warnings and diagnostics, looks again every 3 seconds while a job runs, and offers to run a queued, failed or interrupted job. The proposal panel says what a proposal would change, and a manager accepts it (naming the new version) or declines it. Publish, Withdraw, Accept and Decline ask first in the page and say what can't be undone (running an import doesn't ask: it stores at most a staged snapshot); accepting lists what gets sealed for good and asks the manager to check it for personal details. The panels have no powers of their own: they call the host's site from the browser as the signed-in person, so the host's `access` callbacks decide, and a refusal reads "Nothing was changed." with the site's reason. New: the `@captiveorigins/payload-plugin-onetaxonomy/client` entry (found through Payload's import map, so run `payload generate:importmap` after adding the plugin), the `admin.panels` option, and one endpoint, `GET /api//:id/summary` (and `summarizeSnapshot`), which answers counts and capped lists, never names or trees, with the same 404 as the other endpoints. For a snapshot made from a proposal it gives the name the host shows for each sealed id to signed-in requesters only, looked up per request and never stored. `@payloadcms/ui` and `react` are new optional peer dependencies, and the plugin now depends on `@captiveorigins/onetaxonomy-react` for the drawing. Colours come from Payload's theme variables; a test reads the installed `@payloadcms/ui` stylesheets and checks text at 4.5:1 and control edges at 3:1 in light and dark, and a status is never colour alone. The example site shows the panels to its curator and turns Payload's Gravatar avatar off. Tested: 94 new tests (74 render the panels in jsdom with Payload's hooks and `fetch` replaced by stand-ins, 11 on the stylesheet, 9 on the summary against a real Payload with an in-memory MongoDB), and 8 new checks over HTTP in the example site (55 in all). The snapshot, view and proposal panels were also clicked through once in headless Chrome on the example site, in light and dark; that run is not in CI. Not tested: the import panel in a real admin, a screen reader, Safari, Firefox, a phone, or a custom Payload theme. Found on the way and written in the README: Payload's own admin loads its JSON editor from cdn.jsdelivr.net and, unless `admin.avatar` is `'default'`, asks gravatar.com with a hash of the signed-in person's email address. See [ADR 0012](docs/architecture/adr/0012-payload-plugin.md), point 16. - Benchmarks: frame times in a browser. `apps/playground/bench.html` (built with the playground) pans, zooms and scrolls made-up trees of 125 to 10,000 tips as SVG and on the canvas and times every frame; open it in any browser, press Run and save the numbers. `node scripts/bench-browser.mjs` runs it in the Chrome already installed, headless, at pixel density 1 and 2 and with the main thread slowed 4 times, and writes `docs/benchmarks-browser.md`; `--from ` adds a section from numbers saved in another browser. It uses a throwaway profile that it deletes, listens on this machine only and sends nothing anywhere. First numbers (headless Chrome 157, M2 Max, other work running): the canvas held 60 frames a second at every size up to 19,999 nodes, slowed thread included; SVG held it through 1,999 nodes, and through 499 on the slowed thread. A second section is from a headless build of WebKit 27.2 (Playwright's: the engine Safari and the Mac app use, but neither program), driven by hand as the doc describes, at normal speed only: the canvas held 60 frames a second at every size, and SVG held it through 1,999 nodes. When a browser's clock ticks in whole milliseconds, as WebKit's does, the script says so in the section and prints no decimal places. The 1,000-node switch is unchanged. Not measured: Safari itself, the Mac app's web view, Firefox, a window on a real screen, a real slow device, radial layouts. The benchmark page and script have no tests of their own. - Payload plugin docs: moving a departed person's proposals to another account. The README's "When someone leaves" now says that `author` and `decidedBy` can't be changed through Payload, that the lock is a hook and so doesn't cover a host's own database code, and that a host making that move there must change only those two fields and leave `proposal`, `checksum` and the snapshot alone. A new test makes the move on MongoDB and checks that the proposal still matches its checksum, the version made from it still verifies, and the proposal stays locked. No code changed; an option to allow the move through Payload was considered and not built (ADR 0012). - Payload plugin: people are recorded by id, never by name (changes the unreleased proposals, before anyone used them). `proposals.nameOf` is gone. `proposals.personId({ req })` gives an opaque id for the signed-in person (default: their user document's id), and that id is all a proposal stores about its author and all an accepted proposal seals into the new snapshot for the author and the manager. An id is 1 to 64 letters, digits, `.`, `_`, `:` and `-`, starting with a letter or digit, so an email address or a name with a space in it is refused: the action stops, nothing is saved, and the refused value isn't written to the log. A one-word handle would pass; the plugin can't tell it from an id. `proposals.displayName({ req, personId })` is optional and gives the name to show: it is looked up each time a proposal is read (once per person per request), returned as `authorName` on the proposal and in the preview, and never stored or sealed, so a name changed or removed at the site changes what is shown at once and leaves every snapshot's checksum alone. Why: a name inside a sealed snapshot can't be taken out when a person leaves. Core is unchanged apart from its comments and one error text: `acceptIntoSnapshot` still seals the `author` and `acceptedBy` strings it is given. Still sealed as written, and not screened: a proposal's title, each move's note and the manager's note. No endpoint turns an id from a snapshot's record back into a name. Tested on Payload 3.93 with an in-memory MongoDB (142 plugin tests, 526 in all) and through the example site (47 checks); eight deliberate breaks of the new code each failed a test. Not tested: Postgres or SQLite, and a host with more than one auth collection. diff --git a/apps/payload-example/README.md b/apps/payload-example/README.md index ce2bcfa..cb7ff51 100644 --- a/apps/payload-example/README.md +++ b/apps/payload-example/README.md @@ -1,8 +1,8 @@ # OneTaxonomy Payload example -A small Payload and Next.js site that uses [`@captiveorigins/payload-plugin-onetaxonomy`](../../packages/payload). It shows the plugin set up the way a real site would: accounts with two roles, public pages that draw published views, and proposals that record a person by an id, with a public handle shown beside it and no email address. +A small Payload and Next.js site that uses [`@captiveorigins/payload-plugin-onetaxonomy`](../../packages/payload). It shows the plugin set up the way a real site would: accounts with two roles, public pages that draw published views, import jobs over uploaded files, and proposals that record a person by an id, with a public handle shown beside it and no email address. -> **Status: an example, not a product.** It runs on this machine only, on a throwaway in-memory MongoDB, with made-up accounts and the repository's illustrative fixtures. Tested: 58 checks over HTTP with sign-in cookies (`pnpm smoke`), on `next dev` and on the built site, with Payload 3.93 and Next.js 16; the three public pages were looked at in headless Chrome, in light and dark, at desktop width and at 520 px; and the admin's panels were clicked through in headless Chrome as the curator, in light and dark (publishing a snapshot with the keyboard, a view, accepting a proposal, and storing a snapshot pasted into the new-snapshot form). Those runs are not in CI. Not tested: a screen reader, Safari, Firefox, a phone, Postgres or SQLite, or a deployed site. +> **Status: an example, not a product.** It runs on this machine only, on a throwaway in-memory MongoDB, with made-up accounts and the repository's illustrative fixtures. Tested: 74 checks over HTTP with sign-in cookies (`pnpm smoke`, in CI), on `next dev` and on the built site, with Payload 3.93 and Next.js 16; the admin's panels clicked through in headless Chrome as the curator, in light and dark (`pnpm click-through`, 40 checks, not in CI: it needs Chrome and the network); and the three public pages looked at in headless Chrome by hand, in light and dark, at desktop width and at 520 px. Not tested: a screen reader, Safari, Firefox, a phone, Postgres or SQLite, or a deployed site. ## What it shows @@ -10,10 +10,12 @@ A small Payload and Next.js site that uses [`@captiveorigins/payload-plugin-onet - **Two roles.** A curator manages snapshots and views, uses the admin, and decides on proposals. A member reads what is published and makes proposals. Only a curator hands out roles: a member can't raise their own, and sees no account but their own. - **Proposals by id, with a handle shown.** A proposal stores its author's account id, and when a curator accepts one, the new snapshot version records the proposer and the curator by those ids, inside its checksum. No name or email address is sealed. `proposals.displayName` looks up each person's `handle` when a proposal is shown, so a changed handle shows at once and a removed one leaves the id alone. - **Staged, then published.** The seed stores one published snapshot with two views, and one staged snapshot that only the curator sees. An accepted proposal makes a new staged version; visitors see nothing until the curator publishes it. -- **Panels in the admin.** Signed in as the curator, a snapshot's, a view's and a proposal's edit page each open with the plugin's panel: what the record holds, its status in words, and the buttons to publish, withdraw, accept or decline, each asking first. The names beside the ids on those panels are the handles, looked up when the page loads. +- **Import jobs.** A curator uploads a source file, names it in a new import, and runs it from the job's page. A job that passes every step stores one staged snapshot; one that fails stores nothing and says why. The seed adds two jobs over made-up files: one waiting to be run, and one that failed on purpose, so both panels are there to look at. A job doesn't run when it is made (`runOnCreate: false`): the curator presses the button. +- **Source files for curators only.** The [`sources`](src/sources.ts) upload collection lets only a curator read, add, change or delete a file, so nobody else can fetch one by its address. An import reads its file as the person running it, so the same rule decides that. Payload refuses HTML, scripts and programs as uploads on its own. +- **Panels in the admin.** Signed in as the curator, a snapshot's, a view's, an import's and a proposal's edit page each open with the plugin's panel: what the record holds, its status in words, and the buttons to publish, withdraw, run, accept or decline. Each asks first, except running an import, which stores nothing a reader can see. The names beside the ids on those panels are the handles, looked up when the page loads. - **Cookies from this site only.** Payload's `csrf` list holds the site's own address, so a sign-in cookie sent from another site's page counts for nothing. -The setup is in [`payload.config.ts`](payload.config.ts) (the plugin's options), [`src/users.ts`](src/users.ts) (accounts and roles) and [`src/seed.ts`](src/seed.ts) (the example data). +The setup is in [`payload.config.ts`](payload.config.ts) (the plugin's options), [`src/users.ts`](src/users.ts) (accounts and roles), [`src/sources.ts`](src/sources.ts) (source files) and [`src/seed.ts`](src/seed.ts) (the example data). ## Run it @@ -33,12 +35,14 @@ Admin: http://localhost:3000/admin Curator: curator@example.org Member: member@example.org Password for both, made up for this run: … +Uploads: … (deleted when this stops) ``` -Stop it with Ctrl-C. The database goes with it: every run starts from the seed. +Stop it with Ctrl-C. The database and the uploaded files go with it: every run starts from the seed. - `PORT=3100` picks another port. - `EXAMPLE_PASSWORD=…` sets the password instead of making one up. +- `EXAMPLE_UPLOADS=…` keeps uploaded files in a folder of yours instead of a throwaway one. That folder is left alone when the site stops. - The site listens on `localhost` only. It is not set up to face a network. ## Check it @@ -59,12 +63,29 @@ This builds the site, starts it on a free port with its own database, and sends - a changed handle shows on the accepted proposal at once, and the sealed version doesn't change; - an accepted proposal can't be deleted; - the curator's edit pages in the admin carry the plugin's panels; +- a visitor and a member are refused the source files, a file by its address, the import jobs, running one, and uploading; +- the curator runs the waiting job into a staged snapshot with the checksum of `fixtures/checklist.json`, a second job over the same file finds that snapshot, and the failed job fails the same way and stores nothing; +- a job over a newly uploaded file works, an HTML file is refused as a source file, and a finished job's results can't be edited; +- the folder of uploaded files is gone once the site has stopped; - a stored snapshot's admin page is sent without its content and without the content field, and the new-snapshot form still has that field; - the new version's summary shows the curator both people by id with their handles, is "not found" for a visitor and the member while it is staged, and once published gives a visitor the ids and no names; - a member can't publish it, and the curator's publish answers without the snapshot's content. `--dev` runs `next dev` instead of a build, and `--skip-build` reuses the last build. CI runs the smoke test on every pull request. +### In a browser + +```bash +pnpm --filter @captiveorigins/onetaxonomy-payload-example click-through +``` + +This builds the site, starts it the same way, opens a Chrome or Chromium already on the machine with no window and a throwaway profile, signs in as the curator and does what a curator would: publishes a snapshot with the keyboard, changes and saves a draft view, runs the three import jobs, renames a finished one with Payload's own Save, accepts a proposal into a new version, pastes a sealed snapshot into the new-snapshot form, and publishes it with Payload's own status field. It takes about two minutes. It checks what the panels say at each step, what was sent, that the form under a panel doesn't repeat it, that the panel's parts are spaced apart as the browser lays them out, and that no page asks any site but this one, apart from Payload's code editor. + +- It needs Chrome or Chromium. `CHROME=/path/to/chrome` names one; without it the usual places are tried, and the script stops with exit code 2 if there is none. It installs nothing. +- It needs the network, because Payload's code editor loads from cdn.jsdelivr.net (see below). +- `--shots ` also saves a screenshot of each step. `--dev` and `--skip-build` work as above. +- It isn't in CI, for those two reasons. Run it after changing a panel or a collection's fields. + ## Using your own database Set `DATABASE_URI` to a MongoDB address and the site uses it instead of the in-memory one. Use an empty, throwaway database: this is still an example. @@ -76,8 +97,8 @@ Set `DATABASE_URI` to a MongoDB address and the site uses it instead of the in-m ## What it leaves out - **No page for making a proposal.** A member makes one through the API. The curator decides on it in the admin, on the proposal's panel. -- **No imports.** The `imports` option is off, so there are no import jobs and the import panel doesn't appear. It is tested in the plugin's own package. -- **Two requests Payload's admin makes to other sites.** The admin's JSON fields load their code editor (Monaco) from cdn.jsdelivr.net, on the new-snapshot form and on a view's and a proposal's page. That is Payload's, and this example leaves it. Payload's default avatar would also ask gravatar.com with a hash of the signed-in address; this example turns that off (`admin.avatar: 'default'`). +- **No imports from a URL.** Import jobs read uploaded files only. The plugin's `imports.fetch` option, which reads a URL the site allows, is tested in the plugin's own package. +- **Two requests Payload's admin makes to other sites.** The admin's JSON fields load their code editor (Monaco) from cdn.jsdelivr.net, on the new-snapshot form and on a view's, an import's and a proposal's page. That is Payload's, and this example leaves it. Payload's default avatar would also ask gravatar.com with a hash of the signed-in address; this example turns that off (`admin.avatar: 'default'`). - **No tenants.** The plugin's tenant scope is tested in its own package, not shown here. - **No sign-up.** A curator adds accounts in the admin. - **Nothing for production.** No email, no rate limits, no backups, no deployment setup. diff --git a/apps/payload-example/package.json b/apps/payload-example/package.json index b4e2443..afbadb0 100644 --- a/apps/payload-example/package.json +++ b/apps/payload-example/package.json @@ -8,6 +8,7 @@ "site:build": "node scripts/run.mjs build", "start": "node scripts/run.mjs start", "smoke": "node scripts/smoke.mjs", + "click-through": "node scripts/click-through.mjs", "typecheck": "tsc --noEmit" }, "dependencies": { diff --git a/apps/payload-example/scripts/click-through.mjs b/apps/payload-example/scripts/click-through.mjs new file mode 100644 index 0000000..93eaeb7 --- /dev/null +++ b/apps/payload-example/scripts/click-through.mjs @@ -0,0 +1,475 @@ +#!/usr/bin/env node +// Clicks through the plugin's admin panels in a real browser. It starts the example site on a free port with a +// throwaway database, opens a Chrome or Chromium that is already installed (headless, with a throwaway profile that +// is deleted afterwards and a debugging port on 127.0.0.1), signs in as the made-up curator and does what a curator +// does: publishes a snapshot by keyboard, runs the import jobs, accepts a proposal, pastes a new snapshot into +// Payload's own form and publishes it with Payload's own Save. Exits 0 when every check passes. +// +// node scripts/click-through.mjs build, then run the built site +// node scripts/click-through.mjs --skip-build run the site built last time +// node scripts/click-through.mjs --dev run `next dev` instead of a build +// node scripts/click-through.mjs --shots dir also save a screenshot of each step into `dir` +// CHROME=/path/to/chrome node scripts/click-through.mjs +// +// Nothing is installed and nothing is sent anywhere by this script. The pages do ask one other site: Payload's code +// editor loads from cdn.jsdelivr.net, so the steps that paste a snapshot need the network. Any other request off +// the site fails the run. +import { spawn } from 'node:child_process' +import { randomBytes } from 'node:crypto' +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { createServer } from 'node:net' +import os from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const args = process.argv.slice(2) +const flags = new Set() +let shots = null +for (let i = 0; i < args.length; i += 1) { + if (args[i] === '--shots' && args[i + 1]) shots = path.resolve(args[(i += 1)]) + else flags.add(args[i]) +} +const unknown = [...flags].filter((f) => !['--dev', '--skip-build'].includes(f)) +if (unknown.length) { + console.error(`Unknown option: ${unknown.join(' ')}\nUsage: node scripts/click-through.mjs [--dev | --skip-build] [--shots ]`) + process.exit(2) +} +const chrome = [process.env.CHROME, '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', '/Applications/Chromium.app/Contents/MacOS/Chromium', '/usr/bin/google-chrome', '/usr/bin/chromium', '/usr/bin/chromium-browser'].find((p) => p && existsSync(p)) +if (!chrome) { + console.error('No Chrome or Chromium found. Set CHROME to its path.') + process.exit(2) +} +if (shots) mkdirSync(shots, { recursive: true }) + +const app = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const runner = path.join(app, 'scripts', 'run.mjs') +const fixture = (name) => readFileSync(path.join(app, '../../fixtures', name), 'utf8') +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) +const finished = (child) => new Promise((resolve) => (child.exitCode !== null || child.signalCode !== null ? resolve(child.exitCode) : child.once('exit', (code) => resolve(code)))) +const freePort = () => + new Promise((resolve, reject) => { + const server = createServer() + server.once('error', reject) + server.listen(0, '127.0.0.1', () => { + const { port } = server.address() + server.close(() => resolve(port)) + }) + }) + +const port = await freePort() +const site = `http://localhost:${port}` +const password = randomBytes(12).toString('base64url') +// DATABASE_URI and EXAMPLE_UPLOADS are cleared so this never runs against a database or a folder you named for +// something else. +const env = { ...process.env, PORT: String(port), EXAMPLE_URL: site, EXAMPLE_PASSWORD: password, DATABASE_URI: '', EXAMPLE_UPLOADS: '' } +const run = (how, quiet) => spawn(process.execPath, [runner, how], { cwd: app, env, stdio: quiet ? ['ignore', 'pipe', 'pipe'] : 'inherit' }) + +if (!flags.has('--dev') && !flags.has('--skip-build')) { + if ((await finished(run('build', false))) !== 0) { + console.error('The build failed, so nothing was checked.') + process.exit(1) + } +} + +const server = run(flags.has('--dev') ? 'dev' : 'start', true) +let log = '' +for (const stream of [server.stdout, server.stderr]) stream.on('data', (chunk) => (log = (log + chunk).slice(-8000))) + +let failed = 0 +let passed = 0 +function check(name, ok, detail) { + if (ok) passed += 1 + else failed += 1 + console.log(`${ok ? 'ok ' : 'FAIL'} ${name}${ok || detail === undefined ? '' : `\n ${typeof detail === 'string' ? detail : JSON.stringify(detail)}`}`) +} + +/** One request from this script, not from the browser: to set a step up, or to read back what a click stored. */ +async function api(method, pathname, { as, body } = {}) { + const res = await fetch(site + pathname, { method, headers: { ...(as ? { Cookie: as } : {}), Origin: site, ...(body ? { 'Content-Type': 'application/json' } : {}) }, ...(body ? { body: JSON.stringify(body) } : {}), signal: AbortSignal.timeout(120_000) }) + return { status: res.status, json: await res.json().catch(() => null), res } +} +const signIn = async (email) => (await api('POST', '/api/users/login', { body: { email, password } })).res.headers.getSetCookie().find((c) => c.startsWith('payload-token='))?.split(';')[0] + +const profile = mkdtempSync(path.join(os.tmpdir(), 'onetaxonomy-click-through-')) +let browser = null +try { + let up = false + for (let i = 0; i < 180 && !up && server.exitCode === null; i += 1) { + up = await fetch(site + '/').then((r) => r.status === 200, () => false) + if (!up) await sleep(1000) + } + if (!up) throw new Error(`the site didn't start:\n${log}`) + + // Made-up things to look at, beside what the site seeds: a shared proposal from the member, a draft view, and a + // second import job over the file the first one reads. + const member = await signIn('member@example.org') + const curator = await signIn('curator@example.org') + const snaps = (await api('GET', '/api/taxonomy-snapshots?depth=0&select[status]=true&select[snapshotId]=true', { as: curator })).json.docs + const published = snaps.find((d) => d.status === 'published') + const staged = snaps.find((d) => d.status === 'staged') + const views = (await api('GET', '/api/taxonomy-views?depth=0&select[title]=true', { as: curator })).json.docs + const made = await api('POST', '/api/taxonomy-proposals?depth=0', { as: member, body: { snapshot: published.id, proposal: { title: 'Birds are reptiles', notes: 'A made-up note.', base: { kind: 'classification', id: 'traditional', version: '1' }, moves: [{ node: 'c:aves', from: null, to: 'c:reptilia', note: 'A made-up move note.' }] } } }) + await api('PATCH', `/api/taxonomy-proposals/${made.json.doc.id}?depth=0`, { as: member, body: { status: 'shared' } }) + const firstView = (await api('GET', `/api/taxonomy-views/${views[0].id}?depth=0`, { as: curator })).json + const draft = await api('POST', '/api/taxonomy-views?depth=0', { as: curator, body: { title: 'A draft view', snapshot: published.id, view: { ...firstView.view, id: 'draft-view', title: 'A draft view' } } }) + const jobs = (await api('GET', '/api/taxonomy-imports?depth=0', { as: curator })).json.docs + const waiting = jobs.find((j) => j.status === 'queued') + const broken = jobs.find((j) => j.status === 'failed') + const same = await api('POST', '/api/taxonomy-imports?depth=0', { as: curator, body: { title: 'The same file again', format: 'csv', file: waiting?.file, settings: waiting?.settings } }) + check('made a shared proposal, a draft view and a second import job to look at', made.status === 201 && draft.status === 201 && same.status === 201 && Boolean(broken), { proposal: made.status, draft: draft.status, job: same.status, why: draft.json?.errors ?? same.json?.errors }) + + browser = spawn(chrome, ['--headless=new', '--remote-debugging-port=0', '--remote-debugging-address=127.0.0.1', `--user-data-dir=${profile}`, '--window-size=1300,1500', '--no-first-run', '--no-default-browser-check', '--disable-extensions', '--disable-background-networking', '--disable-component-update', '--disable-sync', '--metrics-recording-only', 'about:blank'], { stdio: ['ignore', 'ignore', 'pipe'] }) + const endpoint = await new Promise((resolve, reject) => { + let text = '' + const timer = setTimeout(() => reject(new Error('Chrome did not start')), 30_000) + browser.stderr.on('data', (chunk) => { + text += chunk + const m = text.match(/DevTools listening on (ws:\/\/127\.0\.0\.1:\d+\/\S+)/) + if (m) { + clearTimeout(timer) + resolve(m[1]) + } + }) + }) + const ws = new WebSocket(endpoint) + await new Promise((resolve, reject) => { + ws.addEventListener('open', resolve, { once: true }) + ws.addEventListener('error', () => reject(new Error('no connection to Chrome')), { once: true }) + }) + + // What the pages did, as Chrome reports it. + let next = 0 + const answers = new Map() + const problems = [] // page errors, console errors and requests off the site + const requests = [] // the pages' own calls to the site's API + const editor = [] // Payload's code editor, fetched from jsDelivr + const editorCancelled = [] + const readsByPost = new Set() // Payload reads some lists with a POST marked as a GET + const writes = [] // writes to the snapshots collection, to look at what was sent + ws.addEventListener('message', (event) => { + const msg = JSON.parse(String(event.data)) + const sent = msg.method === 'Network.requestWillBeSent' ? msg.params.request : null + if (msg.method === 'Runtime.exceptionThrown') { + const what = msg.params.exceptionDetails.exception?.description ?? msg.params.exceptionDetails.text + // The code editor says this when its page is left while it loads. + if (/Canceled: Canceled/.test(what)) editorCancelled.push(what) + else problems.push(`exception: ${what}`) + } + if (msg.method === 'Runtime.consoleAPICalled' && msg.params.type === 'error') problems.push(`console.error: ${msg.params.args.map((a) => a.value ?? a.description ?? '').join(' ').slice(0, 400)}`) + if (sent) { + const here = sent.url.replace(site, '') + const marked = Boolean(sent.headers['X-Payload-HTTP-Method-Override']) + if (sent.url.startsWith(site)) { + if (here.includes('/api/') && !here.includes('/api/users') && !here.includes('payload-preferences')) requests.push(`${sent.method} ${here}`) + if (sent.method === 'POST' && marked) readsByPost.add(`${sent.method} ${here}`) + if (['PATCH', 'POST'].includes(sent.method) && /\/api\/taxonomy-snapshots(\/\w+)?(\?|$)/.test(sent.url) && !marked) writes.push({ id: msg.params.requestId, method: sent.method, url: here }) + } else if (sent.url.startsWith('https://cdn.jsdelivr.net/npm/monaco-editor@')) editor.push(sent.url) + else if (!/^(data|about|blob):/.test(sent.url)) problems.push(`request off the site: ${sent.url}`) + } + // A question from the browser itself, such as "Leave site?" over a form with unsaved changes, holds every later + // step until it is answered. Nothing here should raise one, so it counts as a problem, and is answered to go on. + if (msg.method === 'Page.javascriptDialogOpening') { + problems.push(`the browser asked: ${msg.params.type} ${JSON.stringify(msg.params.message.slice(0, 120))}`) + ws.send(JSON.stringify({ id: (next += 1), method: 'Page.handleJavaScriptDialog', params: { accept: true }, sessionId: msg.sessionId })) + } + const waiter = answers.get(msg.id) + if (!waiter) return + clearTimeout(waiter.timer) + answers.delete(msg.id) + if (msg.error) waiter.reject(new Error(msg.error.message)) + else waiter.resolve(msg.result) + }) + const raw = (method, params = {}, sessionId) => + new Promise((resolve, reject) => { + const id = (next += 1) + // A browser that stops answering ends the run with a failure, not a hang. + const timer = setTimeout(() => answers.delete(id) && reject(new Error(`the browser didn't answer ${method} within 2 minutes`)), 120_000) + answers.set(id, { resolve, reject, timer }) + ws.send(JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) })) + }) + const { targetId } = await raw('Target.createTarget', { url: 'about:blank' }) + const { sessionId } = await raw('Target.attachToTarget', { targetId, flatten: true }) + const send = (method, params) => raw(method, params, sessionId) + await send('Page.enable') + await send('Runtime.enable') + await send('Network.enable') + + // What a person does: look, wait, go somewhere, click, press a key. + const theme = (value) => send('Emulation.setEmulatedMedia', { features: [{ name: 'prefers-color-scheme', value }] }) + const js = async (expression) => { + const answer = await send('Runtime.evaluate', { expression, awaitPromise: true, returnByValue: true }) + if (answer.exceptionDetails) throw new Error(answer.exceptionDetails.exception?.description ?? answer.exceptionDetails.text) + return answer.result.value + } + const panelText = () => js(`document.querySelector('.onetaxonomy-panel')?.innerText ?? ''`) + const until = async (expression, what, ms = 60_000) => { + const end = Date.now() + ms + while (Date.now() < end) { + if (await js(`!!(${expression})`).catch(() => false)) return true + await sleep(250) + } + check(`waited for ${what}`, false, await js(`document.querySelector('.onetaxonomy-panel')?.innerText?.slice(0, 600) ?? document.body.innerText.slice(0, 600)`).catch(String)) + return false + } + const panelSays = (words, what, ms) => until(`document.querySelector('.onetaxonomy-panel')?.innerText.includes(${JSON.stringify(words)})`, what, ms) + const go = async (pathname) => { + await send('Page.navigate', { url: site + pathname }) + await sleep(500) + await until(`document.readyState === 'complete'`, `${pathname} to load`) + } + const flushAt = [] + /** Looks at the page as it stands: notes any panel parts sitting flush against each other, and saves a screenshot. */ + const look = async (name) => { + const parts = await flush() + if (parts.length) flushAt.push({ at: name, parts }) + if (!shots) return + const { data } = await send('Page.captureScreenshot', { format: 'png' }) + writeFileSync(path.join(shots, `${name}.png`), Buffer.from(data, 'base64')) + } + const button = (name, within) => `[...document.querySelectorAll(${JSON.stringify(`${within} button`)})].find((b) => b.textContent.trim() === ${JSON.stringify(name)})` + const clickButton = (name, within = '.onetaxonomy-panel') => js(`(() => { const b = ${button(name, within)}; if (!b) return false; b.click(); return true })()`) + const focusButton = (name, within = '.onetaxonomy-panel') => js(`(() => { const b = ${button(name, within)}; if (!b) return false; b.focus(); return true })()`) + /** A real mouse press at the middle of an element, for Payload's own controls, which listen for one. */ + const clickAt = async (selector) => { + const at = await js(`(() => { const e = document.querySelector(${JSON.stringify(selector)}); if (!e) return null; e.scrollIntoView({ block: 'center' }); const r = e.getBoundingClientRect(); return { x: r.x + r.width / 2, y: r.y + r.height / 2 } })()`) + if (!at) return false + for (const type of ['mousePressed', 'mouseReleased']) await send('Input.dispatchMouseEvent', { type, x: at.x, y: at.y, button: 'left', clickCount: 1 }) + return true + } + const key = async (name) => { + const code = { Enter: 13, Escape: 27 }[name] + const base = { key: name, code: name, windowsVirtualKeyCode: code, nativeVirtualKeyCode: code } + await send('Input.dispatchKeyEvent', { type: 'rawKeyDown', ...base }) + if (name === 'Enter') await send('Input.dispatchKeyEvent', { type: 'char', text: '\r', ...base }) + await send('Input.dispatchKeyEvent', { type: 'keyUp', ...base }) + } + const type = (selector, value) => js(`(() => { const i = document.querySelector(${JSON.stringify(selector)}); Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value').set.call(i, ${JSON.stringify(value)}); i.dispatchEvent(new Event('input', { bubbles: true })) })()`) + const focused = () => js(`(() => { const e = document.activeElement; return e ? (e.getAttribute('role') ?? e.tagName.toLowerCase()) + ':' + (e.getAttribute('aria-labelledby') ? document.getElementById(e.getAttribute('aria-labelledby'))?.textContent.slice(0, 40) : e.textContent.trim().slice(0, 40)) : '' })()`) + const saveIsOff = () => js(`(() => { const b = document.querySelector('#action-save'); return b ? b.disabled || b.getAttribute('aria-disabled') === 'true' : null })()`) + /** The panel's parts, and the gap above each one after the first, as the browser lays them out. */ + const flush = () => js(`[...document.querySelectorAll('.onetaxonomy-panel > *, .onetaxonomy-panel__block > *, .onetaxonomy-panel__live > *')].filter((e) => e.previousElementSibling && parseFloat(getComputedStyle(e).marginTop) + parseFloat(getComputedStyle(e).paddingTop) + parseFloat(getComputedStyle(e.previousElementSibling).marginBottom) < 3).map((e) => e.tagName.toLowerCase() + '.' + e.className + ': ' + e.textContent.trim().slice(0, 30))`) + const linkTo = (label) => js(`[...document.querySelectorAll('.onetaxonomy-panel a')].find((a) => a.textContent.trim() === ${JSON.stringify(label)})?.getAttribute('href') ?? ''`) + const snapshotPage = /^\/admin\/collections\/taxonomy-snapshots\/\w+$/ + + // Sign in from the browser, so the cookie is the browser's own. + await theme('light') + await go('/admin/login') + await until(`document.querySelector('input[name="email"], input#field-email')`, 'the sign-in form') + const signedIn = await js(`fetch('/api/users/login', { method: 'POST', credentials: 'include', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ email: 'curator@example.org', password: ${JSON.stringify(password)} }) }).then((r) => r.status)`) + check('the curator signs in from the browser', signedIn === 200, signedIn) + + // A staged snapshot, published with the keyboard. + await go(`/admin/collections/taxonomy-snapshots/${staged.id}`) + await panelSays('Staged', 'the snapshot panel') + let text = await panelText() + check('the snapshot panel shows status, counts and tables', text.includes('Staged') && /\d+ names/.test(text) && text.includes('Phylogenies'), text.slice(0, 400)) + await look('01-snapshot-staged') + const bare = await js(`({ content: document.documentElement.innerHTML.includes('schemaVersion'), field: Boolean(document.querySelector('#field-snapshot, .json-field, .monaco-editor')), label: document.querySelector('#field-label')?.value ?? null })`) + check('a stored snapshot\'s page is sent without its content, has no code editor, and asks nobody for one', bare.content === false && bare.field === false && editor.length === 0 && bare.label === 'fixture-structures 1', { ...bare, editor: editor.length }) + await focusButton('Publish this snapshot') + await key('Enter') + await sleep(300) + check('Enter on the button opens the question and moves the focus to it', (await focused()).startsWith('group:Publish this snapshot?'), await focused()) + await key('Escape') + await sleep(300) + check('Escape closes it and returns the focus to the button', (await focused()) === 'button:Publish this snapshot', await focused()) + await key('Enter') + await sleep(300) + await look('02-snapshot-confirm') + requests.length = 0 + await clickButton('Publish') + await panelSays('This snapshot is published.', 'the snapshot to publish') + await sleep(1500) + text = await panelText() + const formStatus = await js(`document.querySelector('#field-status')?.innerText ?? ''`) + check('after publishing, the panel says Published and offers Withdraw', text.includes('Published') && text.includes('Withdraw this snapshot') && !text.includes('Publish this snapshot'), text.slice(0, 300)) + check('after publishing, Payload\'s own status field shows published and its form has nothing to save', /published/i.test(formStatus) && (await saveIsOff()) === true, { formStatus: formStatus.slice(0, 80), saveIsOff: await saveIsOff() }) + check('publishing sent one PATCH that asks for three fields back, then read the summary again', requests.filter((r) => r.startsWith('PATCH')).length === 1 && requests[0].includes('select[status]=true') && requests.some((r) => r.startsWith('GET') && r.endsWith('/summary')), requests) + await look('03-snapshot-published') + check('Payload follows the system theme: light', (await js(`document.documentElement.getAttribute('data-theme')`)) === 'light') + await theme('dark') + await go(`/admin/collections/taxonomy-snapshots/${staged.id}`) + check('and dark, with the panel still there after a reload', await until(`document.documentElement.getAttribute('data-theme') === 'dark' && document.querySelector('.onetaxonomy-panel')?.innerText.includes('Published')`, 'the dark theme after a reload')) + await sleep(500) + await look('04-snapshot-published-dark') + + // A published view, and a draft one with an unsaved change. + await go(`/admin/collections/taxonomy-views/${views[0].id}`) + await until(`document.querySelector('.onetaxonomy-panel svg')`, 'the view panel drawing') + text = await panelText() + check('the view panel draws the saved view and offers Withdraw on a published one', text.includes('Published') && text.includes('Withdraw this view') && text.includes('The same tree as an outline'), text.slice(0, 300)) + await look('05-view-published-dark') + const drawn = await js(`(() => { const s = document.querySelector('.onetaxonomy-figure svg'); return { shown: Math.round(s.getBoundingClientRect().width), own: Math.round(s.viewBox.baseVal.width) } })()`) + check('a small tree is drawn at its own size, not blown up', drawn.shown === drawn.own, drawn) + await theme('light') + await go(`/admin/collections/taxonomy-views/${draft.json.doc.id}`) + await until(`document.documentElement.getAttribute('data-theme') === 'light' && document.querySelector('.onetaxonomy-panel svg')`, 'the draft view panel drawing') + await type('#field-title', 'A draft view, edited') + await until(`${button('Publish this view', '.onetaxonomy-panel')}?.disabled === true`, 'Publish to wait for unsaved changes', 10_000) + text = await panelText() + check('with an unsaved change, Publish waits and says to save first', text.includes('Save your changes first.'), text.slice(0, 300)) + await look('06-view-draft-unsaved') + await js(`document.querySelector('#action-save').click()`) + await until(`document.querySelector('.onetaxonomy-panel [aria-label="Drawing of A draft view, edited"]') && ${button('Publish this view', '.onetaxonomy-panel')}?.disabled === false`, 'the view panel to follow Payload\'s Save', 20_000) + text = await panelText() + check('after Payload\'s own Save the view panel shows the saved title and offers Publish again', !text.includes('Save your changes first.') && (await js(`Boolean(document.querySelector('.onetaxonomy-panel [aria-label="Drawing of A draft view, edited"]'))`)) === true, text.slice(0, 300)) + + // The import jobs: one waiting, one that failed, and one over a file that was imported already. + const release1 = JSON.parse(fixture('checklist.json')) + await go(`/admin/collections/taxonomy-imports/${waiting.id}`) + await panelSays('Run this import', 'the import panel') + text = await panelText() + check('a waiting job says Queued, shows four steps not yet done, and offers to run', text.includes('Queued') && text.includes('It hasn’t run yet.') && (text.match(/not yet/g) ?? []).length === 4 && !text.includes('Runs started'), text.slice(0, 400)) + await look('07-import-queued') + requests.length = 0 + await focusButton('Run this import') + await key('Enter') + await panelSays('The import finished and stored a staged snapshot.', 'the import to finish') + await sleep(1500) + text = await panelText() + const storedLink = await linkTo('Open the staged snapshot this job stored') + check('running it by keyboard ends at Done, with every step done, the snapshot\'s checksum and a link to it', text.includes('Done') && (text.match(/\bdone\b/g) ?? []).length >= 4 && text.includes(release1.checksum) && snapshotPage.test(storedLink) && !text.includes('Run this import') && !text.includes('Run it again'), { storedLink, text: text.slice(0, 500) }) + check('running it sent one POST, and the page\'s own form has nothing to save', requests.filter((r) => r.startsWith('POST') && !readsByPost.has(r)).length === 1 && requests.some((r) => r.endsWith('/run')) && (await saveIsOff()) === true, { requests, saveIsOff: await saveIsOff() }) + const copies = await js(`Object.fromEntries(['message', 'checksum', 'inputSha256', 'diagnostics', 'warnings', 'warningCount', 'progress'].map((n) => { const e = document.querySelector('#field-' + n); return [n, e ? e.type + ':' + String(e.value).slice(0, 80) : null] }))`) + check('the form below the panel doesn\'t show the result a second time, and still holds it after the run', Object.values(copies).every((v) => typeof v === 'string' && v.startsWith('hidden:')) && copies.checksum === `hidden:${release1.checksum}` && copies.message.length > 'hidden:'.length, copies) + await look('08-import-done') + await go(storedLink) + await panelSays('Staged', 'the imported snapshot\'s panel') + const imported = await js(`({ label: document.querySelector('#field-label')?.value ?? null, checksum: document.querySelector('#field-checksum')?.value ?? null })`) + text = await panelText() + check('its link opens the staged snapshot, with the checksum of fixtures/checklist.json', imported.label === 'fixture-checklist 1' && imported.checksum === release1.checksum && text.includes(`${Object.keys(release1.names).length} names`), { imported, text: text.slice(0, 300) }) + await look('09-import-snapshot') + + await go(`/admin/collections/taxonomy-imports/${broken.id}`) + await panelSays('Run it again', 'the failed import\'s panel') + text = await panelText() + const logOpen = await js(`[...document.querySelectorAll('.onetaxonomy-panel details')].find((d) => d.querySelector('summary')?.textContent.trim() === 'Progress log')?.open ?? null`) + check('a failed job says Failed, what stopped it and where it stopped, with its log open', text.includes('Failed') && text.includes('What stopped it') && text.includes('Missing required column') && text.includes('stopped here') && logOpen === true && /Runs started\s+1\b/.test(text), { logOpen, text: text.slice(0, 500) }) + await look('10-import-failed') + await clickButton('Run it again') + await panelSays('The import stopped.', 'the failed import to stop again') + await sleep(1500) + text = await panelText() + check('running it again fails the same way, says so, and still offers to run', text.includes('Failed') && text.includes('Missing required column') && /Runs started\s+2\b/.test(text) && text.includes('Run it again') && !(await linkTo('Open the staged snapshot this job stored')), text.slice(0, 500)) + + await go(`/admin/collections/taxonomy-imports/${same.json.doc.id}`) + await panelSays('Run this import', 'the second job\'s panel') + await clickButton('Run this import') + await panelSays('This file with these settings was imported before.', 'the second job to find the snapshot') + await sleep(1500) + text = await panelText() + check('a job over the same file and settings ends at Already imported and links to the same snapshot', text.includes('Already imported') && (await linkTo('Open the staged snapshot this job found')) === storedLink, { link: await linkTo('Open the staged snapshot this job found'), text: text.slice(0, 300) }) + await look('11-import-already') + await type('#field-title', 'The same file again, renamed') + await until(`(() => { const b = document.querySelector('#action-save'); return b && !b.disabled })()`, 'Save to wake after the title changed', 10_000) + await clickAt('#action-save') + await until(`(() => { const b = document.querySelector('#action-save'); return b && (b.disabled || b.getAttribute('aria-disabled') === 'true') })()`, 'Payload\'s Save on a finished job', 20_000) + await sleep(1000) + const renamed = await api('GET', `/api/taxonomy-imports/${same.json.doc.id}?depth=0`, { as: curator }) + text = await panelText() + check('Payload\'s own Save renames a finished job and leaves its result as it was', renamed.json?.title === 'The same file again, renamed' && renamed.json?.status === 'already-imported' && renamed.json?.checksum === release1.checksum && renamed.json?.attempts === 1 && text.includes('Already imported'), { title: renamed.json?.title, status: renamed.json?.status, checksum: renamed.json?.checksum, attempts: renamed.json?.attempts, text: text.slice(0, 200) }) + + // A shared proposal, accepted into a new version. + await go(`/admin/collections/taxonomy-proposals/${made.json.doc.id}`) + await panelSays('Accept this proposal', 'the proposal panel') + text = await panelText() + check('the proposal panel shows the change in names and the author\'s handle beside the id', text.includes('Aves') && text.includes('Morgan Member') && text.includes('Shared'), text.slice(0, 500)) + await look('12-proposal-shared') + await clickButton('Accept this proposal') + await sleep(300) + const heldBack = await js(`${button('Accept into a new version', '.onetaxonomy-confirm')}?.disabled`) + await type('.onetaxonomy-confirm input', '2') + await sleep(300) + await look('13-proposal-accept-confirm') + const warned = (await panelText()).includes('These are sealed into the new version and can’t be taken out later') + check('accepting waits for a version name and warns what is sealed', heldBack === true && warned, { heldBack, warned }) + requests.length = 0 + await clickButton('Accept into a new version', '.onetaxonomy-confirm') + await panelSays('Accepted into the new staged version 2', 'the proposal to be accepted') + await sleep(1500) + text = await panelText() + const acceptedLink = await linkTo('Open the new snapshot') + check('after accepting: Accepted, no decide buttons, and a link to the new snapshot in the admin', text.includes('Accepted') && !text.includes('Accept this proposal') && snapshotPage.test(acceptedLink), { acceptedLink, text: text.slice(0, 300) }) + check('accepting sent one POST of its own', requests.filter((r) => r.startsWith('POST') && !readsByPost.has(r)).length === 1 && requests.some((r) => r.endsWith('/accept')), { requests, readsByPost: [...readsByPost] }) + await look('14-proposal-accepted') + await go(acceptedLink) + await panelSays('by accepting a proposal', 'the new snapshot\'s panel') + text = await panelText() + check('the new snapshot\'s panel shows who proposed and accepted, by id with the handle looked up', text.includes('Morgan Member') && text.includes('Cara Curator') && text.includes('The ids are sealed in this snapshot.'), text.slice(0, 600)) + await look('15-snapshot-from-proposal') + + // A new snapshot, pasted into Payload's own form, then published with Payload's own status field and Save. + const postData = async (w) => (await send('Network.getRequestPostData', { requestId: w.id }).catch(() => ({ postData: '' }))).postData ?? '' + const pasted = fixture('checklist-v2.json') + const sealed = JSON.parse(pasted) + await go('/admin/collections/taxonomy-snapshots/create') + await until(`document.querySelector('#field-snapshot') || document.querySelector('.json-field')`, 'the new-snapshot form') + const hasEditor = await until(`window.monaco?.editor?.getModels?.().length > 0`, 'Payload\'s code editor on the new-snapshot form (it loads from cdn.jsdelivr.net, so this needs the network)') + check('the new-snapshot form has the field to paste into, and no panel yet', hasEditor && (await js(`Boolean(document.querySelector('.json-field .monaco-editor'))`)) === true && (await panelText()) === '', { panel: await panelText() }) + await look('16-snapshot-create') + await js(`window.monaco.editor.getModels()[0].setValue(${JSON.stringify(pasted)})`) + await sleep(1000) + writes.length = 0 + await clickAt('#action-save') + await until(`/\\/admin\\/collections\\/taxonomy-snapshots\\/[0-9a-f]{24}$/.test(location.pathname)`, 'the new snapshot to be stored and its page opened') + await panelSays('Staged', 'the new snapshot\'s panel') + const createdPath = await js(`location.pathname`) + const created = createdPath.split('/').pop() + text = await panelText() + const afterCreate = await js(`({ content: document.documentElement.innerHTML.includes('schemaVersion'), field: Boolean(document.querySelector('#field-snapshot, .json-field, .monaco-editor')), label: document.querySelector('#field-label')?.value ?? null, checksum: document.querySelector('#field-checksum')?.value ?? null })`) + check('pasting a sealed snapshot and pressing Save stores it and opens its page, with the panel and no editor', writes.length === 1 && writes[0].method === 'POST' && text.includes('Staged') && afterCreate.field === false && afterCreate.label === 'fixture-checklist 2' && afterCreate.checksum === sealed.checksum, { writes, afterCreate, text: text.slice(0, 200) }) + check('the page the form lands on after storing holds no snapshot content', afterCreate.content === false, afterCreate) + await look('17-snapshot-created') + const stored = await api('GET', `/api/taxonomy-snapshots/${created}?depth=0&select[snapshot]=true&select[status]=true`, { as: curator }) + check('what was stored is what was pasted: the same checksum and the same names', stored.json?.snapshot?.checksum === sealed.checksum && stored.json?.status === 'staged' && Object.keys(stored.json?.snapshot?.names ?? {}).length === Object.keys(sealed.names).length, { status: stored.status, got: stored.json?.snapshot?.checksum }) + + await go(createdPath) + await panelSays('Staged', 'the stored snapshot\'s panel after a reload') + await sleep(1500) + await clickAt('#field-status .rs__control') + await until(`document.querySelectorAll('.rs__option').length > 0`, 'the status options', 10_000) + await js(`[...document.querySelectorAll('.rs__option')].find((o) => /^published$/i.test(o.textContent.trim()))?.setAttribute('data-pick', '1')`) + await clickAt('.rs__option[data-pick="1"]') + await until(`(() => { const b = document.querySelector('#action-save'); return b && !b.disabled })()`, 'Save to wake after the status changed', 10_000) + text = await panelText() + check('with the status changed in Payload\'s field and not saved, the panel still shows what is stored', text.includes('Staged') && text.includes('Publish this snapshot'), text.slice(0, 300)) + writes.length = 0 + await clickAt('#action-save') + await panelSays('Published', 'Payload\'s Save to publish the snapshot') + await sleep(1500) + const sentBodies = [] + for (const w of writes) { + const body = await postData(w) + sentBodies.push({ method: w.method, url: w.url, length: body.length, content: /schemaVersion|"snapshot"\s*:/.test(body), status: /published/.test(body) }) + } + const afterSave = await api('GET', `/api/taxonomy-snapshots/${created}?depth=0&select[snapshot]=true&select[status]=true&select[publishedBy]=true`, { as: curator }) + text = await panelText() + check('Payload\'s own Save publishes it: one PATCH that carries the status and no snapshot content', sentBodies.length === 1 && sentBodies[0].method === 'PATCH' && sentBodies[0].content === false && sentBodies[0].status === true && sentBodies[0].length < 4000, sentBodies) + check('after that Save the stored content is untouched, the status is published, and the panel says so', afterSave.json?.status === 'published' && afterSave.json?.snapshot?.checksum === sealed.checksum && Boolean(afterSave.json?.publishedBy) && text.includes('Published') && text.includes('Withdraw this snapshot'), { status: afterSave.json?.status, checksum: afterSave.json?.snapshot?.checksum, by: afterSave.json?.publishedBy, text: text.slice(0, 200) }) + await look('18-snapshot-saved-by-payload') + + await go('/admin/collections/taxonomy-snapshots') + await until(`document.querySelectorAll('table tbody tr').length >= 3`, 'the list of snapshots') + check('the list of snapshots in the admin is sent without any snapshot\'s content', (await js(`document.documentElement.innerHTML.includes('schemaVersion')`)) === false) + const listed = await js(`document.querySelector('table')?.innerText ?? ''`) + check('and still names each snapshot by its label, which the form no longer shows', listed.includes('fixture-amniotes 1') && listed.includes('fixture-checklist 2'), listed.slice(0, 400)) + + check('on every page looked at, the panel\'s parts are spaced apart, as the browser lays them out', flushAt.length === 0, flushAt.slice(0, 6)) + check('no page error, console error or request off the site, apart from Payload\'s code editor', problems.length === 0, problems.slice(0, 8)) + console.log(` Payload's code editor was fetched from cdn.jsdelivr.net in ${editor.length} requests${editorCancelled.length ? `, and said a load was cancelled ${editorCancelled.length} times when its page was left` : ''}.`) + ws.close() +} catch (error) { + check('the click-through ran to the end', false, String(error?.stack ?? error)) +} finally { + if (browser) { + browser.kill() + await finished(browser) + } + rmSync(profile, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }) + server.kill('SIGTERM') + await finished(server) +} + +if (failed) console.log(`\nThe site's last output:\n${log.slice(-3000)}`) +console.log(`\n${passed} passed, ${failed} failed.${shots ? ` Screenshots are in ${shots}.` : ''}`) +process.exit(failed ? 1 : 0) diff --git a/docs/architecture/adr/0012-payload-plugin.md b/docs/architecture/adr/0012-payload-plugin.md index f3d0aa8..18c59cf 100644 --- a/docs/architecture/adr/0012-payload-plugin.md +++ b/docs/architecture/adr/0012-payload-plugin.md @@ -44,11 +44,11 @@ - **Browser code is a separate entry.** The panels are client components in `@captiveorigins/payload-plugin-onetaxonomy/client`, which Payload finds through its import map. `@payloadcms/ui` and `react` are optional peer dependencies, so a host that turns the panels off, or uses the plugin without an admin, needs neither. A test checks that the server entry imports no browser code and the client entry no server code. - **Every answer is the site's.** The panels call the host's own site from the browser, as the signed-in person, with their cookie. Publishing and withdrawing are Payload's standard `PATCH` of `status`, asking back only the fields the panel shows, so the answer never carries a snapshot's content. Running an import, accepting and declining are the endpoints of points 12 and 15. Which buttons appear follows Payload's own word on whether this person may save the document; that is a courtesy, and a button shown by mistake still gets the site's 403 or 404. A refusal is shown as "Nothing was changed." with the site's reason. - **One new endpoint.** `GET /api//:id/summary` (and `summarizeSnapshot`) answers what a snapshot holds, small enough for a page: its status, counts, sources with their licenses, a line per classification and phylogeny, and what it was made from. Lists are capped (20 by default, never past `limits.maxResults`) and say when they were cut. It never returns names, concepts or trees. Same access and the same 404 as point 8, after the checksum check of point 3. For a snapshot made from an accepted proposal it gives the two sealed person ids, and beside each the name `proposals.displayName` gives now: for a signed-in requester only, looked up per request, never stored. A visitor who may read the snapshot gets the ids and no names. - - **Ask first, and say what can't be undone.** Every button opens a confirm in the page, and nothing is sent before it. Publishing and withdrawing say that nothing moves back. Accepting lists what is sealed into the new version for good (the title, the notes and move notes, the manager's note, the date and the two ids) and asks the manager to check that none of it holds personal details, because point 15 doesn't screen free text. Declining says the author will see the note. Accepting sends the checksum the panel showed, so a proposal that changed meanwhile is refused (point 15). + - **Ask first, and say what can't be undone.** Publish, Withdraw, Accept and Decline open a confirm in the page, and nothing is sent before it. Running an import doesn't ask: the most it does is store a staged snapshot, which only managers see and which can be deleted. Publishing and withdrawing say that nothing moves back. Accepting lists what is sealed into the new version for good (the title, the notes and move notes, the manager's note, the date and the two ids) and asks the manager to check that none of it holds personal details, because point 15 doesn't screen free text. Declining says the author will see the note. Accepting sends the checksum the panel showed, so a proposal that changed meanwhile is refused (point 15). - **A view's drawing is bounded and has an outline.** The view panel draws the saved view from its projection, cut at 200 nodes and saying so, with the same tree as an outline for the keyboard and the lines readers are told about it. Publish waits, with the reason beside it, until the form is saved and the snapshot is published. - **An import's progress is asked for, not pushed.** While a job is running the panel reads its few run fields again every 3 seconds, and stops when it isn't or when the page is left. A job running past `staleAfterMinutes` is called probably interrupted and can be run again; a running one can't be started twice from the panel, and point 12's claim refuses it anyway. - **Readable without colour, in Payload's themes.** A status is a word, with a coloured edge beside it; the state colours are used for edges only, never for text. The panels take their colours from Payload's theme variables, and a test reads the installed `@payloadcms/ui` stylesheets and checks text and muted text at 4.5:1 and control edges at 3:1 in the light and dark themes. The drawing's colours are copies of the React package's, kept equal by a test, and switch on Payload's own theme marker, because the admin's theme is the person's choice in Payload and not the system's. - - **Limits.** Tested in jsdom with Payload's hooks and `fetch` replaced by stand-ins, over HTTP in the example site, and clicked through in headless Chrome on that site (snapshot, view and proposal panels, light and dark); those runs aren't in CI. The import panel hasn't run in a real admin. No screen reader, Safari, Firefox or phone was tried. Payload's admin makes two requests of its own to other sites on these pages: its JSON field loads its code editor from cdn.jsdelivr.net (not on a stored snapshot's page, which has no JSON field since point 17), and its default avatar asks gravatar.com with a hash of the signed-in person's email address unless the host sets `admin.avatar: 'default'`. The panels themselves ask only the host's site. + - **Limits.** Tested in jsdom with Payload's hooks and `fetch` replaced by stand-ins, over HTTP in the example site, and clicked through in headless Chrome on that site by its `click-through` script (all four panels, light and dark, 40 checks), which needs Chrome and the network and so isn't in CI. That script's first run on the import pages found the panel's paragraphs and lists flush against each other in the real admin: the panel's own margin reset outranked its spacing rule, which jsdom can't show. The reset now carries no weight (`:where`), and the script measures the gaps. The form under a panel no longer repeats it: the fields a panel shows are `admin.hidden`, so they stay in the form's data, where the import panel puts a run's result so that a later Save doesn't send the old one. No screen reader, Safari, Firefox or phone was tried. Payload's admin makes two requests of its own to other sites on these pages: its JSON field loads its code editor from cdn.jsdelivr.net (not on a stored snapshot's page, which has no JSON field since point 17), and its default avatar asks gravatar.com with a hash of the signed-in person's email address unless the host sets `admin.avatar: 'default'`. The panels themselves ask only the host's site. 17. **A snapshot's content is read only when it is asked for by name.** A stored snapshot can be 15 MB, and Payload reads whole documents by default: for a list, for a relationship it fills in, and for every admin page. - **The rule.** A read, create, update or delete of a snapshot that names no `select` answers with every field except `snapshot`. `select: { snapshot: true }` (REST `?select[snapshot]=true`, GraphQL `select: true`) gives the content. A caller's own `select` is never changed, so an exclude-mode select such as `{ status: false }` still carries the content. The export endpoint and the read API of points 8 and 13 are as before. @@ -57,7 +57,7 @@ - **The admin.** The content field is shown on the new-snapshot form only (`admin.condition`), where a manager pastes a sealed snapshot. A stored snapshot's page is sent without the content and without the editor; the panel says what the snapshot holds. Payload's own Save there sends the status and no content. The same condition is what makes the field nullable in GraphQL, so a plain query answers `null` for it instead of failing. - **The panels follow Payload's Save.** The panels read their own data from the site, so they went stale when a manager changed the status with Payload's own field and Save. They now read again whenever Payload reports the document saved. - **For hosts.** Code that read `doc.snapshot` from a plain read, or from the document an after-read or after-change hook is given, has to select it. The plugin has no release yet, so nothing in use changes. - - **Limits.** A stored snapshot's content can't be looked at in the admin; the export gives it. Tested on a real Payload with an in-memory MongoDB, over HTTP in the example site, and in headless Chrome on that site from a scratch script that isn't in CI. In that script, after a snapshot was made through the form and its page loaded again, Payload's read-only Counts group showed its heading with no fields under it; four shorter runs of the same steps drew them, and the cause wasn't found. Not checked: Postgres or SQLite, a browser with a window. + - **Limits.** A stored snapshot's content can't be looked at in the admin; the export gives it. Tested on a real Payload with an in-memory MongoDB, over HTTP in the example site, and in headless Chrome on that site by its `click-through` script, which isn't in CI. In an earlier scratch version of that script, after a snapshot was made through the form and its page loaded again, Payload's read-only Counts group once showed its heading with no fields under it; four shorter runs of the same steps drew them, and the cause wasn't found. The form no longer draws that group, because the panel shows the counts (point 16), so it can't be seen again there; a host's own read-only group in a Payload form could still show it. Not checked: Postgres or SQLite, a browser with a window. **Later parts of M7, not built yet.** diff --git a/docs/test-audit.md b/docs/test-audit.md index 3364acc..8f12e7e 100644 --- a/docs/test-audit.md +++ b/docs/test-audit.md @@ -2,6 +2,35 @@ Each audit lists what this repo promises, the tests that prove it, and the gaps: those closed in the same pass and those left, with why. Audits run after each feature or fix, and at least every 14 days while the repo has commits. Newest first. +## 2026-11-11: imports in the example site, and the admin clicked through + +Scope: `apps/payload-example` (the `sources` upload collection, import jobs on, two seeded jobs, the throwaway uploads folder, `scripts/smoke.mjs`, the new `scripts/click-through.mjs`); and in `packages/payload`, the panels' spacing (`src/client/panels.css`) and the fields a panel already shows, now hidden in the form (`src/collections.ts`, `src/imports.ts`). Plugin tests went from 247 to 249, all unit tests from 631 to 633, the example's smoke checks from 58 to 74. The click-through has 40 checks and is not in CI. + +### Covered + +- **Who can reach a source file** (`scripts/smoke.mjs`, in CI): a visitor and a member are each refused the list of source files, a file by its address, the import jobs, running one, and uploading; afterwards nobody but the curator has run the waiting job or added a file. An HTML file is refused as a source file even from the curator. +- **Import jobs over HTTP** (`scripts/smoke.mjs`): the seed's two jobs are there, one waiting and one failed with its reason; the curator runs the waiting one into a staged snapshot with the checksum of `fixtures/checklist.json`; running it again stores nothing new; a second job over the same file and settings waits until run, then finds that snapshot; the failed job fails the same way and stores nothing; a job over a newly uploaded file gives the checksum of `fixtures/checklist-v2.json`; a finished job's results can't be edited. +- **Nothing left behind** (`scripts/smoke.mjs`): the folder of uploaded files is gone once the site has stopped. +- **The form doesn't repeat the panel, and still holds what it hides** (`snapshots.test.ts`, `imports.test.ts`, 2 new tests): the fields are hidden and read-only, not disabled, and the snapshot's label is still its title and a list column. +- **In a real browser** (`scripts/click-through.mjs`, headless Chrome on the built site, as the made-up curator, not in CI): the earlier scratch run's steps (publishing a snapshot by keyboard, the wait on an unsaved view, accepting a proposal, pasting a snapshot, Payload's own Save on one), and new ones: a waiting job run by keyboard to Done, with its checksum and a link to the staged snapshot; the failed job run again; a job over the same file ending at Already imported; a finished job renamed with Payload's own Save, its result unchanged; the hidden fields holding the run's result; the snapshot list still naming each snapshot; the panel's parts spaced apart on every page looked at; and no request off the site except Payload's code editor. + +### Checked by breaking it + +Six deliberate breaks, each failing its own check, then restored. In the smoke test, three in one run: source files readable by anyone (the visitor's and the member's refusals), jobs running when made (the seeded waiting job), and the uploads folder not deleted. In the unit tests: the hidden fields disabled instead. In the click-through: the old margin reset (the spacing check, seen on the build made before the fix), and the panel no longer putting a run's result into the form (the hidden fields were empty, and Payload's Save on the finished job was refused). That last break first hung the script: the unsaved form made the browser ask "Leave site?", which held every later step. The script now answers such a question, counts it as a problem, and gives up on a browser that doesn't answer within two minutes. + +### Gaps left + +- **The click-through is not in CI.** It needs a Chrome on the machine and the network for Payload's code editor, and takes about two minutes. It is run by hand after a change to a panel or a collection's fields. What CI runs for the same promises: the jsdom panel tests and the smoke test. +- **The panels' looks are measured only as gaps.** The script checks that no two parts of a panel touch. Screenshots were looked at once for this change, in light and dark; nothing compares them. +- **The empty Counts group** from the last audit can't be seen any more, because the form no longer draws that group. Its cause was never found. +- **Imports from a URL** aren't in the example site. The plugin's own tests cover them with a stand-in `fetch`, never a real network. +- **A screen reader, Safari, Firefox, a phone, Postgres and SQLite:** as before, not run. + +### Left out as overkill + +- An upload collection's own behaviour (file names, sizes, MIME sniffing): Payload's. The example checks only its access rules and that Payload's refusal of HTML is on. +- Clicking through every seeded record. One of each state a panel has is enough. + ## 2026-11-11: a snapshot's content only on request Scope: `packages/payload`: the `contentOnRequest` hook and the content field's admin condition (`src/collections.ts`), the panels reading again after Payload's own Save (`src/client/wired.tsx`); and `apps/payload-example`. ADR 0012 point 17. Plugin tests went from 237 to 247, all unit tests from 621 to 631, the example's smoke checks from 55 to 58. diff --git a/packages/payload/README.md b/packages/payload/README.md index 878a3eb..ceae7e0 100644 --- a/packages/payload/README.md +++ b/packages/payload/README.md @@ -2,7 +2,7 @@ OneTaxonomy for Payload: sealed taxonomy snapshots, views pinned to them, import jobs that turn source files into staged snapshots, people's own proposals that a manager can accept into a new staged version, and a small read API for search, ancestors, descendants, MRCA, coverage, projected views and exports, all behind your own access rules. -> **Status: early, parts 1 to 5 of M7.** Not on npm yet. The collections, publishing, tenant scope, import jobs, proposals, the read endpoints and the admin panels work. The server side is tested on a real Payload 3.93 with an in-memory MongoDB (160 tests): bad checksums refused, sealed snapshots locked, a snapshot's content left out of every read that doesn't name it, an import of each format, re-runs that store nothing new, malformed files refused with their line, interrupted jobs resumed, runs started together claimed by one, every endpoint's results, a view's `source` through the Local API and GraphQL, proposals kept private, shared, accepted into a staged version once and declined, access refusals and cross-tenant denial. Thirteen of those tests send real requests through Payload's own REST router, signed in with a token from the login route; the rest call the endpoint handlers and the Local API directly. It also runs in a small real site, [`apps/payload-example`](../../apps/payload-example) (Next.js 16, on this machine only), where 58 more checks go over HTTP with sign-in cookies. The [admin panels](#admin-panels) have 76 tests in jsdom and were clicked through in headless Chrome, outside CI; no screen reader, Safari or Firefox has been tried on them. It hasn't run on a deployed site, or on Postgres or SQLite, and no URL import has fetched over a real network. See [ADR 0012](../../docs/architecture/adr/0012-payload-plugin.md). +> **Status: early, parts 1 to 5 of M7.** Not on npm yet. The collections, publishing, tenant scope, import jobs, proposals, the read endpoints and the admin panels work. The server side is tested on a real Payload 3.93 with an in-memory MongoDB (162 tests): bad checksums refused, sealed snapshots locked, a snapshot's content left out of every read that doesn't name it, an import of each format, re-runs that store nothing new, malformed files refused with their line, interrupted jobs resumed, runs started together claimed by one, every endpoint's results, a view's `source` through the Local API and GraphQL, proposals kept private, shared, accepted into a staged version once and declined, access refusals and cross-tenant denial. Thirteen of those tests send real requests through Payload's own REST router, signed in with a token from the login route; the rest call the endpoint handlers and the Local API directly. It also runs in a small real site, [`apps/payload-example`](../../apps/payload-example) (Next.js 16, on this machine only), where 74 more checks go over HTTP with sign-in cookies. The [admin panels](#admin-panels) have 76 tests in jsdom, and a script clicks through all four in headless Chrome on that site (40 checks, outside CI); no screen reader, Safari or Firefox has been tried on them. It hasn't run on a deployed site, or on Postgres or SQLite, and no URL import has fetched over a real network. See [ADR 0012](../../docs/architecture/adr/0012-payload-plugin.md). ## What it does @@ -171,7 +171,9 @@ Each of the plugin's collections gets a panel at the top of its edit page. It is - **Import job.** The steps of the run with the one it is on, the result, the adapter's warnings and the diagnostics with their line or position. While a job is running the panel asks again every 3 seconds. A manager gets **Run this import** on a queued job and **Run it again** on a failed one, and on a running one only after `imports.staleAfterMinutes`. - **Proposal.** Whose it is, what it is drawn over, and what it would change: taxa with a new parent, groups that gain or lose members, groups only one side has. A manager gets **Accept this proposal**, which asks for the new version's name and an optional note, and **Decline this proposal**, which asks for a note the author will see. -Every button asks first, in the page, and says what can't be undone. Nothing is sent until you confirm. If the site refuses, the panel says "Nothing was changed." and gives the site's reason. Before a proposal is accepted the panel lists what gets sealed into the new version for good (the title, the notes, your note, the date and the two person ids) and asks you to check none of it holds personal details. +The form under a panel doesn't repeat it. A snapshot's `label`, `snapshotId`, `version`, `checksum` and `counts`, and an import job's `message`, `checksum`, `inputSha256`, `diagnostics`, `warnings`, `warningCount` and `progress`, are hidden in the form (`admin.hidden`). They are still in the form's data, in the API, and available as columns in the list. + +Publish, Withdraw, Accept and Decline ask first, in the page, and say what can't be undone. Nothing is sent until you confirm. Running an import doesn't ask: the most it does is store a staged snapshot, which only managers see and which can be deleted. If the site refuses, the panel says "Nothing was changed." and gives the site's reason. Before a proposal is accepted the panel lists what gets sealed into the new version for good (the title, the notes, your note, the date and the two person ids) and asks you to check none of it holds personal details. The panels are a page over the same API as everything else. They run in the browser and call your own site as the signed-in person, so your `access` callbacks decide every answer, and a button the panel shows by mistake still gets a 403 or 404. They add one endpoint, `GET /api/taxonomy-snapshots/:id/summary` (and `summarizeSnapshot(req, id)`): the counts, tables and sources above, with each list capped by `limit` (default 20), never the names or trees themselves. It answers the same 404 as the other endpoints. @@ -189,7 +191,7 @@ Without that the edit pages load with nothing where the panel should be, and Pay **Colours.** The panels take their colours from Payload's theme variables, so they follow the admin's light and dark themes and a host's own theme. A status is always a word with a coloured edge, never colour alone. A test reads the installed `@payloadcms/ui` stylesheets and checks that text and muted text reach 4.5:1 and control edges 3:1 in both themes. -**What was checked.** 76 tests render the panels in jsdom with Payload's hooks and `fetch` replaced by stand-ins: which buttons each person gets, that nothing is sent before the confirm, what a refusal says, and where Escape and the focus go. The example site's smoke test checks over HTTP that the built admin pages carry the panels and that the routes they call refuse a member. The panels were also clicked through in headless Chrome on the example site, in light and dark: publishing a snapshot with the keyboard, the wait on an unsaved view, accepting a proposal into a new version, storing a sealed snapshot by pasting it into the new-snapshot form, and Payload's own Save on a stored snapshot, which sent the status and no content. Those runs are not in CI. One thing in them isn't explained: after a snapshot was made through the form and its page loaded again, Payload's read-only Counts group showed its heading with no fields under it. Four shorter runs of the same steps drew the fields, and it hasn't been seen in a browser with a window. The import panel has not run in a real admin, because the example site has no imports. Not checked: a screen reader, Safari, Firefox, a phone, or a custom Payload theme. +**What was checked.** 76 tests render the panels in jsdom with Payload's hooks and `fetch` replaced by stand-ins: which buttons each person gets, that nothing is sent before the confirm, what a refusal says, and where Escape and the focus go. The example site's smoke test checks over HTTP that the built admin pages carry the panels and that the routes they call refuse a member. The example site's `click-through` script also does a curator's work in headless Chrome, in light and dark (40 checks): publishing a snapshot with the keyboard, the wait on an unsaved view, running an import to Done, a failed one again, and one over a file imported already, renaming a finished job with Payload's own Save, accepting a proposal into a new version, storing a sealed snapshot by pasting it into the new-snapshot form, and Payload's own Save on a stored snapshot, which sent the status and no content. It needs Chrome and the network, so it is not in CI. Its first run on the import pages found the panel's paragraphs and lists sitting flush against each other in the real admin, which jsdom can't see; that is fixed, and the script now measures the gaps. Not checked: a screen reader, Safari, Firefox, a phone, or a custom Payload theme. **What Payload's own admin asks other sites for.** The panels ask only your site. Payload's JSON field loads its code editor (Monaco) from cdn.jsdelivr.net, and its default avatar asks gravatar.com with a hash of the signed-in person's email address. Both are Payload's. The editor loads on pages with a JSON field you can see: the new-snapshot form and the view, import and proposal pages. A stored snapshot's page has none, so it doesn't ask. The avatar is asked for on every admin page; set `admin.avatar: 'default'` in your Payload config to stop it. diff --git a/packages/payload/src/client/ImportPanelView.tsx b/packages/payload/src/client/ImportPanelView.tsx index 7f911d6..fb58af0 100644 --- a/packages/payload/src/client/ImportPanelView.tsx +++ b/packages/payload/src/client/ImportPanelView.tsx @@ -89,7 +89,7 @@ export function ImportPanelView({ job, canRun, starting, staleAfterMinutes, now, return ( {/* Polite live region: a screen reader hears the status and message when they change, not on every check. */} -
+
{status.means} diff --git a/packages/payload/src/client/SnapshotPanelView.tsx b/packages/payload/src/client/SnapshotPanelView.tsx index b04c64c..fce8e18 100644 --- a/packages/payload/src/client/SnapshotPanelView.tsx +++ b/packages/payload/src/client/SnapshotPanelView.tsx @@ -159,7 +159,7 @@ export function SnapshotPanelView({ summary, canManage, outcome, onSetStatus, on {s.sources.map((x) => (
  • {x.title} - {x.illustrative ? Made-up example data : null} + {x.illustrative ? <> Made-up example data : null} {x.citation ?
    {x.citation}
    : null}
    {[x.license ? `License: ${x.license}` : 'No license recorded', x.externalVersion ? `version ${x.externalVersion}` : ''].filter(Boolean).join(', ')} diff --git a/packages/payload/src/client/panels.css b/packages/payload/src/client/panels.css index 14192c6..37e0905 100644 --- a/packages/payload/src/client/panels.css +++ b/packages/payload/src/client/panels.css @@ -34,14 +34,20 @@ .onetaxonomy-panel *::before { box-sizing: border-box; } -.onetaxonomy-panel p, -.onetaxonomy-panel h3, -.onetaxonomy-panel h4 { +/* + * Spacing comes from the stacks below, not from each element. `:where` keeps this reset weaker than they are: + * written as `.onetaxonomy-panel p` it would beat them, and paragraphs and lists would sit flush against each other. + */ +.onetaxonomy-panel :where(p, h3, h4, ul, ol, dl) { margin: 0; } .onetaxonomy-panel > * + * { margin-top: calc(var(--otp-gap) * 0.6); } +/* What a screen reader hears when it changes: the status, then what the last run said. */ +.onetaxonomy-panel__live > * + * { + margin-top: 0.35em; +} .onetaxonomy-panel__title { font-size: 1.15em; font-weight: 600; @@ -133,7 +139,6 @@ display: grid; grid-template-columns: repeat(auto-fill, minmax(14rem, 1fr)); gap: 0.6em var(--otp-gap); - margin: 0; } .onetaxonomy-facts dt { color: var(--otp-muted); @@ -274,7 +279,6 @@ white-space: nowrap; } .onetaxonomy-list { - margin: 0; padding-inline-start: 1.2em; } .onetaxonomy-list > li + li { @@ -306,7 +310,6 @@ display: flex; flex-wrap: wrap; gap: 0.5em; - margin: 0; padding: 0; list-style: none; } diff --git a/packages/payload/src/collections.ts b/packages/payload/src/collections.ts index e5b3fb2..c5f8117 100644 --- a/packages/payload/src/collections.ts +++ b/packages/payload/src/collections.ts @@ -119,6 +119,9 @@ const CONTENT_OPERATIONS = new Set(['read', 'create', 'update', 'delete']) export const contentOnRequest: CollectionBeforeOperationHook = ({ args, operation }) => CONTENT_OPERATIONS.has(operation) && args && (args as { select?: unknown }).select == null ? ({ ...args, select: { snapshot: false } } as typeof args) : args +/** For what a panel already shows. */ +const IN_PANEL = { readOnly: true, hidden: true } + /** Only staged snapshots and draft views can be deleted: published ones may be pinned by views and pages. */ const onlyUnpublishedDeleted = (slug: string, deletable: string, what: string): CollectionBeforeDeleteHook => async ({ id, req }) => { const doc = (await req.payload.findByID({ collection: slug as never, id, depth: 0, req, overrideAccess: true, disableErrors: true, select: { status: true } as never })) as { status?: string } | null @@ -188,7 +191,9 @@ export function snapshotsCollection(opts: ResolvedOptions): CollectionConfig { indexes: [{ fields: [...(opts.tenant ? [opts.tenant.field] : []), 'snapshotId', 'version'], unique: true }], fields: [ ...panelField(opts, 'SnapshotPanel'), - { name: 'label', type: 'text', admin: { readOnly: true, description: 'The snapshot\'s id and version. Set from the snapshot.' } }, + // The panel shows the label, id, version, checksum and counts. They are hidden, not disabled: they stay in the form + // and can still be columns in the list. + { name: 'label', type: 'text', admin: IN_PANEL }, { name: 'snapshot', type: 'json', @@ -200,13 +205,13 @@ export function snapshotsCollection(opts: ResolvedOptions): CollectionConfig { condition: (_data, _sibling, { operation }) => operation !== 'update', }, }, - { name: 'snapshotId', type: 'text', index: true, admin: { readOnly: true } }, - { name: 'version', type: 'text', admin: { readOnly: true } }, - { name: 'checksum', type: 'text', index: true, admin: { readOnly: true } }, + { name: 'snapshotId', type: 'text', index: true, admin: IN_PANEL }, + { name: 'version', type: 'text', admin: IN_PANEL }, + { name: 'checksum', type: 'text', index: true, admin: IN_PANEL }, { name: 'counts', type: 'group', - admin: { readOnly: true }, + admin: IN_PANEL, fields: ['names', 'concepts', 'classifications', 'phylogenies'].map((name) => ({ name, type: 'number' }) as Field), }, { name: 'status', type: 'select', required: true, defaultValue: 'staged', index: true, options: ['staged', 'published', 'withdrawn'], admin: { position: 'sidebar', description: 'Staged: only managers see it. Published: readers see it. Withdrawn: hidden from readers, kept for the views that pin it.' } }, diff --git a/packages/payload/src/imports.ts b/packages/payload/src/imports.ts index 0a417fe..45a81f9 100644 --- a/packages/payload/src/imports.ts +++ b/packages/payload/src/imports.ts @@ -404,6 +404,11 @@ const fillTenant = (opts: ResolvedOptions) => async ({ data, operation, req }: { } const ro = (description?: string) => ({ readOnly: true, ...(description ? { description } : {}) }) +/** + * For what the panel already shows. Hidden, not disabled: the field stays in the form, so the panel can put a run's + * result into it, and it can still be a column in the list. + */ +const inPanel = { readOnly: true, hidden: true } /** `taxonomy-imports`: import jobs. Managers only, in their own tenant. */ export function importsCollection(opts: ResolvedOptions): CollectionConfig { @@ -430,14 +435,14 @@ export function importsCollection(opts: ResolvedOptions): CollectionConfig { }, { name: 'status', type: 'select', required: true, defaultValue: 'queued', index: true, options: ['queued', 'running', 'done', 'already-imported', 'failed'], admin: { position: 'sidebar', ...ro() } }, { name: 'phase', type: 'select', options: ['reading', 'parsing', 'sealing', 'storing', 'finished'], admin: { position: 'sidebar', ...ro() } }, - { name: 'message', type: 'text', admin: ro('What the last run did, or why it stopped.') }, + { name: 'message', type: 'text', admin: inPanel }, { name: 'snapshot', type: 'relationship', relationTo: opts.slugs.snapshots as never, admin: { position: 'sidebar', ...ro('The staged snapshot this job stored or found.') } }, - { name: 'checksum', type: 'text', index: true, admin: ro() }, - { name: 'inputSha256', type: 'text', admin: ro('SHA-256 of the file as read.') }, - { name: 'diagnostics', type: 'json', admin: ro('Errors that stopped the job, and validation warnings.') }, - { name: 'warnings', type: 'json', admin: ro('What the adapter noticed and handled conservatively (the first 200).') }, - { name: 'warningCount', type: 'number', admin: ro() }, - { name: 'progress', type: 'json', admin: ro() }, + { name: 'checksum', type: 'text', index: true, admin: inPanel }, + { name: 'inputSha256', type: 'text', admin: inPanel }, + { name: 'diagnostics', type: 'json', admin: inPanel }, + { name: 'warnings', type: 'json', admin: inPanel }, + { name: 'warningCount', type: 'number', admin: inPanel }, + { name: 'progress', type: 'json', admin: inPanel }, { name: 'attempts', type: 'number', admin: { position: 'sidebar', ...ro() } }, { name: 'startedAt', type: 'date', admin: { position: 'sidebar', ...ro() } }, { name: 'finishedAt', type: 'date', admin: { position: 'sidebar', ...ro() } }, diff --git a/packages/payload/test/imports.test.ts b/packages/payload/test/imports.test.ts index d324ad3..2dd70eb 100644 --- a/packages/payload/test/imports.test.ts +++ b/packages/payload/test/imports.test.ts @@ -142,6 +142,22 @@ describe('importing each format into a staged, sealed snapshot', () => { }) }) +describe('the job\'s form in the admin', () => { + it('hides what the panel shows and keeps it in the form, so a later Save sends the run\'s result and not an old one', () => { + const fields = payload.collections[IMPORTS as never]!.config.fields + const admin = (name: string) => (fields.find((f) => 'name' in f && f.name === name) as { admin?: { hidden?: boolean; readOnly?: boolean; disabled?: boolean } }).admin + for (const name of ['message', 'checksum', 'inputSha256', 'diagnostics', 'warnings', 'warningCount', 'progress']) { + expect(admin(name), name).toMatchObject({ hidden: true, readOnly: true }) + expect(admin(name)?.disabled, name).toBeUndefined() + } + // What the panel doesn't repeat stays in view, read-only. + for (const name of ['status', 'phase', 'snapshot', 'attempts', 'startedAt', 'finishedAt', 'runBy']) { + expect(admin(name), name).toMatchObject({ readOnly: true }) + expect(admin(name)?.hidden, name).toBeUndefined() + } + }) +}) + describe('idempotent', () => { it('the same file with the same settings is "already imported": the same snapshot, nothing new stored', async () => { const first = (await payload.find({ collection: IMPORTS as never, where: { checksum: { equals: checklistJson.checksum } }, depth: 0 })).docs[0] as Job diff --git a/packages/payload/test/snapshots.test.ts b/packages/payload/test/snapshots.test.ts index d5eb82f..786cedd 100644 --- a/packages/payload/test/snapshots.test.ts +++ b/packages/payload/test/snapshots.test.ts @@ -189,6 +189,15 @@ describe('a stored snapshot\'s content is read only when it is asked for by name expect(field.admin.condition({}, {}, { operation: 'create' })).toBe(true) expect(field.admin.condition({ id }, {}, { operation: 'update' })).toBe(false) }) + + it('hides in the form what the panel shows, and keeps those fields in it and in the list', () => { + const config = payload.collections[SNAPSHOTS as never]!.config + const admin = (name: string) => (config.fields.find((f) => 'name' in f && f.name === name) as { admin?: { hidden?: boolean; readOnly?: boolean; disabled?: boolean } }).admin + for (const name of ['label', 'snapshotId', 'version', 'checksum', 'counts']) expect(admin(name), name).toMatchObject({ hidden: true, readOnly: true }) + for (const name of ['label', 'snapshotId', 'version', 'checksum', 'counts']) expect(admin(name)?.disabled, name).toBeUndefined() + expect(config.admin.useAsTitle).toBe('label') + expect(config.admin.defaultColumns).toEqual(expect.arrayContaining(['label', 'counts'])) + }) }) describe('publishing and editing', () => {