From bb87b0679f2c99e30886700796b63f812bac6df0 Mon Sep 17 00:00:00 2001 From: Aleksandr Shevchuk Date: Fri, 20 Nov 2026 10:36:35 +0200 Subject: [PATCH 1/2] test: a test's time limit is its package's, for the author package and the verifier (#255) CI failed on main for the third time in three days on a test of packages/author crossing vitest's 5 s on a slow runner: 5010, 5025 and 5011 ms, another test each time, each after the one before it got a number of its own. Two more ran within reach of it, at 3.6 to 5.0 s and 2.3 to 3.1 s over three CI runs. The author's Node project now sets testTimeout and hookTimeout to 120 s in its config, as embed, mcp, the demo and the skills project do, and the five numbers its tests carried go. The verifier gets the same in a config of its own, and its sixteen numbers of 120 s or less go: the three that ask for 240 s stay, and the one that tests a time. design/testing.md has the rule: a limit stops a hang and does not time a test, so it is set once for a package, for a slow runner. Closes #255. --- design/testing.md | 9 +++++++++ packages/author/tests/build.test.ts | 5 ++--- packages/author/tests/bundle.test.ts | 5 ++--- packages/author/tests/read.test.ts | 2 +- packages/author/vitest.config.ts | 8 +++++++- packages/verify/tests/preview.test.ts | 12 ++++++------ packages/verify/tests/verify.test.ts | 22 +++++++++++----------- packages/verify/vitest.config.ts | 12 ++++++++++++ 8 files changed, 50 insertions(+), 25 deletions(-) create mode 100644 packages/verify/vitest.config.ts diff --git a/design/testing.md b/design/testing.md index 8455874f..5aa772a9 100644 --- a/design/testing.md +++ b/design/testing.md @@ -22,6 +22,15 @@ which also runs when `packages/vscode` changes. The slow files are `stalling-hos publishing the player, run CI by hand on main (`gh workflow run ci.yml`) and publish once it is green. +**A test's time limit is its package's, set once in its config for a slow runner** (#255). A +limit stops a hang; it does not time a test. The author's Node tests ran on vitest's 5 s a test, +and a CI runner builds packages and writes bundles two to four times slower than a laptop: three +tests failed there in three days, each at 5.0 s, each after the one before it got a number of its +own. So embed, mcp, the demo, the skills project, the author package and the verifier each set +`testTimeout` and `hookTimeout` to 120 s in their config, and a test names its own only to ask for +more, or where the time is what it tests. A package whose tests run in milliseconds keeps the +default, until one of them needs a number: then the package gets its limit, not the test. + The `browser` project runs in Chromium unless `H5P_TEST_BROWSER` names `firefox` or `webkit`. `pnpm test:engine` runs it, then the app's tests, which is what the `engines` job does in those two. One engine per run, never a list: the dev server's switches (a host's outage, the worker version it diff --git a/packages/author/tests/build.test.ts b/packages/author/tests/build.test.ts index bea486d1..ce5ad4d6 100644 --- a/packages/author/tests/build.test.ts +++ b/packages/author/tests/build.test.ts @@ -899,8 +899,7 @@ describe('media', () => { expect(await problems(source(['a stray line', text(500, 600)], []), { base: dir })).toEqual([ expect.stringMatching(/^params\.interactiveVideo\.assets\.interactions\[0\]: expected an object/) ]) - // Several builds of a video: 1.3 s here, 2 to 3 s on CI, past vitest's 5 s once on a slow runner (#242). - }, 20_000) + }) it('gives a download up once nothing has arrived for a while, however long a slow one takes', async () => { const reported: string[] = [] @@ -1013,7 +1012,7 @@ describe('the build', () => { expect((await exec(process.execPath, [bin, '-h'])).stdout).toMatch(/^Usage:\n {2}h5p-author build/) expect((await exec(process.execPath, [bin, '-v'])).stdout).toMatch(/^@missing-elements\/h5p-author \d+\.\d+\.\d+, library pack \d+\.\d+\.\d+\n$/) await expect(exec(process.execPath, [bin, 'build', 'a.json', 'a.h5p', '--markdown'])).rejects.toMatchObject({ code: 1, stderr: 'h5p-author: build does not take --markdown\n' }) - }, 120_000) + }) }) describe('the package', () => { diff --git a/packages/author/tests/bundle.test.ts b/packages/author/tests/bundle.test.ts index 1f04a47d..1ce3c965 100644 --- a/packages/author/tests/bundle.test.ts +++ b/packages/author/tests/bundle.test.ts @@ -224,8 +224,7 @@ describe('the bundle command', () => { await expect(exec(process.execPath, [bin, 'bundle', 'H5P.TrueFalse'])).rejects.toMatchObject({ code: 1, stderr: expect.stringContaining('bundle takes a content type and an output') }) await expect(exec(process.execPath, [bin, 'bundle', '--markdown', 'H5P.TrueFalse', 'x.zip'])).rejects.toMatchObject({ code: 1, stderr: expect.stringContaining('bundle does not take --markdown') }) - // Every content type's bundle, written twice: 2.2 s here, past vitest's 5 s once on a slow runner (#242). - }, 20_000) + }) }) describe('the command line as one file', () => { @@ -255,5 +254,5 @@ describe('the command line as one file', () => { const { stdout: resolved } = await exec(process.execPath, ['--input-type=module', '-e', script], { cwd: scope }) const real = await realpath(scope) expect(resolved).toBe(files.map((file) => `${pathToFileURL(join(real, 'dist', file)).href}\n`).join('')) - }, 60_000) + }) }) diff --git a/packages/author/tests/read.test.ts b/packages/author/tests/read.test.ts index b2f2a0ee..c79d4816 100644 --- a/packages/author/tests/read.test.ts +++ b/packages/author/tests/read.test.ts @@ -122,7 +122,7 @@ describe.concurrent('every type file', () => { for (const path of read.files) expect(new Uint8Array(await readFile(join(at, 'read', path))), path).toEqual(await entryOf(join(at, 'example.h5p'), `content/${path}`)) const again = await buildPackage(read.source, undefined, { base: join(at, 'read'), check: true }) expect(withoutIds(again.content)).toEqual(withoutIds(first.content)) - }, 60_000) + }) }) describe('a package read back', () => { diff --git a/packages/author/vitest.config.ts b/packages/author/vitest.config.ts index 41fdf0ea..06d18d84 100644 --- a/packages/author/vitest.config.ts +++ b/packages/author/vitest.config.ts @@ -34,7 +34,13 @@ export default defineConfig({ name: 'node', environment: 'node', include: ['tests/**/*.test.ts'], - exclude: [...configDefaults.exclude, 'tests/browser/**'] + exclude: [...configDefaults.exclude, 'tests/browser/**'], + // The package's limit, and no test's own (design/testing.md, #255). These tests build + // packages from the library pack and write every content type's bundle, which a CI + // runner does two to four times slower than a laptop: on vitest's 5 s a test, three + // of them failed there in three days, each after the one before it got its own number. + testTimeout: 120_000, + hookTimeout: 120_000 } }, { diff --git a/packages/verify/tests/preview.test.ts b/packages/verify/tests/preview.test.ts index 324d9c17..0fe1633b 100644 --- a/packages/verify/tests/preview.test.ts +++ b/packages/verify/tests/preview.test.ts @@ -143,7 +143,7 @@ describe('h5p-verify --preview', () => { preview.close() await browser.close() } - }, 120_000) + }) it('offers the package for download, until its server has gone', async () => { const preview = await previewPackage({ file: quiz, port: 0 }) @@ -174,7 +174,7 @@ describe('h5p-verify --preview', () => { preview.close() await browser.close() } - }, 60_000) + }) it('follows a theme switched in another tab of the same preview', async () => { const preview = await previewPackage({ file: quiz, port: 0 }) @@ -193,7 +193,7 @@ describe('h5p-verify --preview', () => { preview.close() await browser.close() } - }, 60_000) + }) it('ends by itself when no tab opens it in time', async () => { const preview = await previewPackage({ file: quiz, port: 0, openWait: 100 }) @@ -222,7 +222,7 @@ describe('h5p-verify --preview', () => { } finally { if (child.exitCode === null) child.kill() } - }, 30_000) + }) // npm runs the command as its child and passes on the signals it gets, while a terminal sends // Ctrl+C to the whole process group, so through npx the command gets SIGINT twice (#79). The @@ -248,11 +248,11 @@ describe('h5p-verify --preview', () => { } finally { if (group.exitCode === null) process.kill(-group.pid!, 'SIGKILL') } - }, 30_000) + }) it("refuses the check's own options, and --no-open without --preview", async () => { expect(await run(quiz, '--preview', '--json', '--out', dir)).toEqual({ code: 2, stderr: '--preview does not take --out, --json\n' }) expect(await run(quiz, '--no-open')).toEqual({ code: 2, stderr: '--no-open goes with --preview\n' }) expect((await run(join(dir, 'missing.h5p'), '--preview', '--no-open')).code).toBe(2) - }, 30_000) + }) }) diff --git a/packages/verify/tests/verify.test.ts b/packages/verify/tests/verify.test.ts index 9e342063..f1e1dad7 100644 --- a/packages/verify/tests/verify.test.ts +++ b/packages/verify/tests/verify.test.ts @@ -10,7 +10,7 @@ import { contentChanges, summarize, verifyPackage } from '../lib/verify.mjs' /** * The real thing end to end: a browser, the player's Service Worker, the demo's committed - * content. Each case launches a browser, so the timeouts are generous. + * content. Each case launches a browser, so the package's limit is generous (vitest.config.ts). */ const bin = resolve(import.meta.dirname, '../bin/h5p-verify.mjs') @@ -181,7 +181,7 @@ describe('h5p-verify', () => { expect(report.verdict).toBe('fail') expect(report.reasons).toEqual(['1 uncaught error while booting: Uncaught ReferenceError: H5PEditor is not defined']) expect(report.screenshot).not.toBeNull() - }, 120_000) + }) it('passes a complete package, with its revision and a screenshot', async () => { const out = join(dir, 'quiz') @@ -201,7 +201,7 @@ describe('h5p-verify', () => { expect(report.screenshot).toBe(join(out, 'screenshot.png')) expect(existsSync(report.screenshot!)).toBe(true) expect(summarize(report)).toMatch(/^pass {2}quiz\.h5p {2}ready in \d+\.\d\d s {2}revision sha256:/) - }, 120_000) + }) it('fails a package without its libraries, naming them', async () => { const report = await verifyPackage({ file: await strippedQuiz(), out: join(dir, 'stripped') }) @@ -214,7 +214,7 @@ describe('h5p-verify', () => { expect(report.reasons[0]).toMatch(/^bad-archive: /) expect(report.screenshot).toBeNull() expect(summarize(report)).toContain('--libraries pack') - }, 120_000) + }) it('warns of a video that does not play, as a web page saved under its name (#218)', async () => { const page = await rewritten(video, 'page-video.h5p', async (entry, bytes, writer) => { @@ -234,7 +234,7 @@ describe('h5p-verify', () => { else await writer.add(entry.filename, new Uint8ArrayReader(bytes)) }) expect((await verifyPackage({ file: odd })).warnings).toContainEqual(expect.stringMatching(/^video (?:does not play|not checked): videos\/odd%zz\.mp4: /)) - }, 120_000) + }) it('fails a file that is not an archive', async () => { const path = join(dir, 'not-a-zip.h5p') @@ -243,7 +243,7 @@ describe('h5p-verify', () => { expect(report.verdict).toBe('fail') expect(report.errors[0]?.code).toBe('bad-archive') - }, 120_000) + }) it('played untrusted, reports what the content check removed and the notes it wrote, and still passes', async () => { const file = await quizWithContent('hostile.h5p', (params) => { @@ -262,7 +262,7 @@ describe('h5p-verify', () => { ]) ) expect(report.warnings).toContain('content check: questions[0].params.question: markup removed:
, ') - }, 120_000) + }) it('prints a report larger than a pipe holds whole to a program that reads it through one (#117)', async () => { // Each field the semantics do not name is removed and reported: a report of tens of kilobytes. @@ -274,7 +274,7 @@ describe('h5p-verify', () => { expect(Buffer.byteLength(stdout)).toBeGreaterThan(16 * 1024) expect(JSON.parse(stdout)).toEqual(JSON.parse(await readFile(join(out, 'report.json'), 'utf8'))) - }, 120_000) + }) it('played untrusted, passes the quiz unchanged', async () => { const report = await verifyPackage({ file: quiz, untrusted: true, libraries: pack }) @@ -283,7 +283,7 @@ describe('h5p-verify', () => { expect(report.untrusted).toBe(true) expect(report.contentCheck).toEqual({ notes: [], changes: [] }) expect(report.warnings).toEqual([]) - }, 120_000) + }) it('played untrusted, plays a package made for a newer minor than the pack has at the pack\'s, with a warning (#85)', async () => { const report = await verifyPackage({ file: await quizNeedingNewerLibrary(), untrusted: true, libraries: pack }) @@ -292,7 +292,7 @@ describe('h5p-verify', () => { expect(report.downgraded).toEqual([{ library: 'H5P.QuestionSet', from: '1.99', to: expect.stringMatching(/^1\.21\.\d+$/) }]) expect(report.warnings.some((warning) => warning.startsWith('H5P.QuestionSet played at 1.21.') && warning.includes('1.99'))).toBe(true) expect(summarize(report)).toContain('warning: H5P.QuestionSet played at 1.21.') - }, 120_000) + }) it('played trusted with no library source, warns of a library played at the older minor the package carries (#85)', async () => { // Nothing supplied anything, so there is no library bundle. The report took the list from it @@ -302,7 +302,7 @@ describe('h5p-verify', () => { expect(report.verdict).toBe('pass') expect(report.downgraded).toEqual([{ library: 'H5P.QuestionSet', from: '1.99', to: expect.stringMatching(/^1\.\d+\.\d+$/) }]) expect(report.warnings.some((warning) => warning.startsWith('H5P.QuestionSet played at 1.') && warning.includes('1.99'))).toBe(true) - }, 120_000) + }) it('played untrusted with --no-downgrade-libraries, refuses a package that needs its own copy of a library, and says what to do', async () => { const report = await verifyPackage({ file: await quizNeedingNewerLibrary(), untrusted: true, libraries: pack, downgradeLibraries: false }) diff --git a/packages/verify/vitest.config.ts b/packages/verify/vitest.config.ts new file mode 100644 index 00000000..68a37cce --- /dev/null +++ b/packages/verify/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from 'vitest/config' + +// The package's limit, and no test's own (design/testing.md, #255): every test here that plays a +// package starts a browser, 9 to 18 s on a CI runner, so one written without a number of its own +// would fail on vitest's 5 s there. A test names its own only to ask for more, as the three that +// play a package several ways do, or where the time is what it tests. +export default defineConfig({ + test: { + testTimeout: 120_000, + hookTimeout: 120_000 + } +}) From 9b61284db858fa2476bc25c1ca32576c3e0e2f51 Mon Sep 17 00:00:00 2001 From: Aleksandr Shevchuk Date: Fri, 20 Nov 2026 10:39:40 +0200 Subject: [PATCH 2/2] test: the author's browser project names its hook limit, and the rule says both projects' (#255) --- design/testing.md | 7 ++++--- packages/author/vitest.config.ts | 4 ++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/design/testing.md b/design/testing.md index 5aa772a9..b30dcf7d 100644 --- a/design/testing.md +++ b/design/testing.md @@ -26,9 +26,10 @@ green. limit stops a hang; it does not time a test. The author's Node tests ran on vitest's 5 s a test, and a CI runner builds packages and writes bundles two to four times slower than a laptop: three tests failed there in three days, each at 5.0 s, each after the one before it got a number of its -own. So embed, mcp, the demo, the skills project, the author package and the verifier each set -`testTimeout` and `hookTimeout` to 120 s in their config, and a test names its own only to ask for -more, or where the time is what it tests. A package whose tests run in milliseconds keeps the +own. So embed, mcp, the demo, the skills project, the author's Node project and the verifier each +set `testTimeout` and `hookTimeout` to 120 s in their config, and the author's browser project sets +both to 180 s, for a build in Chromium and a play in the verifier. A test names its own only to ask +for more, or where the time is what it tests. A package whose tests run in milliseconds keeps the default, until one of them needs a number: then the package gets its limit, not the test. The `browser` project runs in Chromium unless `H5P_TEST_BROWSER` names `firefox` or `webkit`. diff --git a/packages/author/vitest.config.ts b/packages/author/vitest.config.ts index 06d18d84..64c4d80a 100644 --- a/packages/author/vitest.config.ts +++ b/packages/author/vitest.config.ts @@ -50,7 +50,11 @@ export default defineConfig({ test: { name: 'browser', include: ['tests/browser/**/*.test.ts'], + // This project's limit, longer than the Node one: a test here builds in Chromium and + // plays the result in the verifier. Named here because a project takes nothing from + // the other. testTimeout: 180_000, + hookTimeout: 180_000, browser: { enabled: true, headless: true,