diff --git a/design/testing.md b/design/testing.md index 8455874f..b30dcf7d 100644 --- a/design/testing.md +++ b/design/testing.md @@ -22,6 +22,16 @@ which also runs when `packages/vscode` changes. The slow files are `stalling-hos publishing the player, run CI by hand on main (`gh workflow run ci.yml`) and publish once it is green. +**A test's time limit is its package's, set once in its config for a slow runner** (#255). A +limit stops a hang; it does not time a test. The author's Node tests ran on vitest's 5 s a test, +and a CI runner builds packages and writes bundles two to four times slower than a laptop: three +tests failed there in three days, each at 5.0 s, each after the one before it got a number of its +own. So embed, mcp, the demo, the skills project, the author's Node project and the verifier each +set `testTimeout` and `hookTimeout` to 120 s in their config, and the author's browser project sets +both to 180 s, for a build in Chromium and a play in the verifier. A test names its own only to ask +for more, or where the time is what it tests. A package whose tests run in milliseconds keeps the +default, until one of them needs a number: then the package gets its limit, not the test. + The `browser` project runs in Chromium unless `H5P_TEST_BROWSER` names `firefox` or `webkit`. `pnpm test:engine` runs it, then the app's tests, which is what the `engines` job does in those two. One engine per run, never a list: the dev server's switches (a host's outage, the worker version it diff --git a/packages/author/tests/build.test.ts b/packages/author/tests/build.test.ts index bea486d1..ce5ad4d6 100644 --- a/packages/author/tests/build.test.ts +++ b/packages/author/tests/build.test.ts @@ -899,8 +899,7 @@ describe('media', () => { expect(await problems(source(['a stray line', text(500, 600)], []), { base: dir })).toEqual([ expect.stringMatching(/^params\.interactiveVideo\.assets\.interactions\[0\]: expected an object/) ]) - // Several builds of a video: 1.3 s here, 2 to 3 s on CI, past vitest's 5 s once on a slow runner (#242). - }, 20_000) + }) it('gives a download up once nothing has arrived for a while, however long a slow one takes', async () => { const reported: string[] = [] @@ -1013,7 +1012,7 @@ describe('the build', () => { expect((await exec(process.execPath, [bin, '-h'])).stdout).toMatch(/^Usage:\n {2}h5p-author build/) expect((await exec(process.execPath, [bin, '-v'])).stdout).toMatch(/^@missing-elements\/h5p-author \d+\.\d+\.\d+, library pack \d+\.\d+\.\d+\n$/) await expect(exec(process.execPath, [bin, 'build', 'a.json', 'a.h5p', '--markdown'])).rejects.toMatchObject({ code: 1, stderr: 'h5p-author: build does not take --markdown\n' }) - }, 120_000) + }) }) describe('the package', () => { diff --git a/packages/author/tests/bundle.test.ts b/packages/author/tests/bundle.test.ts index 1f04a47d..1ce3c965 100644 --- a/packages/author/tests/bundle.test.ts +++ b/packages/author/tests/bundle.test.ts @@ -224,8 +224,7 @@ describe('the bundle command', () => { await expect(exec(process.execPath, [bin, 'bundle', 'H5P.TrueFalse'])).rejects.toMatchObject({ code: 1, stderr: expect.stringContaining('bundle takes a content type and an output') }) await expect(exec(process.execPath, [bin, 'bundle', '--markdown', 'H5P.TrueFalse', 'x.zip'])).rejects.toMatchObject({ code: 1, stderr: expect.stringContaining('bundle does not take --markdown') }) - // Every content type's bundle, written twice: 2.2 s here, past vitest's 5 s once on a slow runner (#242). - }, 20_000) + }) }) describe('the command line as one file', () => { @@ -255,5 +254,5 @@ describe('the command line as one file', () => { const { stdout: resolved } = await exec(process.execPath, ['--input-type=module', '-e', script], { cwd: scope }) const real = await realpath(scope) expect(resolved).toBe(files.map((file) => `${pathToFileURL(join(real, 'dist', file)).href}\n`).join('')) - }, 60_000) + }) }) diff --git a/packages/author/tests/read.test.ts b/packages/author/tests/read.test.ts index b2f2a0ee..c79d4816 100644 --- a/packages/author/tests/read.test.ts +++ b/packages/author/tests/read.test.ts @@ -122,7 +122,7 @@ describe.concurrent('every type file', () => { for (const path of read.files) expect(new Uint8Array(await readFile(join(at, 'read', path))), path).toEqual(await entryOf(join(at, 'example.h5p'), `content/${path}`)) const again = await buildPackage(read.source, undefined, { base: join(at, 'read'), check: true }) expect(withoutIds(again.content)).toEqual(withoutIds(first.content)) - }, 60_000) + }) }) describe('a package read back', () => { diff --git a/packages/author/vitest.config.ts b/packages/author/vitest.config.ts index 41fdf0ea..64c4d80a 100644 --- a/packages/author/vitest.config.ts +++ b/packages/author/vitest.config.ts @@ -34,7 +34,13 @@ export default defineConfig({ name: 'node', environment: 'node', include: ['tests/**/*.test.ts'], - exclude: [...configDefaults.exclude, 'tests/browser/**'] + exclude: [...configDefaults.exclude, 'tests/browser/**'], + // The package's limit, and no test's own (design/testing.md, #255). These tests build + // packages from the library pack and write every content type's bundle, which a CI + // runner does two to four times slower than a laptop: on vitest's 5 s a test, three + // of them failed there in three days, each after the one before it got its own number. + testTimeout: 120_000, + hookTimeout: 120_000 } }, { @@ -44,7 +50,11 @@ export default defineConfig({ test: { name: 'browser', include: ['tests/browser/**/*.test.ts'], + // This project's limit, longer than the Node one: a test here builds in Chromium and + // plays the result in the verifier. Named here because a project takes nothing from + // the other. testTimeout: 180_000, + hookTimeout: 180_000, browser: { enabled: true, headless: true, diff --git a/packages/verify/tests/preview.test.ts b/packages/verify/tests/preview.test.ts index 324d9c17..0fe1633b 100644 --- a/packages/verify/tests/preview.test.ts +++ b/packages/verify/tests/preview.test.ts @@ -143,7 +143,7 @@ describe('h5p-verify --preview', () => { preview.close() await browser.close() } - }, 120_000) + }) it('offers the package for download, until its server has gone', async () => { const preview = await previewPackage({ file: quiz, port: 0 }) @@ -174,7 +174,7 @@ describe('h5p-verify --preview', () => { preview.close() await browser.close() } - }, 60_000) + }) it('follows a theme switched in another tab of the same preview', async () => { const preview = await previewPackage({ file: quiz, port: 0 }) @@ -193,7 +193,7 @@ describe('h5p-verify --preview', () => { preview.close() await browser.close() } - }, 60_000) + }) it('ends by itself when no tab opens it in time', async () => { const preview = await previewPackage({ file: quiz, port: 0, openWait: 100 }) @@ -222,7 +222,7 @@ describe('h5p-verify --preview', () => { } finally { if (child.exitCode === null) child.kill() } - }, 30_000) + }) // npm runs the command as its child and passes on the signals it gets, while a terminal sends // Ctrl+C to the whole process group, so through npx the command gets SIGINT twice (#79). The @@ -248,11 +248,11 @@ describe('h5p-verify --preview', () => { } finally { if (group.exitCode === null) process.kill(-group.pid!, 'SIGKILL') } - }, 30_000) + }) it("refuses the check's own options, and --no-open without --preview", async () => { expect(await run(quiz, '--preview', '--json', '--out', dir)).toEqual({ code: 2, stderr: '--preview does not take --out, --json\n' }) expect(await run(quiz, '--no-open')).toEqual({ code: 2, stderr: '--no-open goes with --preview\n' }) expect((await run(join(dir, 'missing.h5p'), '--preview', '--no-open')).code).toBe(2) - }, 30_000) + }) }) diff --git a/packages/verify/tests/verify.test.ts b/packages/verify/tests/verify.test.ts index 9e342063..f1e1dad7 100644 --- a/packages/verify/tests/verify.test.ts +++ b/packages/verify/tests/verify.test.ts @@ -10,7 +10,7 @@ import { contentChanges, summarize, verifyPackage } from '../lib/verify.mjs' /** * The real thing end to end: a browser, the player's Service Worker, the demo's committed - * content. Each case launches a browser, so the timeouts are generous. + * content. Each case launches a browser, so the package's limit is generous (vitest.config.ts). */ const bin = resolve(import.meta.dirname, '../bin/h5p-verify.mjs') @@ -181,7 +181,7 @@ describe('h5p-verify', () => { expect(report.verdict).toBe('fail') expect(report.reasons).toEqual(['1 uncaught error while booting: Uncaught ReferenceError: H5PEditor is not defined']) expect(report.screenshot).not.toBeNull() - }, 120_000) + }) it('passes a complete package, with its revision and a screenshot', async () => { const out = join(dir, 'quiz') @@ -201,7 +201,7 @@ describe('h5p-verify', () => { expect(report.screenshot).toBe(join(out, 'screenshot.png')) expect(existsSync(report.screenshot!)).toBe(true) expect(summarize(report)).toMatch(/^pass {2}quiz\.h5p {2}ready in \d+\.\d\d s {2}revision sha256:/) - }, 120_000) + }) it('fails a package without its libraries, naming them', async () => { const report = await verifyPackage({ file: await strippedQuiz(), out: join(dir, 'stripped') }) @@ -214,7 +214,7 @@ describe('h5p-verify', () => { expect(report.reasons[0]).toMatch(/^bad-archive: /) expect(report.screenshot).toBeNull() expect(summarize(report)).toContain('--libraries pack') - }, 120_000) + }) it('warns of a video that does not play, as a web page saved under its name (#218)', async () => { const page = await rewritten(video, 'page-video.h5p', async (entry, bytes, writer) => { @@ -234,7 +234,7 @@ describe('h5p-verify', () => { else await writer.add(entry.filename, new Uint8ArrayReader(bytes)) }) expect((await verifyPackage({ file: odd })).warnings).toContainEqual(expect.stringMatching(/^video (?:does not play|not checked): videos\/odd%zz\.mp4: /)) - }, 120_000) + }) it('fails a file that is not an archive', async () => { const path = join(dir, 'not-a-zip.h5p') @@ -243,7 +243,7 @@ describe('h5p-verify', () => { expect(report.verdict).toBe('fail') expect(report.errors[0]?.code).toBe('bad-archive') - }, 120_000) + }) it('played untrusted, reports what the content check removed and the notes it wrote, and still passes', async () => { const file = await quizWithContent('hostile.h5p', (params) => { @@ -262,7 +262,7 @@ describe('h5p-verify', () => { ]) ) expect(report.warnings).toContain('content check: questions[0].params.question: markup removed: