Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions design/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,16 @@ which also runs when `packages/vscode` changes. The slow files are `stalling-hos
publishing the player, run CI by hand on main (`gh workflow run ci.yml`) and publish once it is
green.

**A test's time limit is its package's, set once in its config for a slow runner** (#255). A
limit stops a hang; it does not time a test. The author's Node tests ran on vitest's 5 s a test,
and a CI runner builds packages and writes bundles two to four times slower than a laptop: three
tests failed there in three days, each at 5.0 s, each after the one before it got a number of its
own. So embed, mcp, the demo, the skills project, the author's Node project and the verifier each
set `testTimeout` and `hookTimeout` to 120 s in their config, and the author's browser project sets
both to 180 s, for a build in Chromium and a play in the verifier. A test names its own only to ask
for more, or where the time is what it tests. A package whose tests run in milliseconds keeps the
default, until one of them needs a number: then the package gets its limit, not the test.

The `browser` project runs in Chromium unless `H5P_TEST_BROWSER` names `firefox` or `webkit`.
`pnpm test:engine` runs it, then the app's tests, which is what the `engines` job does in those two.
One engine per run, never a list: the dev server's switches (a host's outage, the worker version it
Expand Down
5 changes: 2 additions & 3 deletions packages/author/tests/build.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -899,8 +899,7 @@ describe('media', () => {
expect(await problems(source(['a stray line', text(500, 600)], []), { base: dir })).toEqual([
expect.stringMatching(/^params\.interactiveVideo\.assets\.interactions\[0\]: expected an object/)
])
// Several builds of a video: 1.3 s here, 2 to 3 s on CI, past vitest's 5 s once on a slow runner (#242).
}, 20_000)
})

it('gives a download up once nothing has arrived for a while, however long a slow one takes', async () => {
const reported: string[] = []
Expand Down Expand Up @@ -1013,7 +1012,7 @@ describe('the build', () => {
expect((await exec(process.execPath, [bin, '-h'])).stdout).toMatch(/^Usage:\n {2}h5p-author build/)
expect((await exec(process.execPath, [bin, '-v'])).stdout).toMatch(/^@missing-elements\/h5p-author \d+\.\d+\.\d+, library pack \d+\.\d+\.\d+\n$/)
await expect(exec(process.execPath, [bin, 'build', 'a.json', 'a.h5p', '--markdown'])).rejects.toMatchObject({ code: 1, stderr: 'h5p-author: build does not take --markdown\n' })
}, 120_000)
})
})

describe('the package', () => {
Expand Down
5 changes: 2 additions & 3 deletions packages/author/tests/bundle.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -224,8 +224,7 @@ describe('the bundle command', () => {

await expect(exec(process.execPath, [bin, 'bundle', 'H5P.TrueFalse'])).rejects.toMatchObject({ code: 1, stderr: expect.stringContaining('bundle takes a content type and an output') })
await expect(exec(process.execPath, [bin, 'bundle', '--markdown', 'H5P.TrueFalse', 'x.zip'])).rejects.toMatchObject({ code: 1, stderr: expect.stringContaining('bundle does not take --markdown') })
// Every content type's bundle, written twice: 2.2 s here, past vitest's 5 s once on a slow runner (#242).
}, 20_000)
})
})

describe('the command line as one file', () => {
Expand Down Expand Up @@ -255,5 +254,5 @@ describe('the command line as one file', () => {
const { stdout: resolved } = await exec(process.execPath, ['--input-type=module', '-e', script], { cwd: scope })
const real = await realpath(scope)
expect(resolved).toBe(files.map((file) => `${pathToFileURL(join(real, 'dist', file)).href}\n`).join(''))
}, 60_000)
})
})
2 changes: 1 addition & 1 deletion packages/author/tests/read.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ describe.concurrent('every type file', () => {
for (const path of read.files) expect(new Uint8Array(await readFile(join(at, 'read', path))), path).toEqual(await entryOf(join(at, 'example.h5p'), `content/${path}`))
const again = await buildPackage(read.source, undefined, { base: join(at, 'read'), check: true })
expect(withoutIds(again.content)).toEqual(withoutIds(first.content))
}, 60_000)
})
})

describe('a package read back', () => {
Expand Down
12 changes: 11 additions & 1 deletion packages/author/vitest.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,13 @@ export default defineConfig({
name: 'node',
environment: 'node',
include: ['tests/**/*.test.ts'],
exclude: [...configDefaults.exclude, 'tests/browser/**']
exclude: [...configDefaults.exclude, 'tests/browser/**'],
// The package's limit, and no test's own (design/testing.md, #255). These tests build
// packages from the library pack and write every content type's bundle, which a CI
// runner does two to four times slower than a laptop: on vitest's 5 s a test, three
// of them failed there in three days, each after the one before it got its own number.
testTimeout: 120_000,
hookTimeout: 120_000
Comment thread
alekswebnet marked this conversation as resolved.
Resolved
}
},
{
Expand All @@ -44,7 +50,11 @@ export default defineConfig({
test: {
name: 'browser',
include: ['tests/browser/**/*.test.ts'],
// This project's limit, longer than the Node one: a test here builds in Chromium and
// plays the result in the verifier. Named here because a project takes nothing from
// the other.
testTimeout: 180_000,
hookTimeout: 180_000,
browser: {
enabled: true,
headless: true,
Expand Down
12 changes: 6 additions & 6 deletions packages/verify/tests/preview.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ describe('h5p-verify --preview', () => {
preview.close()
await browser.close()
}
}, 120_000)
})

it('offers the package for download, until its server has gone', async () => {
const preview = await previewPackage({ file: quiz, port: 0 })
Expand Down Expand Up @@ -174,7 +174,7 @@ describe('h5p-verify --preview', () => {
preview.close()
await browser.close()
}
}, 60_000)
})

it('follows a theme switched in another tab of the same preview', async () => {
const preview = await previewPackage({ file: quiz, port: 0 })
Expand All @@ -193,7 +193,7 @@ describe('h5p-verify --preview', () => {
preview.close()
await browser.close()
}
}, 60_000)
})

it('ends by itself when no tab opens it in time', async () => {
const preview = await previewPackage({ file: quiz, port: 0, openWait: 100 })
Expand Down Expand Up @@ -222,7 +222,7 @@ describe('h5p-verify --preview', () => {
} finally {
if (child.exitCode === null) child.kill()
}
}, 30_000)
})

// npm runs the command as its child and passes on the signals it gets, while a terminal sends
// Ctrl+C to the whole process group, so through npx the command gets SIGINT twice (#79). The
Expand All @@ -248,11 +248,11 @@ describe('h5p-verify --preview', () => {
} finally {
if (group.exitCode === null) process.kill(-group.pid!, 'SIGKILL')
}
}, 30_000)
})

it("refuses the check's own options, and --no-open without --preview", async () => {
expect(await run(quiz, '--preview', '--json', '--out', dir)).toEqual({ code: 2, stderr: '--preview does not take --out, --json\n' })
expect(await run(quiz, '--no-open')).toEqual({ code: 2, stderr: '--no-open goes with --preview\n' })
expect((await run(join(dir, 'missing.h5p'), '--preview', '--no-open')).code).toBe(2)
}, 30_000)
})
})
22 changes: 11 additions & 11 deletions packages/verify/tests/verify.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import { contentChanges, summarize, verifyPackage } from '../lib/verify.mjs'

/**
* The real thing end to end: a browser, the player's Service Worker, the demo's committed
* content. Each case launches a browser, so the timeouts are generous.
* content. Each case launches a browser, so the package's limit is generous (vitest.config.ts).
*/

const bin = resolve(import.meta.dirname, '../bin/h5p-verify.mjs')
Expand Down Expand Up @@ -181,7 +181,7 @@ describe('h5p-verify', () => {
expect(report.verdict).toBe('fail')
expect(report.reasons).toEqual(['1 uncaught error while booting: Uncaught ReferenceError: H5PEditor is not defined'])
expect(report.screenshot).not.toBeNull()
}, 120_000)
})

it('passes a complete package, with its revision and a screenshot', async () => {
const out = join(dir, 'quiz')
Expand All @@ -201,7 +201,7 @@ describe('h5p-verify', () => {
expect(report.screenshot).toBe(join(out, 'screenshot.png'))
expect(existsSync(report.screenshot!)).toBe(true)
expect(summarize(report)).toMatch(/^pass {2}quiz\.h5p {2}ready in \d+\.\d\d s {2}revision sha256:/)
}, 120_000)
})

it('fails a package without its libraries, naming them', async () => {
const report = await verifyPackage({ file: await strippedQuiz(), out: join(dir, 'stripped') })
Expand All @@ -214,7 +214,7 @@ describe('h5p-verify', () => {
expect(report.reasons[0]).toMatch(/^bad-archive: /)
expect(report.screenshot).toBeNull()
expect(summarize(report)).toContain('--libraries pack')
}, 120_000)
})

it('warns of a video that does not play, as a web page saved under its name (#218)', async () => {
const page = await rewritten(video, 'page-video.h5p', async (entry, bytes, writer) => {
Expand All @@ -234,7 +234,7 @@ describe('h5p-verify', () => {
else await writer.add(entry.filename, new Uint8ArrayReader(bytes))
})
expect((await verifyPackage({ file: odd })).warnings).toContainEqual(expect.stringMatching(/^video (?:does not play|not checked): videos\/odd%zz\.mp4: /))
}, 120_000)
})

it('fails a file that is not an archive', async () => {
const path = join(dir, 'not-a-zip.h5p')
Expand All @@ -243,7 +243,7 @@ describe('h5p-verify', () => {

expect(report.verdict).toBe('fail')
expect(report.errors[0]?.code).toBe('bad-archive')
}, 120_000)
})

it('played untrusted, reports what the content check removed and the notes it wrote, and still passes', async () => {
const file = await quizWithContent('hostile.h5p', (params) => {
Expand All @@ -262,7 +262,7 @@ describe('h5p-verify', () => {
])
)
expect(report.warnings).toContain('content check: questions[0].params.question: markup removed: <form>, <input>')
}, 120_000)
})

it('prints a report larger than a pipe holds whole to a program that reads it through one (#117)', async () => {
// Each field the semantics do not name is removed and reported: a report of tens of kilobytes.
Expand All @@ -274,7 +274,7 @@ describe('h5p-verify', () => {

expect(Buffer.byteLength(stdout)).toBeGreaterThan(16 * 1024)
expect(JSON.parse(stdout)).toEqual(JSON.parse(await readFile(join(out, 'report.json'), 'utf8')))
}, 120_000)
})

it('played untrusted, passes the quiz unchanged', async () => {
const report = await verifyPackage({ file: quiz, untrusted: true, libraries: pack })
Expand All @@ -283,7 +283,7 @@ describe('h5p-verify', () => {
expect(report.untrusted).toBe(true)
expect(report.contentCheck).toEqual({ notes: [], changes: [] })
expect(report.warnings).toEqual([])
}, 120_000)
})

it('played untrusted, plays a package made for a newer minor than the pack has at the pack\'s, with a warning (#85)', async () => {
const report = await verifyPackage({ file: await quizNeedingNewerLibrary(), untrusted: true, libraries: pack })
Expand All @@ -292,7 +292,7 @@ describe('h5p-verify', () => {
expect(report.downgraded).toEqual([{ library: 'H5P.QuestionSet', from: '1.99', to: expect.stringMatching(/^1\.21\.\d+$/) }])
expect(report.warnings.some((warning) => warning.startsWith('H5P.QuestionSet played at 1.21.') && warning.includes('1.99'))).toBe(true)
expect(summarize(report)).toContain('warning: H5P.QuestionSet played at 1.21.')
}, 120_000)
})

it('played trusted with no library source, warns of a library played at the older minor the package carries (#85)', async () => {
// Nothing supplied anything, so there is no library bundle. The report took the list from it
Expand All @@ -302,7 +302,7 @@ describe('h5p-verify', () => {
expect(report.verdict).toBe('pass')
expect(report.downgraded).toEqual([{ library: 'H5P.QuestionSet', from: '1.99', to: expect.stringMatching(/^1\.\d+\.\d+$/) }])
expect(report.warnings.some((warning) => warning.startsWith('H5P.QuestionSet played at 1.') && warning.includes('1.99'))).toBe(true)
}, 120_000)
})

it('played untrusted with --no-downgrade-libraries, refuses a package that needs its own copy of a library, and says what to do', async () => {
const report = await verifyPackage({ file: await quizNeedingNewerLibrary(), untrusted: true, libraries: pack, downgradeLibraries: false })
Expand Down
12 changes: 12 additions & 0 deletions packages/verify/vitest.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
import { defineConfig } from 'vitest/config'

// The package's limit, and no test's own (design/testing.md, #255): every test here that plays a
// package starts a browser, 9 to 18 s on a CI runner, so one written without a number of its own
// would fail on vitest's 5 s there. A test names its own only to ask for more, as the three that
// play a package several ways do, or where the time is what it tests.
export default defineConfig({
test: {
testTimeout: 120_000,
hookTimeout: 120_000
}
})