Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions src/verify/fingerprintStability.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,26 @@ describe('normalizeFailureOutput', () => {
expect(a).not.toBe(normalizeFailureOutput(out('1.93').replace('1 error', '2 errors'), paths(SANDBOX_A)));
});

it('normalizes a pytest summary that carries a (H:MM:SS) suffix (AGT-4680)', () => {
// pytest -q adds `(0:02:04)` once a run passes a minute, which every full cgf-portal
// suite does. The seconds were normalised but the suffix was not, so two runs of the
// SAME failure on main hashed differently and read as a new regression.
const out = (summary: string) => [
'=========================== short test summary info ============================',
'FAILED tests/test_b5_repository.py::test_the_empty_result_has_the_same_shape_as_a_real_one',
'!!!!!!!!!!!!!!!!!!!!!!!!!! stopping after 1 failures !!!!!!!!!!!!!!!!!!!!!!!!!!!',
summary,
'',
].join('\n');
const a = normalizeFailureOutput(out('1 failed, 4234 passed, 581 skipped in 124.27s (0:02:04)'), paths(SANDBOX_A));
expect(a).toBe(normalizeFailureOutput(out('1 failed, 4234 passed, 581 skipped in 113.47s (0:01:53)'), paths(SANDBOX_B)));
expect(a).toContain('1 failed, 4234 passed, 581 skipped in <DURATION>');
// A different count is still a different failure.
expect(a).not.toBe(normalizeFailureOutput(
out('2 failed, 4233 passed, 581 skipped in 124.27s (0:02:04)'), paths(SANDBOX_A),
));
});

it('normalizes paths outside the command cwd — the subdirectory-cwd bug', () => {
// A command with `cwd: packages/api` fails, but the traceback points at a
// sibling package. Normalizing only the cwd left this line sandbox-specific.
Expand Down
5 changes: 3 additions & 2 deletions src/verify/runner.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
import { execFile, spawn, spawnSync } from 'node:child_process';
import { createHash, randomUUID } from 'node:crypto';
import { constants } from 'node:fs';
Expand Down Expand Up @@ -233,8 +233,9 @@
// carries no `=` decoration: `3 skipped, 1 error in 1.54s`. Left alone,
// base and head fingerprints differed by timing alone, and every
// pre-existing failure read as a new regression (vega-agent AGT-4118:
// the same ModuleNotFoundError on both sides, 1.93s vs 1.54s).
.replace(/^(\d+ [a-z]+(?:, \d+ [a-z]+)* in )\d+(?:\.\d+)?s$/gm, '$1<DURATION>')
// the same ModuleNotFoundError on both sides, 1.93s vs 1.54s). A run past a minute adds
// ` (0:02:04)` after the seconds, so that suffix is optional too (AGT-4680).
.replace(/^(\d+ [a-z]+(?:, \d+ [a-z]+)* in )\d+(?:\.\d+)?s(?: \(\d+:\d{2}:\d{2}\))?$/gm, '$1<DURATION>')
.replace(/(Ran \d+ tests? in )\d+(?:\.\d+)?s/g, '$1<DURATION>')
.replace(/(finished in )\d+(?:\.\d+)?s/gi, '$1<DURATION>')
// pytest-xdist assigns the same failure to different workers on base and
Expand Down
Loading