From a5b5a7e0cbe8d7109bdcdb2385c0e11620c37be4 Mon Sep 17 00:00:00 2001 From: unohee Date: Sat, 3 Oct 2026 23:58:15 +0900 Subject: [PATCH] fix(verify): normalise the pytest summary duration when it carries a (H:MM:SS) suffix (AGT-4680) --- src/verify/fingerprintStability.test.ts | 20 ++++++++++++++++++++ src/verify/runner.ts | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/verify/fingerprintStability.test.ts b/src/verify/fingerprintStability.test.ts index ef81c70d..84028927 100644 --- a/src/verify/fingerprintStability.test.ts +++ b/src/verify/fingerprintStability.test.ts @@ -67,6 +67,26 @@ describe('normalizeFailureOutput', () => { expect(a).not.toBe(normalizeFailureOutput(out('1.93').replace('1 error', '2 errors'), paths(SANDBOX_A))); }); + it('normalizes a pytest summary that carries a (H:MM:SS) suffix (AGT-4680)', () => { + // pytest -q adds `(0:02:04)` once a run passes a minute, which every full cgf-portal + // suite does. The seconds were normalised but the suffix was not, so two runs of the + // SAME failure on main hashed differently and read as a new regression. + const out = (summary: string) => [ + '=========================== short test summary info ============================', + 'FAILED tests/test_b5_repository.py::test_the_empty_result_has_the_same_shape_as_a_real_one', + '!!!!!!!!!!!!!!!!!!!!!!!!!! stopping after 1 failures !!!!!!!!!!!!!!!!!!!!!!!!!!!', + summary, + '', + ].join('\n'); + const a = normalizeFailureOutput(out('1 failed, 4234 passed, 581 skipped in 124.27s (0:02:04)'), paths(SANDBOX_A)); + expect(a).toBe(normalizeFailureOutput(out('1 failed, 4234 passed, 581 skipped in 113.47s (0:01:53)'), paths(SANDBOX_B))); + expect(a).toContain('1 failed, 4234 passed, 581 skipped in '); + // A different count is still a different failure. + expect(a).not.toBe(normalizeFailureOutput( + out('2 failed, 4233 passed, 581 skipped in 124.27s (0:02:04)'), paths(SANDBOX_A), + )); + }); + it('normalizes paths outside the command cwd — the subdirectory-cwd bug', () => { // A command with `cwd: packages/api` fails, but the traceback points at a // sibling package. Normalizing only the cwd left this line sandbox-specific. diff --git a/src/verify/runner.ts b/src/verify/runner.ts index 60d53b99..632a6d90 100644 --- a/src/verify/runner.ts +++ b/src/verify/runner.ts @@ -233,8 +233,9 @@ export function normalizeFailureOutput(output: string, paths: Array<[string, str // carries no `=` decoration: `3 skipped, 1 error in 1.54s`. Left alone, // base and head fingerprints differed by timing alone, and every // pre-existing failure read as a new regression (vega-agent AGT-4118: - // the same ModuleNotFoundError on both sides, 1.93s vs 1.54s). - .replace(/^(\d+ [a-z]+(?:, \d+ [a-z]+)* in )\d+(?:\.\d+)?s$/gm, '$1') + // the same ModuleNotFoundError on both sides, 1.93s vs 1.54s). A run past a minute adds + // ` (0:02:04)` after the seconds, so that suffix is optional too (AGT-4680). + .replace(/^(\d+ [a-z]+(?:, \d+ [a-z]+)* in )\d+(?:\.\d+)?s(?: \(\d+:\d{2}:\d{2}\))?$/gm, '$1') .replace(/(Ran \d+ tests? in )\d+(?:\.\d+)?s/g, '$1') .replace(/(finished in )\d+(?:\.\d+)?s/gi, '$1') // pytest-xdist assigns the same failure to different workers on base and