From 7ba77c48a582763afe994fea0aa6af52b9aa0ba7 Mon Sep 17 00:00:00 2001 From: "open-swe[bot]" Date: Fri, 7 Aug 2026 08:44:44 -0400 Subject: [PATCH 1/3] fix(test): make the results/ tests hermetic so `npm test` stops failing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `npm test` failed on every run — exit 1, reproducible on a clean main — with ENOTEMPTY/ENOENT against the repo's real results/ directory, and it deleted results/.gitkeep as a side effect. Cause: six tests across four adversarial files each backed up the REAL results/ dir, deleted it, put a temp dir (or symlink) in its place, then restored it in a finally block. The node test runner runs FILES IN PARALLEL, so those six raced on one shared global — file A's rmSync landed on file B's half-restored tree (ENOTEMPTY), or B's cpSync ran after A had already deleted it (ENOENT). Which test reported the error was luck, which is why it read as flake rather than as a design problem. lib/runner.cjs now resolves the results directory through resultsDir(), honoring NF_BENCH_RESULTS_DIR and defaulting to /results exactly as before. It is resolved per call rather than cached at import, so a test can set the variable after the module is loaded. All six sites set that variable to a directory inside their own temp dir and restore the previous value in finally — nothing touches the shared directory at all now. Verified: 3/3 consecutive full `npm test` runs pass (exit 0) with the fix; 2/2 fail (exit 1) with it stashed on the same tree. results/.gitkeep survives. Note for later: BUG 11 in adversarial4 does the same backup/delete/restore dance on challenges/ — same class, same shared global, not yet triggered. Co-Authored-By: Claude Opus 5 (1M context) --- lib/runner.cjs | 23 +++++++++++++------- test/adversarial.test.cjs | 25 ++++++++++------------ test/adversarial2.test.cjs | 25 ++++++++++------------ test/adversarial3.test.cjs | 43 +++++++++++++++++--------------------- test/adversarial4.test.cjs | 43 +++++++++++++++++--------------------- 5 files changed, 76 insertions(+), 83 deletions(-) diff --git a/lib/runner.cjs b/lib/runner.cjs index c00ac93..a2765f5 100644 --- a/lib/runner.cjs +++ b/lib/runner.cjs @@ -5,7 +5,16 @@ const path = require('path'); const os = require('os'); const { execSync, spawnSync } = require('child_process'); -const RESULTS_DIR = path.join(__dirname, '..', 'results'); +// Results land in /results by default. NF_BENCH_RESULTS_DIR redirects them, +// which is what makes the suite hermetic: six tests across four files used to back up +// the REAL results/ dir, delete it, symlink a temp dir in its place, then restore. The +// node test runner runs files in parallel, so those six raced — one file's rmSync hit +// another's half-restored tree (ENOTEMPTY) or its cpSync found nothing (ENOENT), and +// results/.gitkeep was a casualty. Resolved per call, not cached, so a test can set the +// env var after this module is already loaded. +function resultsDir() { + return process.env.NF_BENCH_RESULTS_DIR || path.join(__dirname, '..', 'results'); +} const SNAPSHOT_EXTENSIONS = ['.json', '.tla']; @@ -329,24 +338,24 @@ function cleanupIsolatedRoot(tempDir) { } function saveResult(challengeId, result) { - if (!fs.existsSync(RESULTS_DIR)) fs.mkdirSync(RESULTS_DIR, { recursive: true }); + if (!fs.existsSync(resultsDir())) fs.mkdirSync(resultsDir(), { recursive: true }); const filename = `${challengeId}-${Date.now().toString(36)}.json`; fs.writeFileSync( - path.join(RESULTS_DIR, filename), + path.join(resultsDir(), filename), JSON.stringify(result, null, 2) + '\n' ); return filename; } function loadResults() { - if (!fs.existsSync(RESULTS_DIR)) return []; - const files = fs.readdirSync(RESULTS_DIR) + if (!fs.existsSync(resultsDir())) return []; + const files = fs.readdirSync(resultsDir()) .filter(f => f.endsWith('.json')) .sort(); const results = []; for (const f of files) { try { - results.push(JSON.parse(fs.readFileSync(path.join(RESULTS_DIR, f), 'utf8'))); + results.push(JSON.parse(fs.readFileSync(path.join(resultsDir(), f), 'utf8'))); } catch (e) { // Skip malformed JSON files console.warn(`Skipping malformed result file: ${f} (${e.message})`); @@ -371,5 +380,5 @@ module.exports = { readSutVersion, saveResult, loadResults, - RESULTS_DIR + resultsDir, }; diff --git a/test/adversarial.test.cjs b/test/adversarial.test.cjs index ae47546..604c764 100644 --- a/test/adversarial.test.cjs +++ b/test/adversarial.test.cjs @@ -204,25 +204,22 @@ test('--focus argument should not contain literal quote characters', () => { describe('BUG 9: loadResults crashes on malformed JSON', () => { test('loadResults should handle malformed JSON files gracefully', () => { const tmp = makeTmpDir(); - const origResultsDir = path.join(__dirname, '..', 'results'); - const backupDir = path.join(tmp, 'results_backup'); - - if (fs.existsSync(origResultsDir)) { - fs.cpSync(origResultsDir, backupDir, { recursive: true }); - fs.rmSync(origResultsDir, { recursive: true }); - } - fs.mkdirSync(origResultsDir, { recursive: true }); - fs.writeFileSync(path.join(origResultsDir, 'bad.json'), 'not valid json {{{'); + // Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of + // backing up, deleting and symlinking the real results/ — six sites did that, and + // the parallel test runner raced them into ENOENT/ENOTEMPTY. + const tmpResultsDir = path.join(tmp, 'results-under-test'); + fs.mkdirSync(tmpResultsDir, { recursive: true }); + const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; + fs.mkdirSync(tmpResultsDir, { recursive: true }); + fs.writeFileSync(path.join(tmpResultsDir, 'bad.json'), 'not valid json {{{'); try { assert.doesNotThrow(() => loadResults(), 'loadResults should not throw on malformed JSON'); } finally { - fs.rmSync(origResultsDir, { recursive: true }); - if (fs.existsSync(backupDir)) { - fs.cpSync(backupDir, origResultsDir, { recursive: true }); - fs.rmSync(backupDir, { recursive: true }); - } + if (prevResultsEnv === undefined) delete process.env.NF_BENCH_RESULTS_DIR; + else process.env.NF_BENCH_RESULTS_DIR = prevResultsEnv; } fs.rmSync(tmp, { recursive: true }); }); diff --git a/test/adversarial2.test.cjs b/test/adversarial2.test.cjs index c36536c..a26c6ae 100644 --- a/test/adversarial2.test.cjs +++ b/test/adversarial2.test.cjs @@ -69,24 +69,21 @@ describe('NEW BUG 3: computeReport with negative scores', () => { describe('NEW BUG 4: loadResults with empty result files', () => { test('loadResults handles empty JSON files', () => { const tmp = makeTmpDir(); - const origResultsDir = path.join(__dirname, '..', 'results'); - const backupDir = path.join(tmp, 'results_backup'); - - if (fs.existsSync(origResultsDir)) { - fs.cpSync(origResultsDir, backupDir, { recursive: true }); - fs.rmSync(origResultsDir, { recursive: true }); - } - fs.mkdirSync(origResultsDir, { recursive: true }); - fs.writeFileSync(path.join(origResultsDir, 'empty.json'), ''); + // Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of + // backing up, deleting and symlinking the real results/ — six sites did that, and + // the parallel test runner raced them into ENOENT/ENOTEMPTY. + const tmpResultsDir = path.join(tmp, 'results-under-test'); + fs.mkdirSync(tmpResultsDir, { recursive: true }); + const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; + fs.mkdirSync(tmpResultsDir, { recursive: true }); + fs.writeFileSync(path.join(tmpResultsDir, 'empty.json'), ''); try { assert.doesNotThrow(() => loadResults()); } finally { - fs.rmSync(origResultsDir, { recursive: true }); - if (fs.existsSync(backupDir)) { - fs.cpSync(backupDir, origResultsDir, { recursive: true }); - fs.rmSync(backupDir, { recursive: true }); - } + if (prevResultsEnv === undefined) delete process.env.NF_BENCH_RESULTS_DIR; + else process.env.NF_BENCH_RESULTS_DIR = prevResultsEnv; } fs.rmSync(tmp, { recursive: true }); }); diff --git a/test/adversarial3.test.cjs b/test/adversarial3.test.cjs index 6f12938..44b3c97 100644 --- a/test/adversarial3.test.cjs +++ b/test/adversarial3.test.cjs @@ -53,13 +53,14 @@ describe('ADV BUG 3: loadResults with many result files', () => { fs.writeFileSync(path.join(resultsDir, `result-${i}.json`), JSON.stringify({ challenge: { id: `BENCH-${i}` }, score: { passed: true } })); } - const origResultsDir = path.join(__dirname, '..', 'results'); - const backupDir = path.join(tmp, 'backup'); - if (fs.existsSync(origResultsDir)) { - fs.cpSync(origResultsDir, backupDir, { recursive: true }); - fs.rmSync(origResultsDir, { recursive: true }); - } - fs.cpSync(resultsDir, origResultsDir, { recursive: true }); + // Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of + // backing up, deleting and symlinking the real results/ — six sites did that, and + // the parallel test runner raced them into ENOENT/ENOTEMPTY. + const tmpResultsDir = path.join(tmp, 'results-under-test'); + fs.mkdirSync(tmpResultsDir, { recursive: true }); + const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; + fs.cpSync(resultsDir, tmpResultsDir, { recursive: true }); try { assert.doesNotThrow(() => { @@ -67,11 +68,8 @@ describe('ADV BUG 3: loadResults with many result files', () => { assert.strictEqual(results.length, 1000); }); } finally { - fs.rmSync(origResultsDir, { recursive: true }); - if (fs.existsSync(backupDir)) { - fs.cpSync(backupDir, origResultsDir, { recursive: true }); - fs.rmSync(backupDir, { recursive: true }); - } + if (prevResultsEnv === undefined) delete process.env.NF_BENCH_RESULTS_DIR; + else process.env.NF_BENCH_RESULTS_DIR = prevResultsEnv; fs.rmSync(tmp, { recursive: true }); } }); @@ -129,24 +127,21 @@ describe('ADV BUG 7: saveResult with special characters in challenge ID', () => const resultsDir = path.join(tmp, 'results'); fs.mkdirSync(resultsDir); - const origResultsDir = path.join(__dirname, '..', 'results'); - const backupDir = path.join(tmp, 'backup'); - if (fs.existsSync(origResultsDir)) { - fs.cpSync(origResultsDir, backupDir, { recursive: true }); - fs.rmSync(origResultsDir, { recursive: true }); - } - fs.symlinkSync(resultsDir, origResultsDir); + // Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of + // backing up, deleting and symlinking the real results/ — six sites did that, and + // the parallel test runner raced them into ENOENT/ENOTEMPTY. + const tmpResultsDir = path.join(tmp, 'results-under-test'); + fs.mkdirSync(tmpResultsDir, { recursive: true }); + const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; try { assert.doesNotThrow(() => { saveResult('BENCH-001!@#$%^&*()', { test: 'data' }); }); } finally { - fs.rmSync(origResultsDir); - if (fs.existsSync(backupDir)) { - fs.cpSync(backupDir, origResultsDir, { recursive: true }); - fs.rmSync(backupDir, { recursive: true }); - } + if (prevResultsEnv === undefined) delete process.env.NF_BENCH_RESULTS_DIR; + else process.env.NF_BENCH_RESULTS_DIR = prevResultsEnv; fs.rmSync(tmp, { recursive: true }); } }); diff --git a/test/adversarial4.test.cjs b/test/adversarial4.test.cjs index 93e04de..edbb632 100644 --- a/test/adversarial4.test.cjs +++ b/test/adversarial4.test.cjs @@ -67,24 +67,22 @@ describe('EXTREME BUG 3: loadResults with corrupted binary data', () => { fs.writeFileSync(path.join(resultsDir, 'binary1.json'), Buffer.from([0x00, 0x01, 0x02, 0xFF])); fs.writeFileSync(path.join(resultsDir, 'binary2.json'), Buffer.from('not json at all \x00\x01\x02')); - const origResultsDir = path.join(__dirname, '..', 'results'); - const backupDir = path.join(tmp, 'backup'); - if (fs.existsSync(origResultsDir)) { - fs.cpSync(origResultsDir, backupDir, { recursive: true }); - fs.rmSync(origResultsDir, { recursive: true }); - } - fs.cpSync(resultsDir, origResultsDir, { recursive: true }); + // Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of + // backing up, deleting and symlinking the real results/ — six sites did that, and + // the parallel test runner raced them into ENOENT/ENOTEMPTY. + const tmpResultsDir = path.join(tmp, 'results-under-test'); + fs.mkdirSync(tmpResultsDir, { recursive: true }); + const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; + fs.cpSync(resultsDir, tmpResultsDir, { recursive: true }); try { assert.doesNotThrow(() => { loadResults(); }); } finally { - fs.rmSync(origResultsDir, { recursive: true }); - if (fs.existsSync(backupDir)) { - fs.cpSync(backupDir, origResultsDir, { recursive: true }); - fs.rmSync(backupDir, { recursive: true }); - } + if (prevResultsEnv === undefined) delete process.env.NF_BENCH_RESULTS_DIR; + else process.env.NF_BENCH_RESULTS_DIR = prevResultsEnv; fs.rmSync(tmp, { recursive: true }); } }); @@ -216,13 +214,13 @@ describe('EXTREME BUG 10: saveResult with concurrent writes', () => { const resultsDir = path.join(tmp, 'results'); fs.mkdirSync(resultsDir); - const origResultsDir = path.join(__dirname, '..', 'results'); - const backupDir = path.join(tmp, 'backup'); - if (fs.existsSync(origResultsDir)) { - fs.cpSync(origResultsDir, backupDir, { recursive: true }); - fs.rmSync(origResultsDir, { recursive: true }); - } - fs.symlinkSync(resultsDir, origResultsDir); + // Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of + // backing up, deleting and symlinking the real results/ — six sites did that, and + // the parallel test runner raced them into ENOENT/ENOTEMPTY. + const tmpResultsDir = path.join(tmp, 'results-under-test'); + fs.mkdirSync(tmpResultsDir, { recursive: true }); + const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; try { const saves = []; @@ -233,11 +231,8 @@ describe('EXTREME BUG 10: saveResult with concurrent writes', () => { // All saves should complete without issues }); } finally { - fs.rmSync(origResultsDir); - if (fs.existsSync(backupDir)) { - fs.cpSync(backupDir, origResultsDir, { recursive: true }); - fs.rmSync(backupDir, { recursive: true }); - } + if (prevResultsEnv === undefined) delete process.env.NF_BENCH_RESULTS_DIR; + else process.env.NF_BENCH_RESULTS_DIR = prevResultsEnv; fs.rmSync(tmp, { recursive: true }); } }); From 5ce3c5cf92c0ebc5fda5a1e56258ed7a1faf21c9 Mon Sep 17 00:00:00 2001 From: "open-swe[bot]" Date: Fri, 7 Aug 2026 09:21:36 -0400 Subject: [PATCH 2/3] fix(test): make the challenges/ swap hermetic too (same class as results/) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BUG 11 in adversarial4 did the identical backup/delete/repopulate/restore dance on the REAL challenges/ directory that the six results/ sites did — the same shared global under the same parallel runner. It hadn't been unlucky yet; that is the only difference. lib/challenges.cjs now resolves through challengesDir(), honoring NF_BENCH_CHALLENGES_DIR and defaulting to /challenges as before, resolved per call rather than cached at import. The test redirects the env var at its own temp fixture tree and restores the previous value in finally. Flagged in the PR body as noted-not-fixed; fixing it now instead, since leaving a known race armed to catch someone later is worse than the ten lines it costs. 3/3 consecutive full `npm test` runs pass; results/.gitkeep survives. Co-Authored-By: Claude Opus 5 (1M context) --- lib/challenges.cjs | 15 +++++++++++---- test/adversarial4.test.cjs | 18 ++++++------------ 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/lib/challenges.cjs b/lib/challenges.cjs index 95ac565..d6b6e70 100644 --- a/lib/challenges.cjs +++ b/lib/challenges.cjs @@ -3,16 +3,23 @@ const fs = require('fs'); const path = require('path'); -const CHALLENGES_DIR = path.join(__dirname, '..', 'challenges'); +// Same hermetic contract as resultsDir() in lib/runner.cjs: NF_BENCH_CHALLENGES_DIR +// redirects, default unchanged. adversarial4's BUG 11 test used to back up the REAL +// challenges/ dir, delete it, drop a fixture tree in its place and restore it — the +// identical shared-global race that made the results/ tests fail under the parallel +// runner. Resolved per call so a test can set it after this module is loaded. +function challengesDir() { + return process.env.NF_BENCH_CHALLENGES_DIR || path.join(__dirname, '..', 'challenges'); +} function loadAllChallenges() { - const files = fs.readdirSync(CHALLENGES_DIR) + const files = fs.readdirSync(challengesDir()) .filter(f => f.endsWith('.json') && f !== 'challenge.schema.json') .sort(); const all = []; for (const f of files) { try { - const data = JSON.parse(fs.readFileSync(path.join(CHALLENGES_DIR, f), 'utf8')); + const data = JSON.parse(fs.readFileSync(path.join(challengesDir(), f), 'utf8')); all.push(...data); } catch (e) { console.warn(`Skipping corrupted challenge file: ${f} (${e.message})`); @@ -125,5 +132,5 @@ module.exports = { validateChallenge, validateAll, printSummary, - CHALLENGES_DIR + challengesDir, }; diff --git a/test/adversarial4.test.cjs b/test/adversarial4.test.cjs index edbb632..c0c701d 100644 --- a/test/adversarial4.test.cjs +++ b/test/adversarial4.test.cjs @@ -252,13 +252,10 @@ describe('EXTREME BUG 11: loadAllChallenges with filesystem corruption', () => { fs.writeFileSync(path.join(challengesDir, '03-empty.json'), ''); fs.writeFileSync(path.join(challengesDir, '04-binary.json'), Buffer.from([0x00, 0x01, 0x02])); - const origChallengesDir = path.join(__dirname, '..', 'challenges'); - const backupDir = path.join(tmp, 'backup'); - if (fs.existsSync(origChallengesDir)) { - fs.cpSync(origChallengesDir, backupDir, { recursive: true }); - fs.rmSync(origChallengesDir, { recursive: true }); - } - fs.cpSync(challengesDir, origChallengesDir, { recursive: true }); + // Hermetic, same as the results/ sites: redirect via NF_BENCH_CHALLENGES_DIR + // instead of backing up, deleting and repopulating the real challenges/ dir. + const prevChallengesEnv = process.env.NF_BENCH_CHALLENGES_DIR; + process.env.NF_BENCH_CHALLENGES_DIR = challengesDir; try { assert.doesNotThrow(() => { @@ -267,11 +264,8 @@ describe('EXTREME BUG 11: loadAllChallenges with filesystem corruption', () => { assert.ok(challenges.length >= 1); }); } finally { - fs.rmSync(origChallengesDir, { recursive: true }); - if (fs.existsSync(backupDir)) { - fs.cpSync(backupDir, origChallengesDir, { recursive: true }); - fs.rmSync(backupDir, { recursive: true }); - } + if (prevChallengesEnv === undefined) delete process.env.NF_BENCH_CHALLENGES_DIR; + else process.env.NF_BENCH_CHALLENGES_DIR = prevChallengesEnv; fs.rmSync(tmp, { recursive: true }); } }); From f8a8aef6491b142d404d34e55bf5d7f45722a880 Mon Sep 17 00:00:00 2001 From: "open-swe[bot]" Date: Fri, 7 Aug 2026 09:35:40 -0400 Subject: [PATCH 3/3] =?UTF-8?q?fix(test):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20shared=20resolver=20in=20the=20CLIs,=20env=20set=20inside=20?= =?UTF-8?q?try?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two valid findings on the hermetic-results change, plus a naming cleanup: - bin/compute-improvement.cjs and bin/nf-benchmark.cjs each carried their own `path.join(__dirname, '..', 'results')`. With NF_BENCH_RESULTS_DIR set, results were WRITTEN to the redirected dir and READ from the fixed one. Both now import resultsDir() from lib/runner.cjs, so one env var means one directory everywhere. - All six sites set NF_BENCH_RESULTS_DIR before entering `try`, so a throw during fixture setup would skip `finally` and leak the override into every later test in that file (process.env is global). The assignment and the seeding now happen inside the protected block. The third finding — that `fs.cpSync(resultsDir, ...)` passes a function and throws ERR_INVALID_ARG_TYPE — is a false positive: `resultsDir` there is a local `const resultsDir = path.join(tmp, 'results')` fixture path, and those files do not import the export (see the destructure at adversarial3:11). The suite passes, which it could not if a function were reaching cpSync. The concern about the NAME is fair though, so the locals are now `fixtureResultsDir` — nothing shadows the resolver even if a future edit imports it. Full `npm test` green across 3 consecutive runs. Co-Authored-By: Claude Opus 5 (1M context) --- bin/compute-improvement.cjs | 12 +++++++----- bin/nf-benchmark.cjs | 18 ++++++++++-------- test/adversarial.test.cjs | 7 +++---- test/adversarial2.test.cjs | 7 +++---- test/adversarial3.test.cjs | 18 ++++++++---------- test/adversarial4.test.cjs | 20 +++++++++----------- 6 files changed, 40 insertions(+), 42 deletions(-) diff --git a/bin/compute-improvement.cjs b/bin/compute-improvement.cjs index 1adacad..01b683d 100644 --- a/bin/compute-improvement.cjs +++ b/bin/compute-improvement.cjs @@ -4,14 +4,16 @@ const fs = require('fs'); const path = require('path'); -const RESULTS_DIR = path.join(__dirname, '..', 'results'); +// Single source of truth for the results directory (honors NF_BENCH_RESULTS_DIR); +// duplicating the literal here meant a redirected write and a non-redirected read. +const { resultsDir } = require(path.join(__dirname, '..', 'lib', 'runner.cjs')); function loadResults() { - if (!fs.existsSync(RESULTS_DIR)) return []; - const files = fs.readdirSync(RESULTS_DIR) + if (!fs.existsSync(resultsDir())) return []; + const files = fs.readdirSync(resultsDir()) .filter(f => f.endsWith('.json') && !f.startsWith('report-')) .sort(); - return files.map(f => JSON.parse(fs.readFileSync(path.join(RESULTS_DIR, f), 'utf8'))); + return files.map(f => JSON.parse(fs.readFileSync(path.join(resultsDir(), f), 'utf8'))); } function computeImprovementScore() { @@ -90,7 +92,7 @@ function computeImprovementScore() { trends: allTrends }; - const reportPath = path.join(RESULTS_DIR, `improvement-${Date.now().toString(36)}.json`); + const reportPath = path.join(resultsDir(), `improvement-${Date.now().toString(36)}.json`); fs.writeFileSync(reportPath, JSON.stringify(report, null, 2) + '\n'); console.log(`\nSaved to ${reportPath}`); diff --git a/bin/nf-benchmark.cjs b/bin/nf-benchmark.cjs index 4239d9a..de37b78 100644 --- a/bin/nf-benchmark.cjs +++ b/bin/nf-benchmark.cjs @@ -149,9 +149,11 @@ function focusLayerFor(challenge) { return LAYER_ALIASES[layer] || layer; } -const RESULTS_DIR = path.join(__dirname, '..', 'results'); +// Single source of truth for the results directory (honors NF_BENCH_RESULTS_DIR); +// duplicating the literal here meant a redirected write and a non-redirected read. +const { resultsDir } = require(path.join(__dirname, '..', 'lib', 'runner.cjs')); const BASELINE_PATH = path.join(__dirname, '..', 'baseline.json'); -const TREND_PATH = path.join(RESULTS_DIR, 'trend.jsonl'); +const TREND_PATH = path.join(resultsDir(), 'trend.jsonl'); const FIXTURES_DIR = path.join(__dirname, '..', 'fixtures'); // Per-challenge project routing: a challenge may declare `project: ""` to @@ -203,7 +205,7 @@ function getFilterOptions() { } function appendTrend(report, results) { - if (!fs.existsSync(RESULTS_DIR)) fs.mkdirSync(RESULTS_DIR, { recursive: true }); + if (!fs.existsSync(resultsDir())) fs.mkdirSync(resultsDir(), { recursive: true }); const avgReductionScore = results .filter(r => r.score && r.score.reduction_score !== undefined) @@ -515,8 +517,8 @@ async function runBenchmark() { console.log(`\n(Track A exactness skipped: ${e.message})`); } - const reportPath = path.join(RESULTS_DIR, `report-${Date.now().toString(36)}.json`); - if (!fs.existsSync(RESULTS_DIR)) fs.mkdirSync(RESULTS_DIR, { recursive: true }); + const reportPath = path.join(resultsDir(), `report-${Date.now().toString(36)}.json`); + if (!fs.existsSync(resultsDir())) fs.mkdirSync(resultsDir(), { recursive: true }); fs.writeFileSync(reportPath, JSON.stringify({ report, results: results.map(r => ({ id: r.challenge.id, title: r.challenge.title, @@ -624,12 +626,12 @@ function validate() { } function calibrate() { - if (!fs.existsSync(RESULTS_DIR)) { + if (!fs.existsSync(resultsDir())) { console.log('No results directory found. Run benchmarks first.'); return; } - const reportFiles = fs.readdirSync(RESULTS_DIR) + const reportFiles = fs.readdirSync(resultsDir()) .filter(f => f.startsWith('report-') && f.endsWith('.json')) .sort(); @@ -643,7 +645,7 @@ function calibrate() { for (const f of reportFiles) { try { - const data = JSON.parse(fs.readFileSync(path.join(RESULTS_DIR, f), 'utf8')); + const data = JSON.parse(fs.readFileSync(path.join(resultsDir(), f), 'utf8')); const challengeResults = data.results || []; for (const r of challengeResults) { if (!r.id) continue; diff --git a/test/adversarial.test.cjs b/test/adversarial.test.cjs index 604c764..d784289 100644 --- a/test/adversarial.test.cjs +++ b/test/adversarial.test.cjs @@ -210,11 +210,10 @@ describe('BUG 9: loadResults crashes on malformed JSON', () => { const tmpResultsDir = path.join(tmp, 'results-under-test'); fs.mkdirSync(tmpResultsDir, { recursive: true }); const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; - process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; - fs.mkdirSync(tmpResultsDir, { recursive: true }); - fs.writeFileSync(path.join(tmpResultsDir, 'bad.json'), 'not valid json {{{'); - try { + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; + fs.mkdirSync(tmpResultsDir, { recursive: true }); + fs.writeFileSync(path.join(tmpResultsDir, 'bad.json'), 'not valid json {{{'); assert.doesNotThrow(() => loadResults(), 'loadResults should not throw on malformed JSON'); } finally { diff --git a/test/adversarial2.test.cjs b/test/adversarial2.test.cjs index a26c6ae..6d3888f 100644 --- a/test/adversarial2.test.cjs +++ b/test/adversarial2.test.cjs @@ -75,11 +75,10 @@ describe('NEW BUG 4: loadResults with empty result files', () => { const tmpResultsDir = path.join(tmp, 'results-under-test'); fs.mkdirSync(tmpResultsDir, { recursive: true }); const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; - process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; - fs.mkdirSync(tmpResultsDir, { recursive: true }); - fs.writeFileSync(path.join(tmpResultsDir, 'empty.json'), ''); - try { + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; + fs.mkdirSync(tmpResultsDir, { recursive: true }); + fs.writeFileSync(path.join(tmpResultsDir, 'empty.json'), ''); assert.doesNotThrow(() => loadResults()); } finally { if (prevResultsEnv === undefined) delete process.env.NF_BENCH_RESULTS_DIR; diff --git a/test/adversarial3.test.cjs b/test/adversarial3.test.cjs index 44b3c97..e64a982 100644 --- a/test/adversarial3.test.cjs +++ b/test/adversarial3.test.cjs @@ -45,12 +45,12 @@ describe('ADV BUG 2: parseJsonPath with deeply nested paths', () => { describe('ADV BUG 3: loadResults with many result files', () => { test('handles directory with thousands of result files', () => { const tmp = makeTmpDir(); - const resultsDir = path.join(tmp, 'results'); - fs.mkdirSync(resultsDir); + const fixtureResultsDir = path.join(tmp, 'results'); + fs.mkdirSync(fixtureResultsDir); // Create 1000 result files for (let i = 0; i < 1000; i++) { - fs.writeFileSync(path.join(resultsDir, `result-${i}.json`), JSON.stringify({ challenge: { id: `BENCH-${i}` }, score: { passed: true } })); + fs.writeFileSync(path.join(fixtureResultsDir, `result-${i}.json`), JSON.stringify({ challenge: { id: `BENCH-${i}` }, score: { passed: true } })); } // Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of @@ -59,10 +59,9 @@ describe('ADV BUG 3: loadResults with many result files', () => { const tmpResultsDir = path.join(tmp, 'results-under-test'); fs.mkdirSync(tmpResultsDir, { recursive: true }); const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; - process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; - fs.cpSync(resultsDir, tmpResultsDir, { recursive: true }); - try { + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; + fs.cpSync(fixtureResultsDir, tmpResultsDir, { recursive: true }); assert.doesNotThrow(() => { const results = loadResults(); assert.strictEqual(results.length, 1000); @@ -124,8 +123,8 @@ describe('ADV BUG 6: runSolve with very long timeout', () => { describe('ADV BUG 7: saveResult with special characters in challenge ID', () => { test('handles special characters in result filenames', () => { const tmp = makeTmpDir(); - const resultsDir = path.join(tmp, 'results'); - fs.mkdirSync(resultsDir); + const fixtureResultsDir = path.join(tmp, 'results'); + fs.mkdirSync(fixtureResultsDir); // Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of // backing up, deleting and symlinking the real results/ — six sites did that, and @@ -133,9 +132,8 @@ describe('ADV BUG 7: saveResult with special characters in challenge ID', () => const tmpResultsDir = path.join(tmp, 'results-under-test'); fs.mkdirSync(tmpResultsDir, { recursive: true }); const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; - process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; - try { + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; assert.doesNotThrow(() => { saveResult('BENCH-001!@#$%^&*()', { test: 'data' }); }); diff --git a/test/adversarial4.test.cjs b/test/adversarial4.test.cjs index c0c701d..ed8cbee 100644 --- a/test/adversarial4.test.cjs +++ b/test/adversarial4.test.cjs @@ -60,12 +60,12 @@ describe('EXTREME BUG 2: parseJsonPath with Unicode characters', () => { describe('EXTREME BUG 3: loadResults with corrupted binary data', () => { test('handles result files containing binary data', () => { const tmp = makeTmpDir(); - const resultsDir = path.join(tmp, 'results'); - fs.mkdirSync(resultsDir); + const fixtureResultsDir = path.join(tmp, 'results'); + fs.mkdirSync(fixtureResultsDir); // Create files with binary data - fs.writeFileSync(path.join(resultsDir, 'binary1.json'), Buffer.from([0x00, 0x01, 0x02, 0xFF])); - fs.writeFileSync(path.join(resultsDir, 'binary2.json'), Buffer.from('not json at all \x00\x01\x02')); + fs.writeFileSync(path.join(fixtureResultsDir, 'binary1.json'), Buffer.from([0x00, 0x01, 0x02, 0xFF])); + fs.writeFileSync(path.join(fixtureResultsDir, 'binary2.json'), Buffer.from('not json at all \x00\x01\x02')); // Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of // backing up, deleting and symlinking the real results/ — six sites did that, and @@ -73,10 +73,9 @@ describe('EXTREME BUG 3: loadResults with corrupted binary data', () => { const tmpResultsDir = path.join(tmp, 'results-under-test'); fs.mkdirSync(tmpResultsDir, { recursive: true }); const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; - process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; - fs.cpSync(resultsDir, tmpResultsDir, { recursive: true }); - try { + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; + fs.cpSync(fixtureResultsDir, tmpResultsDir, { recursive: true }); assert.doesNotThrow(() => { loadResults(); }); @@ -211,8 +210,8 @@ describe('EXTREME BUG 9: runSolve with malformed environment', () => { describe('EXTREME BUG 10: saveResult with concurrent writes', () => { test('handles concurrent saveResult calls', async () => { const tmp = makeTmpDir(); - const resultsDir = path.join(tmp, 'results'); - fs.mkdirSync(resultsDir); + const fixtureResultsDir = path.join(tmp, 'results'); + fs.mkdirSync(fixtureResultsDir); // Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of // backing up, deleting and symlinking the real results/ — six sites did that, and @@ -220,9 +219,8 @@ describe('EXTREME BUG 10: saveResult with concurrent writes', () => { const tmpResultsDir = path.join(tmp, 'results-under-test'); fs.mkdirSync(tmpResultsDir, { recursive: true }); const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; - process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; - try { + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; const saves = []; for (let i = 0; i < 100; i++) { saves.push(saveResult(`BENCH-${i}`, { test: `data${i}` }));