diff --git a/bin/compute-improvement.cjs b/bin/compute-improvement.cjs index 1adacad..01b683d 100644 --- a/bin/compute-improvement.cjs +++ b/bin/compute-improvement.cjs @@ -4,14 +4,16 @@ const fs = require('fs'); const path = require('path'); -const RESULTS_DIR = path.join(__dirname, '..', 'results'); +// Single source of truth for the results directory (honors NF_BENCH_RESULTS_DIR); +// duplicating the literal here meant a redirected write and a non-redirected read. +const { resultsDir } = require(path.join(__dirname, '..', 'lib', 'runner.cjs')); function loadResults() { - if (!fs.existsSync(RESULTS_DIR)) return []; - const files = fs.readdirSync(RESULTS_DIR) + if (!fs.existsSync(resultsDir())) return []; + const files = fs.readdirSync(resultsDir()) .filter(f => f.endsWith('.json') && !f.startsWith('report-')) .sort(); - return files.map(f => JSON.parse(fs.readFileSync(path.join(RESULTS_DIR, f), 'utf8'))); + return files.map(f => JSON.parse(fs.readFileSync(path.join(resultsDir(), f), 'utf8'))); } function computeImprovementScore() { @@ -90,7 +92,7 @@ function computeImprovementScore() { trends: allTrends }; - const reportPath = path.join(RESULTS_DIR, `improvement-${Date.now().toString(36)}.json`); + const reportPath = path.join(resultsDir(), `improvement-${Date.now().toString(36)}.json`); fs.writeFileSync(reportPath, JSON.stringify(report, null, 2) + '\n'); console.log(`\nSaved to ${reportPath}`); diff --git a/bin/nf-benchmark.cjs b/bin/nf-benchmark.cjs index 4239d9a..de37b78 100644 --- a/bin/nf-benchmark.cjs +++ b/bin/nf-benchmark.cjs @@ -149,9 +149,11 @@ function focusLayerFor(challenge) { return LAYER_ALIASES[layer] || layer; } -const RESULTS_DIR = path.join(__dirname, '..', 'results'); +// Single source of truth for the results directory (honors NF_BENCH_RESULTS_DIR); +// duplicating the literal here meant a redirected write and a non-redirected read. +const { resultsDir } = require(path.join(__dirname, '..', 'lib', 'runner.cjs')); const BASELINE_PATH = path.join(__dirname, '..', 'baseline.json'); -const TREND_PATH = path.join(RESULTS_DIR, 'trend.jsonl'); +const TREND_PATH = path.join(resultsDir(), 'trend.jsonl'); const FIXTURES_DIR = path.join(__dirname, '..', 'fixtures'); // Per-challenge project routing: a challenge may declare `project: ""` to @@ -203,7 +205,7 @@ function getFilterOptions() { } function appendTrend(report, results) { - if (!fs.existsSync(RESULTS_DIR)) fs.mkdirSync(RESULTS_DIR, { recursive: true }); + if (!fs.existsSync(resultsDir())) fs.mkdirSync(resultsDir(), { recursive: true }); const avgReductionScore = results .filter(r => r.score && r.score.reduction_score !== undefined) @@ -515,8 +517,8 @@ async function runBenchmark() { console.log(`\n(Track A exactness skipped: ${e.message})`); } - const reportPath = path.join(RESULTS_DIR, `report-${Date.now().toString(36)}.json`); - if (!fs.existsSync(RESULTS_DIR)) fs.mkdirSync(RESULTS_DIR, { recursive: true }); + const reportPath = path.join(resultsDir(), `report-${Date.now().toString(36)}.json`); + if (!fs.existsSync(resultsDir())) fs.mkdirSync(resultsDir(), { recursive: true }); fs.writeFileSync(reportPath, JSON.stringify({ report, results: results.map(r => ({ id: r.challenge.id, title: r.challenge.title, @@ -624,12 +626,12 @@ function validate() { } function calibrate() { - if (!fs.existsSync(RESULTS_DIR)) { + if (!fs.existsSync(resultsDir())) { console.log('No results directory found. Run benchmarks first.'); return; } - const reportFiles = fs.readdirSync(RESULTS_DIR) + const reportFiles = fs.readdirSync(resultsDir()) .filter(f => f.startsWith('report-') && f.endsWith('.json')) .sort(); @@ -643,7 +645,7 @@ function calibrate() { for (const f of reportFiles) { try { - const data = JSON.parse(fs.readFileSync(path.join(RESULTS_DIR, f), 'utf8')); + const data = JSON.parse(fs.readFileSync(path.join(resultsDir(), f), 'utf8')); const challengeResults = data.results || []; for (const r of challengeResults) { if (!r.id) continue; diff --git a/lib/challenges.cjs b/lib/challenges.cjs index 95ac565..d6b6e70 100644 --- a/lib/challenges.cjs +++ b/lib/challenges.cjs @@ -3,16 +3,23 @@ const fs = require('fs'); const path = require('path'); -const CHALLENGES_DIR = path.join(__dirname, '..', 'challenges'); +// Same hermetic contract as resultsDir() in lib/runner.cjs: NF_BENCH_CHALLENGES_DIR +// redirects, default unchanged. adversarial4's BUG 11 test used to back up the REAL +// challenges/ dir, delete it, drop a fixture tree in its place and restore it — the +// identical shared-global race that made the results/ tests fail under the parallel +// runner. Resolved per call so a test can set it after this module is loaded. +function challengesDir() { + return process.env.NF_BENCH_CHALLENGES_DIR || path.join(__dirname, '..', 'challenges'); +} function loadAllChallenges() { - const files = fs.readdirSync(CHALLENGES_DIR) + const files = fs.readdirSync(challengesDir()) .filter(f => f.endsWith('.json') && f !== 'challenge.schema.json') .sort(); const all = []; for (const f of files) { try { - const data = JSON.parse(fs.readFileSync(path.join(CHALLENGES_DIR, f), 'utf8')); + const data = JSON.parse(fs.readFileSync(path.join(challengesDir(), f), 'utf8')); all.push(...data); } catch (e) { console.warn(`Skipping corrupted challenge file: ${f} (${e.message})`); @@ -125,5 +132,5 @@ module.exports = { validateChallenge, validateAll, printSummary, - CHALLENGES_DIR + challengesDir, }; diff --git a/lib/runner.cjs b/lib/runner.cjs index c00ac93..a2765f5 100644 --- a/lib/runner.cjs +++ b/lib/runner.cjs @@ -5,7 +5,16 @@ const path = require('path'); const os = require('os'); const { execSync, spawnSync } = require('child_process'); -const RESULTS_DIR = path.join(__dirname, '..', 'results'); +// Results land in /results by default. NF_BENCH_RESULTS_DIR redirects them, +// which is what makes the suite hermetic: six tests across four files used to back up +// the REAL results/ dir, delete it, symlink a temp dir in its place, then restore. The +// node test runner runs files in parallel, so those six raced — one file's rmSync hit +// another's half-restored tree (ENOTEMPTY) or its cpSync found nothing (ENOENT), and +// results/.gitkeep was a casualty. Resolved per call, not cached, so a test can set the +// env var after this module is already loaded. +function resultsDir() { + return process.env.NF_BENCH_RESULTS_DIR || path.join(__dirname, '..', 'results'); +} const SNAPSHOT_EXTENSIONS = ['.json', '.tla']; @@ -329,24 +338,24 @@ function cleanupIsolatedRoot(tempDir) { } function saveResult(challengeId, result) { - if (!fs.existsSync(RESULTS_DIR)) fs.mkdirSync(RESULTS_DIR, { recursive: true }); + if (!fs.existsSync(resultsDir())) fs.mkdirSync(resultsDir(), { recursive: true }); const filename = `${challengeId}-${Date.now().toString(36)}.json`; fs.writeFileSync( - path.join(RESULTS_DIR, filename), + path.join(resultsDir(), filename), JSON.stringify(result, null, 2) + '\n' ); return filename; } function loadResults() { - if (!fs.existsSync(RESULTS_DIR)) return []; - const files = fs.readdirSync(RESULTS_DIR) + if (!fs.existsSync(resultsDir())) return []; + const files = fs.readdirSync(resultsDir()) .filter(f => f.endsWith('.json')) .sort(); const results = []; for (const f of files) { try { - results.push(JSON.parse(fs.readFileSync(path.join(RESULTS_DIR, f), 'utf8'))); + results.push(JSON.parse(fs.readFileSync(path.join(resultsDir(), f), 'utf8'))); } catch (e) { // Skip malformed JSON files console.warn(`Skipping malformed result file: ${f} (${e.message})`); @@ -371,5 +380,5 @@ module.exports = { readSutVersion, saveResult, loadResults, - RESULTS_DIR + resultsDir, }; diff --git a/test/adversarial.test.cjs b/test/adversarial.test.cjs index ae47546..d784289 100644 --- a/test/adversarial.test.cjs +++ b/test/adversarial.test.cjs @@ -204,25 +204,21 @@ test('--focus argument should not contain literal quote characters', () => { describe('BUG 9: loadResults crashes on malformed JSON', () => { test('loadResults should handle malformed JSON files gracefully', () => { const tmp = makeTmpDir(); - const origResultsDir = path.join(__dirname, '..', 'results'); - const backupDir = path.join(tmp, 'results_backup'); - - if (fs.existsSync(origResultsDir)) { - fs.cpSync(origResultsDir, backupDir, { recursive: true }); - fs.rmSync(origResultsDir, { recursive: true }); - } - fs.mkdirSync(origResultsDir, { recursive: true }); - fs.writeFileSync(path.join(origResultsDir, 'bad.json'), 'not valid json {{{'); - + // Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of + // backing up, deleting and symlinking the real results/ — six sites did that, and + // the parallel test runner raced them into ENOENT/ENOTEMPTY. + const tmpResultsDir = path.join(tmp, 'results-under-test'); + fs.mkdirSync(tmpResultsDir, { recursive: true }); + const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; try { + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; + fs.mkdirSync(tmpResultsDir, { recursive: true }); + fs.writeFileSync(path.join(tmpResultsDir, 'bad.json'), 'not valid json {{{'); assert.doesNotThrow(() => loadResults(), 'loadResults should not throw on malformed JSON'); } finally { - fs.rmSync(origResultsDir, { recursive: true }); - if (fs.existsSync(backupDir)) { - fs.cpSync(backupDir, origResultsDir, { recursive: true }); - fs.rmSync(backupDir, { recursive: true }); - } + if (prevResultsEnv === undefined) delete process.env.NF_BENCH_RESULTS_DIR; + else process.env.NF_BENCH_RESULTS_DIR = prevResultsEnv; } fs.rmSync(tmp, { recursive: true }); }); diff --git a/test/adversarial2.test.cjs b/test/adversarial2.test.cjs index c36536c..6d3888f 100644 --- a/test/adversarial2.test.cjs +++ b/test/adversarial2.test.cjs @@ -69,24 +69,20 @@ describe('NEW BUG 3: computeReport with negative scores', () => { describe('NEW BUG 4: loadResults with empty result files', () => { test('loadResults handles empty JSON files', () => { const tmp = makeTmpDir(); - const origResultsDir = path.join(__dirname, '..', 'results'); - const backupDir = path.join(tmp, 'results_backup'); - - if (fs.existsSync(origResultsDir)) { - fs.cpSync(origResultsDir, backupDir, { recursive: true }); - fs.rmSync(origResultsDir, { recursive: true }); - } - fs.mkdirSync(origResultsDir, { recursive: true }); - fs.writeFileSync(path.join(origResultsDir, 'empty.json'), ''); - + // Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of + // backing up, deleting and symlinking the real results/ — six sites did that, and + // the parallel test runner raced them into ENOENT/ENOTEMPTY. + const tmpResultsDir = path.join(tmp, 'results-under-test'); + fs.mkdirSync(tmpResultsDir, { recursive: true }); + const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; try { + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; + fs.mkdirSync(tmpResultsDir, { recursive: true }); + fs.writeFileSync(path.join(tmpResultsDir, 'empty.json'), ''); assert.doesNotThrow(() => loadResults()); } finally { - fs.rmSync(origResultsDir, { recursive: true }); - if (fs.existsSync(backupDir)) { - fs.cpSync(backupDir, origResultsDir, { recursive: true }); - fs.rmSync(backupDir, { recursive: true }); - } + if (prevResultsEnv === undefined) delete process.env.NF_BENCH_RESULTS_DIR; + else process.env.NF_BENCH_RESULTS_DIR = prevResultsEnv; } fs.rmSync(tmp, { recursive: true }); }); diff --git a/test/adversarial3.test.cjs b/test/adversarial3.test.cjs index 6f12938..e64a982 100644 --- a/test/adversarial3.test.cjs +++ b/test/adversarial3.test.cjs @@ -45,33 +45,30 @@ describe('ADV BUG 2: parseJsonPath with deeply nested paths', () => { describe('ADV BUG 3: loadResults with many result files', () => { test('handles directory with thousands of result files', () => { const tmp = makeTmpDir(); - const resultsDir = path.join(tmp, 'results'); - fs.mkdirSync(resultsDir); + const fixtureResultsDir = path.join(tmp, 'results'); + fs.mkdirSync(fixtureResultsDir); // Create 1000 result files for (let i = 0; i < 1000; i++) { - fs.writeFileSync(path.join(resultsDir, `result-${i}.json`), JSON.stringify({ challenge: { id: `BENCH-${i}` }, score: { passed: true } })); + fs.writeFileSync(path.join(fixtureResultsDir, `result-${i}.json`), JSON.stringify({ challenge: { id: `BENCH-${i}` }, score: { passed: true } })); } - const origResultsDir = path.join(__dirname, '..', 'results'); - const backupDir = path.join(tmp, 'backup'); - if (fs.existsSync(origResultsDir)) { - fs.cpSync(origResultsDir, backupDir, { recursive: true }); - fs.rmSync(origResultsDir, { recursive: true }); - } - fs.cpSync(resultsDir, origResultsDir, { recursive: true }); - + // Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of + // backing up, deleting and symlinking the real results/ — six sites did that, and + // the parallel test runner raced them into ENOENT/ENOTEMPTY. + const tmpResultsDir = path.join(tmp, 'results-under-test'); + fs.mkdirSync(tmpResultsDir, { recursive: true }); + const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; try { + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; + fs.cpSync(fixtureResultsDir, tmpResultsDir, { recursive: true }); assert.doesNotThrow(() => { const results = loadResults(); assert.strictEqual(results.length, 1000); }); } finally { - fs.rmSync(origResultsDir, { recursive: true }); - if (fs.existsSync(backupDir)) { - fs.cpSync(backupDir, origResultsDir, { recursive: true }); - fs.rmSync(backupDir, { recursive: true }); - } + if (prevResultsEnv === undefined) delete process.env.NF_BENCH_RESULTS_DIR; + else process.env.NF_BENCH_RESULTS_DIR = prevResultsEnv; fs.rmSync(tmp, { recursive: true }); } }); @@ -126,27 +123,23 @@ describe('ADV BUG 6: runSolve with very long timeout', () => { describe('ADV BUG 7: saveResult with special characters in challenge ID', () => { test('handles special characters in result filenames', () => { const tmp = makeTmpDir(); - const resultsDir = path.join(tmp, 'results'); - fs.mkdirSync(resultsDir); - - const origResultsDir = path.join(__dirname, '..', 'results'); - const backupDir = path.join(tmp, 'backup'); - if (fs.existsSync(origResultsDir)) { - fs.cpSync(origResultsDir, backupDir, { recursive: true }); - fs.rmSync(origResultsDir, { recursive: true }); - } - fs.symlinkSync(resultsDir, origResultsDir); - + const fixtureResultsDir = path.join(tmp, 'results'); + fs.mkdirSync(fixtureResultsDir); + + // Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of + // backing up, deleting and symlinking the real results/ — six sites did that, and + // the parallel test runner raced them into ENOENT/ENOTEMPTY. + const tmpResultsDir = path.join(tmp, 'results-under-test'); + fs.mkdirSync(tmpResultsDir, { recursive: true }); + const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; try { + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; assert.doesNotThrow(() => { saveResult('BENCH-001!@#$%^&*()', { test: 'data' }); }); } finally { - fs.rmSync(origResultsDir); - if (fs.existsSync(backupDir)) { - fs.cpSync(backupDir, origResultsDir, { recursive: true }); - fs.rmSync(backupDir, { recursive: true }); - } + if (prevResultsEnv === undefined) delete process.env.NF_BENCH_RESULTS_DIR; + else process.env.NF_BENCH_RESULTS_DIR = prevResultsEnv; fs.rmSync(tmp, { recursive: true }); } }); diff --git a/test/adversarial4.test.cjs b/test/adversarial4.test.cjs index 93e04de..ed8cbee 100644 --- a/test/adversarial4.test.cjs +++ b/test/adversarial4.test.cjs @@ -60,31 +60,28 @@ describe('EXTREME BUG 2: parseJsonPath with Unicode characters', () => { describe('EXTREME BUG 3: loadResults with corrupted binary data', () => { test('handles result files containing binary data', () => { const tmp = makeTmpDir(); - const resultsDir = path.join(tmp, 'results'); - fs.mkdirSync(resultsDir); + const fixtureResultsDir = path.join(tmp, 'results'); + fs.mkdirSync(fixtureResultsDir); // Create files with binary data - fs.writeFileSync(path.join(resultsDir, 'binary1.json'), Buffer.from([0x00, 0x01, 0x02, 0xFF])); - fs.writeFileSync(path.join(resultsDir, 'binary2.json'), Buffer.from('not json at all \x00\x01\x02')); - - const origResultsDir = path.join(__dirname, '..', 'results'); - const backupDir = path.join(tmp, 'backup'); - if (fs.existsSync(origResultsDir)) { - fs.cpSync(origResultsDir, backupDir, { recursive: true }); - fs.rmSync(origResultsDir, { recursive: true }); - } - fs.cpSync(resultsDir, origResultsDir, { recursive: true }); - + fs.writeFileSync(path.join(fixtureResultsDir, 'binary1.json'), Buffer.from([0x00, 0x01, 0x02, 0xFF])); + fs.writeFileSync(path.join(fixtureResultsDir, 'binary2.json'), Buffer.from('not json at all \x00\x01\x02')); + + // Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of + // backing up, deleting and symlinking the real results/ — six sites did that, and + // the parallel test runner raced them into ENOENT/ENOTEMPTY. + const tmpResultsDir = path.join(tmp, 'results-under-test'); + fs.mkdirSync(tmpResultsDir, { recursive: true }); + const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; try { + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; + fs.cpSync(fixtureResultsDir, tmpResultsDir, { recursive: true }); assert.doesNotThrow(() => { loadResults(); }); } finally { - fs.rmSync(origResultsDir, { recursive: true }); - if (fs.existsSync(backupDir)) { - fs.cpSync(backupDir, origResultsDir, { recursive: true }); - fs.rmSync(backupDir, { recursive: true }); - } + if (prevResultsEnv === undefined) delete process.env.NF_BENCH_RESULTS_DIR; + else process.env.NF_BENCH_RESULTS_DIR = prevResultsEnv; fs.rmSync(tmp, { recursive: true }); } }); @@ -213,18 +210,17 @@ describe('EXTREME BUG 9: runSolve with malformed environment', () => { describe('EXTREME BUG 10: saveResult with concurrent writes', () => { test('handles concurrent saveResult calls', async () => { const tmp = makeTmpDir(); - const resultsDir = path.join(tmp, 'results'); - fs.mkdirSync(resultsDir); - - const origResultsDir = path.join(__dirname, '..', 'results'); - const backupDir = path.join(tmp, 'backup'); - if (fs.existsSync(origResultsDir)) { - fs.cpSync(origResultsDir, backupDir, { recursive: true }); - fs.rmSync(origResultsDir, { recursive: true }); - } - fs.symlinkSync(resultsDir, origResultsDir); - + const fixtureResultsDir = path.join(tmp, 'results'); + fs.mkdirSync(fixtureResultsDir); + + // Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of + // backing up, deleting and symlinking the real results/ — six sites did that, and + // the parallel test runner raced them into ENOENT/ENOTEMPTY. + const tmpResultsDir = path.join(tmp, 'results-under-test'); + fs.mkdirSync(tmpResultsDir, { recursive: true }); + const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR; try { + process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir; const saves = []; for (let i = 0; i < 100; i++) { saves.push(saveResult(`BENCH-${i}`, { test: `data${i}` })); @@ -233,11 +229,8 @@ describe('EXTREME BUG 10: saveResult with concurrent writes', () => { // All saves should complete without issues }); } finally { - fs.rmSync(origResultsDir); - if (fs.existsSync(backupDir)) { - fs.cpSync(backupDir, origResultsDir, { recursive: true }); - fs.rmSync(backupDir, { recursive: true }); - } + if (prevResultsEnv === undefined) delete process.env.NF_BENCH_RESULTS_DIR; + else process.env.NF_BENCH_RESULTS_DIR = prevResultsEnv; fs.rmSync(tmp, { recursive: true }); } }); @@ -257,13 +250,10 @@ describe('EXTREME BUG 11: loadAllChallenges with filesystem corruption', () => { fs.writeFileSync(path.join(challengesDir, '03-empty.json'), ''); fs.writeFileSync(path.join(challengesDir, '04-binary.json'), Buffer.from([0x00, 0x01, 0x02])); - const origChallengesDir = path.join(__dirname, '..', 'challenges'); - const backupDir = path.join(tmp, 'backup'); - if (fs.existsSync(origChallengesDir)) { - fs.cpSync(origChallengesDir, backupDir, { recursive: true }); - fs.rmSync(origChallengesDir, { recursive: true }); - } - fs.cpSync(challengesDir, origChallengesDir, { recursive: true }); + // Hermetic, same as the results/ sites: redirect via NF_BENCH_CHALLENGES_DIR + // instead of backing up, deleting and repopulating the real challenges/ dir. + const prevChallengesEnv = process.env.NF_BENCH_CHALLENGES_DIR; + process.env.NF_BENCH_CHALLENGES_DIR = challengesDir; try { assert.doesNotThrow(() => { @@ -272,11 +262,8 @@ describe('EXTREME BUG 11: loadAllChallenges with filesystem corruption', () => { assert.ok(challenges.length >= 1); }); } finally { - fs.rmSync(origChallengesDir, { recursive: true }); - if (fs.existsSync(backupDir)) { - fs.cpSync(backupDir, origChallengesDir, { recursive: true }); - fs.rmSync(backupDir, { recursive: true }); - } + if (prevChallengesEnv === undefined) delete process.env.NF_BENCH_CHALLENGES_DIR; + else process.env.NF_BENCH_CHALLENGES_DIR = prevChallengesEnv; fs.rmSync(tmp, { recursive: true }); } });