Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions bin/compute-improvement.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,16 @@
const fs = require('fs');
const path = require('path');

const RESULTS_DIR = path.join(__dirname, '..', 'results');
// Single source of truth for the results directory (honors NF_BENCH_RESULTS_DIR);
// duplicating the literal here meant a redirected write and a non-redirected read.
const { resultsDir } = require(path.join(__dirname, '..', 'lib', 'runner.cjs'));

function loadResults() {
if (!fs.existsSync(RESULTS_DIR)) return [];
const files = fs.readdirSync(RESULTS_DIR)
if (!fs.existsSync(resultsDir())) return [];
const files = fs.readdirSync(resultsDir())
.filter(f => f.endsWith('.json') && !f.startsWith('report-'))
.sort();
return files.map(f => JSON.parse(fs.readFileSync(path.join(RESULTS_DIR, f), 'utf8')));
return files.map(f => JSON.parse(fs.readFileSync(path.join(resultsDir(), f), 'utf8')));
}

function computeImprovementScore() {
Expand Down Expand Up @@ -90,7 +92,7 @@ function computeImprovementScore() {
trends: allTrends
};

const reportPath = path.join(RESULTS_DIR, `improvement-${Date.now().toString(36)}.json`);
const reportPath = path.join(resultsDir(), `improvement-${Date.now().toString(36)}.json`);
fs.writeFileSync(reportPath, JSON.stringify(report, null, 2) + '\n');
console.log(`\nSaved to ${reportPath}`);

Expand Down
18 changes: 10 additions & 8 deletions bin/nf-benchmark.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -149,9 +149,11 @@ function focusLayerFor(challenge) {
return LAYER_ALIASES[layer] || layer;
}

const RESULTS_DIR = path.join(__dirname, '..', 'results');
// Single source of truth for the results directory (honors NF_BENCH_RESULTS_DIR);
// duplicating the literal here meant a redirected write and a non-redirected read.
const { resultsDir } = require(path.join(__dirname, '..', 'lib', 'runner.cjs'));
const BASELINE_PATH = path.join(__dirname, '..', 'baseline.json');
const TREND_PATH = path.join(RESULTS_DIR, 'trend.jsonl');
const TREND_PATH = path.join(resultsDir(), 'trend.jsonl');
const FIXTURES_DIR = path.join(__dirname, '..', 'fixtures');

// Per-challenge project routing: a challenge may declare `project: "<name>"` to
Expand Down Expand Up @@ -203,7 +205,7 @@ function getFilterOptions() {
}

function appendTrend(report, results) {
if (!fs.existsSync(RESULTS_DIR)) fs.mkdirSync(RESULTS_DIR, { recursive: true });
if (!fs.existsSync(resultsDir())) fs.mkdirSync(resultsDir(), { recursive: true });

const avgReductionScore = results
.filter(r => r.score && r.score.reduction_score !== undefined)
Expand Down Expand Up @@ -515,8 +517,8 @@ async function runBenchmark() {
console.log(`\n(Track A exactness skipped: ${e.message})`);
}

const reportPath = path.join(RESULTS_DIR, `report-${Date.now().toString(36)}.json`);
if (!fs.existsSync(RESULTS_DIR)) fs.mkdirSync(RESULTS_DIR, { recursive: true });
const reportPath = path.join(resultsDir(), `report-${Date.now().toString(36)}.json`);
if (!fs.existsSync(resultsDir())) fs.mkdirSync(resultsDir(), { recursive: true });
fs.writeFileSync(reportPath, JSON.stringify({ report, results: results.map(r => ({
id: r.challenge.id,
title: r.challenge.title,
Expand Down Expand Up @@ -624,12 +626,12 @@ function validate() {
}

function calibrate() {
if (!fs.existsSync(RESULTS_DIR)) {
if (!fs.existsSync(resultsDir())) {
console.log('No results directory found. Run benchmarks first.');
return;
}

const reportFiles = fs.readdirSync(RESULTS_DIR)
const reportFiles = fs.readdirSync(resultsDir())
.filter(f => f.startsWith('report-') && f.endsWith('.json'))
.sort();

Expand All @@ -643,7 +645,7 @@ function calibrate() {

for (const f of reportFiles) {
try {
const data = JSON.parse(fs.readFileSync(path.join(RESULTS_DIR, f), 'utf8'));
const data = JSON.parse(fs.readFileSync(path.join(resultsDir(), f), 'utf8'));
const challengeResults = data.results || [];
for (const r of challengeResults) {
if (!r.id) continue;
Expand Down
15 changes: 11 additions & 4 deletions lib/challenges.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,23 @@
const fs = require('fs');
const path = require('path');

const CHALLENGES_DIR = path.join(__dirname, '..', 'challenges');
// Same hermetic contract as resultsDir() in lib/runner.cjs: NF_BENCH_CHALLENGES_DIR
// redirects, default unchanged. adversarial4's BUG 11 test used to back up the REAL
// challenges/ dir, delete it, drop a fixture tree in its place and restore it — the
// identical shared-global race that made the results/ tests fail under the parallel
// runner. Resolved per call so a test can set it after this module is loaded.
function challengesDir() {
return process.env.NF_BENCH_CHALLENGES_DIR || path.join(__dirname, '..', 'challenges');
}

function loadAllChallenges() {
const files = fs.readdirSync(CHALLENGES_DIR)
const files = fs.readdirSync(challengesDir())
.filter(f => f.endsWith('.json') && f !== 'challenge.schema.json')
.sort();
const all = [];
for (const f of files) {
try {
const data = JSON.parse(fs.readFileSync(path.join(CHALLENGES_DIR, f), 'utf8'));
const data = JSON.parse(fs.readFileSync(path.join(challengesDir(), f), 'utf8'));
all.push(...data);
} catch (e) {
console.warn(`Skipping corrupted challenge file: ${f} (${e.message})`);
Expand Down Expand Up @@ -125,5 +132,5 @@ module.exports = {
validateChallenge,
validateAll,
printSummary,
CHALLENGES_DIR
challengesDir,
};
23 changes: 16 additions & 7 deletions lib/runner.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,16 @@ const path = require('path');
const os = require('os');
const { execSync, spawnSync } = require('child_process');

const RESULTS_DIR = path.join(__dirname, '..', 'results');
// Results land in <repo>/results by default. NF_BENCH_RESULTS_DIR redirects them,
// which is what makes the suite hermetic: six tests across four files used to back up
// the REAL results/ dir, delete it, symlink a temp dir in its place, then restore. The
// node test runner runs files in parallel, so those six raced — one file's rmSync hit
// another's half-restored tree (ENOTEMPTY) or its cpSync found nothing (ENOENT), and
// results/.gitkeep was a casualty. Resolved per call, not cached, so a test can set the
// env var after this module is already loaded.
function resultsDir() {
return process.env.NF_BENCH_RESULTS_DIR || path.join(__dirname, '..', 'results');
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

const SNAPSHOT_EXTENSIONS = ['.json', '.tla'];

Expand Down Expand Up @@ -329,24 +338,24 @@ function cleanupIsolatedRoot(tempDir) {
}

function saveResult(challengeId, result) {
if (!fs.existsSync(RESULTS_DIR)) fs.mkdirSync(RESULTS_DIR, { recursive: true });
if (!fs.existsSync(resultsDir())) fs.mkdirSync(resultsDir(), { recursive: true });
const filename = `${challengeId}-${Date.now().toString(36)}.json`;
fs.writeFileSync(
path.join(RESULTS_DIR, filename),
path.join(resultsDir(), filename),
JSON.stringify(result, null, 2) + '\n'
);
return filename;
}

function loadResults() {
if (!fs.existsSync(RESULTS_DIR)) return [];
const files = fs.readdirSync(RESULTS_DIR)
if (!fs.existsSync(resultsDir())) return [];
const files = fs.readdirSync(resultsDir())
.filter(f => f.endsWith('.json'))
.sort();
const results = [];
for (const f of files) {
try {
results.push(JSON.parse(fs.readFileSync(path.join(RESULTS_DIR, f), 'utf8')));
results.push(JSON.parse(fs.readFileSync(path.join(resultsDir(), f), 'utf8')));
} catch (e) {
// Skip malformed JSON files
console.warn(`Skipping malformed result file: ${f} (${e.message})`);
Expand All @@ -371,5 +380,5 @@ module.exports = {
readSutVersion,
saveResult,
loadResults,
RESULTS_DIR
resultsDir,
};
26 changes: 11 additions & 15 deletions test/adversarial.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -204,25 +204,21 @@ test('--focus argument should not contain literal quote characters', () => {
describe('BUG 9: loadResults crashes on malformed JSON', () => {
test('loadResults should handle malformed JSON files gracefully', () => {
const tmp = makeTmpDir();
const origResultsDir = path.join(__dirname, '..', 'results');
const backupDir = path.join(tmp, 'results_backup');

if (fs.existsSync(origResultsDir)) {
fs.cpSync(origResultsDir, backupDir, { recursive: true });
fs.rmSync(origResultsDir, { recursive: true });
}
fs.mkdirSync(origResultsDir, { recursive: true });
fs.writeFileSync(path.join(origResultsDir, 'bad.json'), 'not valid json {{{');

// Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of
// backing up, deleting and symlinking the real results/ — six sites did that, and
// the parallel test runner raced them into ENOENT/ENOTEMPTY.
const tmpResultsDir = path.join(tmp, 'results-under-test');
fs.mkdirSync(tmpResultsDir, { recursive: true });
const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR;
try {
process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir;
fs.mkdirSync(tmpResultsDir, { recursive: true });
fs.writeFileSync(path.join(tmpResultsDir, 'bad.json'), 'not valid json {{{');
assert.doesNotThrow(() => loadResults(),
'loadResults should not throw on malformed JSON');
} finally {
fs.rmSync(origResultsDir, { recursive: true });
if (fs.existsSync(backupDir)) {
fs.cpSync(backupDir, origResultsDir, { recursive: true });
fs.rmSync(backupDir, { recursive: true });
}
if (prevResultsEnv === undefined) delete process.env.NF_BENCH_RESULTS_DIR;
else process.env.NF_BENCH_RESULTS_DIR = prevResultsEnv;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
fs.rmSync(tmp, { recursive: true });
});
Expand Down
26 changes: 11 additions & 15 deletions test/adversarial2.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -69,24 +69,20 @@ describe('NEW BUG 3: computeReport with negative scores', () => {
describe('NEW BUG 4: loadResults with empty result files', () => {
test('loadResults handles empty JSON files', () => {
const tmp = makeTmpDir();
const origResultsDir = path.join(__dirname, '..', 'results');
const backupDir = path.join(tmp, 'results_backup');

if (fs.existsSync(origResultsDir)) {
fs.cpSync(origResultsDir, backupDir, { recursive: true });
fs.rmSync(origResultsDir, { recursive: true });
}
fs.mkdirSync(origResultsDir, { recursive: true });
fs.writeFileSync(path.join(origResultsDir, 'empty.json'), '');

// Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of
// backing up, deleting and symlinking the real results/ — six sites did that, and
// the parallel test runner raced them into ENOENT/ENOTEMPTY.
const tmpResultsDir = path.join(tmp, 'results-under-test');
fs.mkdirSync(tmpResultsDir, { recursive: true });
const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR;
try {
process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir;
fs.mkdirSync(tmpResultsDir, { recursive: true });
fs.writeFileSync(path.join(tmpResultsDir, 'empty.json'), '');
assert.doesNotThrow(() => loadResults());
} finally {
fs.rmSync(origResultsDir, { recursive: true });
if (fs.existsSync(backupDir)) {
fs.cpSync(backupDir, origResultsDir, { recursive: true });
fs.rmSync(backupDir, { recursive: true });
}
if (prevResultsEnv === undefined) delete process.env.NF_BENCH_RESULTS_DIR;
else process.env.NF_BENCH_RESULTS_DIR = prevResultsEnv;
}
fs.rmSync(tmp, { recursive: true });
});
Expand Down
57 changes: 25 additions & 32 deletions test/adversarial3.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -45,33 +45,30 @@ describe('ADV BUG 2: parseJsonPath with deeply nested paths', () => {
describe('ADV BUG 3: loadResults with many result files', () => {
test('handles directory with thousands of result files', () => {
const tmp = makeTmpDir();
const resultsDir = path.join(tmp, 'results');
fs.mkdirSync(resultsDir);
const fixtureResultsDir = path.join(tmp, 'results');
fs.mkdirSync(fixtureResultsDir);

// Create 1000 result files
for (let i = 0; i < 1000; i++) {
fs.writeFileSync(path.join(resultsDir, `result-${i}.json`), JSON.stringify({ challenge: { id: `BENCH-${i}` }, score: { passed: true } }));
fs.writeFileSync(path.join(fixtureResultsDir, `result-${i}.json`), JSON.stringify({ challenge: { id: `BENCH-${i}` }, score: { passed: true } }));
}

const origResultsDir = path.join(__dirname, '..', 'results');
const backupDir = path.join(tmp, 'backup');
if (fs.existsSync(origResultsDir)) {
fs.cpSync(origResultsDir, backupDir, { recursive: true });
fs.rmSync(origResultsDir, { recursive: true });
}
fs.cpSync(resultsDir, origResultsDir, { recursive: true });

// Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of
// backing up, deleting and symlinking the real results/ — six sites did that, and
// the parallel test runner raced them into ENOENT/ENOTEMPTY.
const tmpResultsDir = path.join(tmp, 'results-under-test');
fs.mkdirSync(tmpResultsDir, { recursive: true });
const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR;
try {
process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir;
fs.cpSync(fixtureResultsDir, tmpResultsDir, { recursive: true });
assert.doesNotThrow(() => {
const results = loadResults();
assert.strictEqual(results.length, 1000);
});
} finally {
fs.rmSync(origResultsDir, { recursive: true });
if (fs.existsSync(backupDir)) {
fs.cpSync(backupDir, origResultsDir, { recursive: true });
fs.rmSync(backupDir, { recursive: true });
}
if (prevResultsEnv === undefined) delete process.env.NF_BENCH_RESULTS_DIR;
else process.env.NF_BENCH_RESULTS_DIR = prevResultsEnv;
fs.rmSync(tmp, { recursive: true });
}
});
Expand Down Expand Up @@ -126,27 +123,23 @@ describe('ADV BUG 6: runSolve with very long timeout', () => {
describe('ADV BUG 7: saveResult with special characters in challenge ID', () => {
test('handles special characters in result filenames', () => {
const tmp = makeTmpDir();
const resultsDir = path.join(tmp, 'results');
fs.mkdirSync(resultsDir);

const origResultsDir = path.join(__dirname, '..', 'results');
const backupDir = path.join(tmp, 'backup');
if (fs.existsSync(origResultsDir)) {
fs.cpSync(origResultsDir, backupDir, { recursive: true });
fs.rmSync(origResultsDir, { recursive: true });
}
fs.symlinkSync(resultsDir, origResultsDir);

const fixtureResultsDir = path.join(tmp, 'results');
fs.mkdirSync(fixtureResultsDir);

// Hermetic: point the library at a temp dir via NF_BENCH_RESULTS_DIR instead of
// backing up, deleting and symlinking the real results/ — six sites did that, and
// the parallel test runner raced them into ENOENT/ENOTEMPTY.
const tmpResultsDir = path.join(tmp, 'results-under-test');
fs.mkdirSync(tmpResultsDir, { recursive: true });
const prevResultsEnv = process.env.NF_BENCH_RESULTS_DIR;
try {
process.env.NF_BENCH_RESULTS_DIR = tmpResultsDir;
assert.doesNotThrow(() => {
saveResult('BENCH-001!@#$%^&*()', { test: 'data' });
});
} finally {
fs.rmSync(origResultsDir);
if (fs.existsSync(backupDir)) {
fs.cpSync(backupDir, origResultsDir, { recursive: true });
fs.rmSync(backupDir, { recursive: true });
}
if (prevResultsEnv === undefined) delete process.env.NF_BENCH_RESULTS_DIR;
else process.env.NF_BENCH_RESULTS_DIR = prevResultsEnv;
fs.rmSync(tmp, { recursive: true });
}
});
Expand Down
Loading
Loading