diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9bdf215 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Task prompts are sent byte for byte; never convert their line endings. +tasks/** text eol=lf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..6dcc355 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,20 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: '22.12.0' + - run: npm test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0f16145 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +node_modules/ +# Generated attempts and prompts are outputs, not source. +runs/ +prompts/ +.DS_Store +.env +.env.* diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000..dbe9d39 --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,5 @@ +{ + "printWidth": 110, + "singleQuote": true, + "arrowParens": "avoid" +} diff --git a/README.md b/README.md index 767f725..7157794 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,54 @@ # 3JSBench -3JSBench evaluates LLMs on procedural text-to-3D generation of [Three.js](https://github.com/mrdoob/three.js) objects. +3JSBench evaluates LLMs on procedural text-to-3D generation of [Three.js](https://github.com/mrdoob/three.js) objects. Each task is a short brief, such as: > a cozy italian restaurant storefront with the restaurant's name on the sign above the door, warm lit letters over a striped awning -| | GPT-6 Astra | Gemini 3.8 Flash | Grok 4.6 | -| :--- | :---: | :---: | :---: | -| | GPT-6 Astra | Gemini 3.8 Flash | Grok 4.6 | -| Restaurant name is readable | ✓ | ✗ | ✗ | -| Awning is level or slopes down outward | ✓ | ✓ | ✗ | -| Awning is connected | ✓ | ✗ | ✗ | -| Sign touches the building facade | ✓ | ✓ | ✓ | -| Awning and sign don't intersect | ✓ | ✓ | ✓ | -| No flickering surfaces | ✓ | ✗ | ✗ | -| **Result** | **Pass** (16/16) | **Fail** (11/16) | **Fail** (12/16) | +Website: [3jsbench.com](https://3jsbench.com) -`tasks//prompt.md` holds the brief for each of the 100 tasks. The grader -source is coming soon. +This repository holds the 100 tasks and the code that generates attempts. We +will release the full benchmark soon, including the grader that renders each +attempt, maps its parts and runs the checks. -Website: [3jsbench.com](https://3jsbench.com) +- `tasks//prompt.md`: the brief. +- [`src/`](src/) and [`tools/3jsbench.js`](tools/3jsbench.js): generation. + +## Generate + +Requires Node 22.12+. There are no package dependencies. + +```sh +node tools/3jsbench.js generate --model vendor/model-name --out runs/my-model +``` + +This calls any OpenRouter model id with `OPENROUTER_API_KEY` and makes three +attempts per task by default. To generate with your own model, give the base +URL of an OpenAI-compatible server: + +```sh +node tools/3jsbench.js generate --model my-model --base-url https://my-server/v1 --out runs/my-model +``` + +Each attempt folder, `runs////`, holds the request, every +response, the parsed completion (served model and provider, tokens, cost) and +the reply, saved verbatim as `asset.ts`. To send the requests yourself, +`node tools/3jsbench.js prompts --out prompts/` writes each task's exact +messages. Run `node tools/3jsbench.js --help` for all options. + +## The request + +Each attempt is one request and one reply, with no tools and no second turn. +Every model gets the same system prompt ([`src/prompt.js`](src/prompt.js)), +then the task's brief as the user message: + +> Write a Three.js 0.184.0 module that builds the object in the brief. Default-export a function that takes no arguments and returns a new THREE.Group. Each call must return a new copy of the same object. Import only from "three", and do not load textures, files or other external resources. Use at most 1,000 meshes and 200,000 triangles in total, counting each instance separately. +Y is up, +Z is the front and +X is to the right. Unless the brief says otherwise, the lowest point is at y=0. Add a floor or other scenery only if the brief asks for it. Return only complete raw TypeScript, saved verbatim as asset.ts: no Markdown, explanation or patch. + +| Setting | Value | +| --- | --- | +| Reasoning | high | +| Output budget | 64,000 tokens | +| Temperature | not set | +| Route | OpenRouter, no fallbacks, data collection denied; `--provider` requires one provider | +| Retries | up to 3 sends, only when the provider did not process the request (rate limit, unavailable gateway, connection refused) | + +An empty or truncated reply is still saved as the attempt. diff --git a/assets/gemini-3.8-flash.jpg b/assets/gemini-3.8-flash.jpg deleted file mode 100644 index d2aadd8..0000000 Binary files a/assets/gemini-3.8-flash.jpg and /dev/null differ diff --git a/assets/gpt-6-astra.jpg b/assets/gpt-6-astra.jpg deleted file mode 100644 index 1d22134..0000000 Binary files a/assets/gpt-6-astra.jpg and /dev/null differ diff --git a/assets/grok-4.6.jpg b/assets/grok-4.6.jpg deleted file mode 100644 index 49c0379..0000000 Binary files a/assets/grok-4.6.jpg and /dev/null differ diff --git a/package.json b/package.json new file mode 100644 index 0000000..160132f --- /dev/null +++ b/package.json @@ -0,0 +1,15 @@ +{ + "name": "3jsbench", + "private": true, + "type": "module", + "description": "A benchmark of 3D assets that language models write as Three.js programs.", + "bin": { + "3jsbench": "tools/3jsbench.js" + }, + "engines": { + "node": ">=22.12.0" + }, + "scripts": { + "test": "node --test \"test/**/*.test.js\"" + } +} diff --git a/src/generation.js b/src/generation.js new file mode 100644 index 0000000..b6d732a --- /dev/null +++ b/src/generation.js @@ -0,0 +1,86 @@ +// How an attempt is generated: one request, one reply, no tools and no repair +// turn. Every model is asked the same request (prompt.js) at high reasoning +// with the same token budget. The settings are written beside the attempts +// they produced. +import { readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { PROMPT_SHA256, requestMessages } from './prompt.js'; +import { callModel } from './model-call.js'; + +/** An OpenAI-compatible server's base URL, such as https://host/v1: http(s), no credentials, query or fragment. */ +function validBaseUrl(value) { + let url; + try { + url = new URL(value); + } catch { + throw new Error('--base-url must be an http(s) URL'); + } + if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || /[?#]/.test(value)) + throw new Error('--base-url must be an http(s) URL without credentials, query or fragment'); + return url.href.replace(/\/+$/, ''); +} + +/** + * The settings for one model. Through OpenRouter (the default), `provider` + * requires one backend, so every attempt of a model is served by the same one; + * otherwise OpenRouter chooses per request. With `baseUrl`, the request goes + * directly to an OpenAI-compatible server. + */ +export function generationSettings({ model, provider = null, baseUrl = null }) { + if (typeof model !== 'string' || !model.trim()) throw new Error('--model is required'); + if (baseUrl !== null && provider !== null) throw new Error('--provider applies only to OpenRouter'); + if (provider !== null && !/^[a-z0-9][a-z0-9-]*$/.test(provider)) + throw new Error('--provider must be an OpenRouter provider slug'); + return { + model, + api: baseUrl !== null ? 'openai-compatible' : 'openrouter', + ...(baseUrl !== null ? { baseUrl: validBaseUrl(baseUrl) } : {}), + ...(provider !== null ? { provider } : {}), + reasoning: 'high', + maxTokens: 64_000, + promptSha256: PROMPT_SHA256, + // No local deadline: a request that timed out may still have been billed. + transport: { attempts: 3, maxResponseBytes: 8 * 1024 * 1024 }, + }; +} + +// Failures that are the model's own: it replied with nothing, or its reply was +// cut off. Its reply is still the attempt. Any other failure produced no reply. +const isModelFailure = reason => reason === 'empty-output' || String(reason).startsWith('incomplete-output:'); + +/** Saves a completion's reply verbatim as asset.ts, unless it is already saved. */ +async function saveReply(dir, completion) { + const replied = completion.ok || isModelFailure(completion.reason); + if (replied) + await writeFile(path.join(dir, 'asset.ts'), completion.source ?? '', { flag: 'wx', mode: 0o600 }).catch( + error => { + if (error.code !== 'EEXIST') throw error; + }, + ); + return { replied, reason: completion.reason }; +} + +/** Generates one attempt into `dir`, which must not exist yet. */ +export async function generateAttempt({ task, settings, dir, call = callModel }) { + return saveReply(dir, await call({ settings, messages: requestMessages(task), dir })); +} + +/** + * An attempt a previous run finished in `dir`, read from its saved files without + * sending anything; null if it has no completion. A reply a run saved in + * completion.json but was stopped before writing to asset.ts is written now. An + * attempt sent with a different request, such as an edited brief, is refused. + */ +export async function savedAttempt({ task, dir }) { + let completion; + try { + completion = JSON.parse(await readFile(path.join(dir, 'completion.json'), 'utf8')); + } catch (error) { + if (error.code === 'ENOENT') return null; + throw error; + } + const request = JSON.parse(await readFile(path.join(dir, 'request.json'), 'utf8')); + if (JSON.stringify(request.messages) !== JSON.stringify(requestMessages(task))) + throw new Error(`${dir} was sent a different request; use another --out`); + return saveReply(dir, completion); +} diff --git a/src/model-call.js b/src/model-call.js new file mode 100644 index 0000000..3692db2 --- /dev/null +++ b/src/model-call.js @@ -0,0 +1,224 @@ +// One model call: the request body, the transport and its retry rule, and the +// reading of the reply. Every call leaves its request, each response and the +// parsed completion in the attempt directory. +import { createHash } from 'node:crypto'; +import http from 'node:http'; +import https from 'node:https'; +import { mkdir, rename, writeFile } from 'node:fs/promises'; +import path from 'node:path'; + +const OPENROUTER_URL = 'https://openrouter.ai/api/v1/chat/completions'; +// An OpenAI-compatible server (a lab's own, vLLM, SGLang, ...) has its own key +// variable, so a key set for another service is never sent to it. The key is +// optional, since local servers often need none. +const KEY_ENV = { openrouter: 'OPENROUTER_API_KEY', 'openai-compatible': 'THREEJSBENCH_ENDPOINT_KEY' }; +const compatible = settings => settings.api === 'openai-compatible'; +const urlOf = settings => (compatible(settings) ? settings.baseUrl + '/chat/completions' : OPENROUTER_URL); +// Finish reasons that mean the model ended its reply. TGI names a natural end eos_token or stop_sequence. +const stops = settings => (compatible(settings) ? ['stop', 'eos_token', 'stop_sequence'] : ['stop']); +const loopback = host => host === 'localhost' || host === '[::1]' || /^127(\.\d{1,3}){3}$/.test(host); +const providerSlug = name => + String(name) + .toLowerCase() + .replace(/[^a-z0-9]/g, ''); +const sha256 = text => createHash('sha256').update(text).digest('hex'); + +/** The JSON body sent for `messages` under generation settings (generation.js). */ +export function requestBody(settings, messages) { + if (compatible(settings)) + // A plain Chat Completions request: no routing block and no usage extension. + return { + model: settings.model, + messages, + max_completion_tokens: settings.maxTokens, + reasoning_effort: settings.reasoning, + stream: false, + }; + return { + model: settings.model, + messages, + max_tokens: settings.maxTokens, + reasoning: { effort: settings.reasoning }, + provider: { + ...(settings.provider ? { only: [settings.provider] } : {}), + allow_fallbacks: false, + require_parameters: true, + data_collection: 'deny', + }, + usage: { include: true }, + stream: false, + }; +} + +function transport(body, { url, key, maxResponseBytes }) { + return new Promise((resolve, reject) => { + const request = (new URL(url).protocol === 'http:' ? http : https).request(url, { + method: 'POST', + headers: { + ...(key ? { Authorization: 'Bearer ' + key } : {}), + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(body), + }, + }); + request.on('error', error => reject(new Error(error.code ?? error.message))); + request.on('response', response => { + const chunks = []; + let bytes = 0; + response.on('data', chunk => { + bytes += chunk.length; + if (bytes > maxResponseBytes) request.destroy(new Error('response-byte-limit')); + else chunks.push(chunk); + }); + response.on('error', error => reject(new Error(error.code ?? 'response-error'))); + response.on('end', () => + resolve({ + status: response.statusCode, + text: Buffer.concat(chunks).toString('utf8'), + requestId: response.headers['x-request-id'] ?? null, + }), + ); + }); + request.end(body); + }); +} + +// OpenRouter reports the cost in USD. +const callCost = usage => + typeof usage?.cost === 'number' && Number.isFinite(usage.cost) ? usage.cost : null; + +/** The completion read from a 200 response body: the reply, who served it, and why it failed if it did. */ +export function parseCompletion(body, settings) { + const choice = body.choices?.[0]; + const usage = body.usage ?? null, + finishReason = choice?.finish_reason ?? null; + const completion = { + source: typeof choice?.message?.content === 'string' ? choice.message.content : '', + completionId: body.id ?? null, + model: body.model ?? null, + // OpenRouter names the backend it routed to; another server records its host. + provider: compatible(settings) ? new URL(settings.baseUrl).host : (body.provider ?? null), + finishReason, + usage, + reasoningTokens: usage?.completion_tokens_details?.reasoning_tokens ?? null, + cost: callCost(usage), + }; + let reason = null; + const error = body.error ?? choice?.error; + if (error) { + completion.providerError = { + code: typeof error.code === 'number' || typeof error.code === 'string' ? error.code : null, + message: typeof error.message === 'string' ? error.message.slice(0, 500) : 'Provider error', + }; + reason = 'provider-error'; + } else if ( + compatible(settings) && + choice && + typeof choice.message?.content !== 'string' && + typeof finishReason === 'string' && + !stops(settings).includes(finishReason) + ) + // vLLM with a reasoning parser returns no content when the budget runs out + // mid-reasoning: the model's truncation, not a malformed response. + reason = 'incomplete-output:' + finishReason; + else if (!choice || typeof choice.message?.content !== 'string') reason = 'invalid-provider-response'; + else if (!choice.message.content.trim()) reason = 'empty-output'; + else if ( + settings.provider && + (typeof completion.provider !== 'string' || + providerSlug(completion.provider) !== providerSlug(settings.provider)) + ) + reason = 'served-provider-mismatch-or-missing'; + // An OpenAI-compatible server may name its model differently; the served name is saved, not checked. + else if (!compatible(settings) && completion.model !== settings.model) + reason = 'served-model-mismatch-or-missing'; + else if (!stops(settings).includes(finishReason)) reason = 'incomplete-output:' + finishReason; + return { ...completion, ok: reason === null, reason }; +} + +const writeJson = (file, value) => + writeFile(file, JSON.stringify(value, null, 2) + '\n', { flag: 'wx', mode: 0o600 }); + +// Written whole or not at all, so a rerun never reads a partial completion. +async function writeCompletion(dir, value) { + const file = path.join(dir, 'completion.json'); + await writeJson(file + '.partial', value); + await rename(file + '.partial', file); +} + +/** + * Sends one request and returns its completion. `dir` must not exist yet: a call + * the provider may have accepted is never sent again by a rerun. The request is + * re-sent only when the provider provably did not process it: rate limiting, an + * unavailable gateway, or a connection that never reached it. + */ +export async function callModel({ + settings, + messages, + dir, + key = process.env[KEY_ENV[settings.api]], + send = transport, + delay = ms => new Promise(resolve => setTimeout(resolve, ms)), +}) { + if (!key && !compatible(settings)) throw new Error(KEY_ENV[settings.api] + ' is required'); + const url = urlOf(settings), + target = new URL(url); + if (key && target.protocol === 'http:' && !loopback(target.hostname)) + throw new Error( + `Refusing to send ${KEY_ENV[settings.api]} over plain http to ${target.host}; use https or a loopback address`, + ); + await mkdir(dir); + const body = requestBody(settings, messages), + text = JSON.stringify(body), + requestSha256 = sha256(text), + calls = []; + await writeJson(path.join(dir, 'request.json'), body); + for (let attempt = 1; attempt <= settings.transport.attempts; attempt++) { + const start = new Date().toISOString(), + clock = performance.now(); + let response = null, + error = null; + try { + response = await send(text, { url, key, maxResponseBytes: settings.transport.maxResponseBytes }); + } catch (e) { + error = String(e.message).slice(0, 200); + } + const call = { + attempt, + start, + durationMs: Math.round(performance.now() - clock), + requestSha256, + status: response?.status ?? null, + requestId: response?.requestId ?? null, + transportError: error, + }; + if (response) { + await writeFile(path.join(dir, `response-${attempt}.json`), response.text, { flag: 'wx', mode: 0o600 }); + call.responseSha256 = sha256(response.text); + } + calls.push(call); + if (response?.status === 200) { + let completion; + try { + completion = parseCompletion(JSON.parse(response.text), settings); + } catch { + completion = { ok: false, reason: 'invalid-provider-json' }; + } + const result = { ...completion, requestSha256, calls }; + await writeCompletion(dir, result); + return result; + } + const unreached = error !== null && /ECONNREFUSED|ENOTFOUND|EAI_AGAIN/.test(error); + const retryable = unreached || [429, 502, 503, 504].includes(response?.status); + if (!retryable || attempt === settings.transport.attempts) { + const result = { + ok: false, + reason: error ? 'transport-failure' : 'http-' + response.status, + requestSha256, + calls, + }; + await writeCompletion(dir, result); + return result; + } + await delay(Math.min(5000, 1000 * 2 ** (attempt - 1))); + } +} diff --git a/src/prompt.js b/src/prompt.js new file mode 100644 index 0000000..fe08b5b --- /dev/null +++ b/src/prompt.js @@ -0,0 +1,23 @@ +// The request a model is sent for a task: one system prompt shared by every +// task, then the task's brief alone. The prompt text is pinned by hash in +// test/prompt.test.js; changing it changes what every model is asked. +import { createHash } from 'node:crypto'; + +export const SYSTEM_PROMPT = + 'Write a Three.js 0.184.0 module that builds the object in the brief. Default-export a function that takes ' + + 'no arguments and returns a new THREE.Group. Each call must return a new copy of the same object. Import only ' + + 'from "three", and do not load textures, files or other external resources. Use at most 1,000 meshes and ' + + '200,000 triangles in total, counting each instance separately. +Y is up, +Z is the front and +X is to the ' + + 'right. Unless the brief says otherwise, the lowest point is at y=0. Add a floor or other scenery only if the ' + + 'brief asks for it. Return only complete raw TypeScript, saved verbatim as asset.ts: no Markdown, explanation ' + + 'or patch.'; + +export const PROMPT_SHA256 = createHash('sha256').update(SYSTEM_PROMPT).digest('hex'); + +/** The exact messages for a task from loadTask. */ +export function requestMessages(task) { + return [ + { role: 'system', content: SYSTEM_PROMPT }, + { role: 'user', content: [{ type: 'text', text: task.prompt }] }, + ]; +} diff --git a/src/tasks.js b/src/tasks.js new file mode 100644 index 0000000..1e42026 --- /dev/null +++ b/src/tasks.js @@ -0,0 +1,25 @@ +// A task is one folder, tasks//, holding prompt.md: the brief the model +// is given, plus exactly one final LF. +import { readdir, readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export const TASKS_DIR = fileURLToPath(new URL('../tasks/', import.meta.url)); + +/** Task folder names, sorted. */ +export async function listTasks(dir = TASKS_DIR) { + const entries = await readdir(dir, { withFileTypes: true }); + return entries + .filter(entry => entry.isDirectory()) + .map(entry => entry.name) + .sort(); +} + +/** One task, by folder name: { id, prompt }. */ +export async function loadTask(name, dir = TASKS_DIR) { + if (!(await listTasks(dir)).includes(name)) throw new Error('Unknown task: ' + name); + const text = await readFile(path.join(dir, name, 'prompt.md'), 'utf8'); + if (text.includes('\r') || !text.endsWith('\n') || text.endsWith('\n\n') || !text.trim()) + throw new Error(`${name}/prompt.md must be nonempty, end with exactly one LF and contain no CR`); + return { id: name, prompt: text.slice(0, -1) }; +} diff --git a/test/generation.test.js b/test/generation.test.js new file mode 100644 index 0000000..1f5d529 --- /dev/null +++ b/test/generation.test.js @@ -0,0 +1,197 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdir, mkdtemp, readFile, readdir, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { loadTask } from '../src/tasks.js'; +import { requestMessages } from '../src/prompt.js'; +import { generateAttempt, generationSettings, savedAttempt } from '../src/generation.js'; +import { callModel, parseCompletion, requestBody } from '../src/model-call.js'; + +const routed = generationSettings({ model: 'vendor/model', provider: 'vendor' }); +const lab = generationSettings({ model: 'my-model', baseUrl: 'https://models.example/v1/' }); +const scratch = async () => path.join(await mkdtemp(path.join(tmpdir(), '3jsbench-')), 'attempt'); +const noDelay = async () => {}; + +const completion = (content, extra = {}) => ({ + id: 'gen-1', + model: 'vendor/model', + provider: 'Vendor', + choices: [{ message: { content }, finish_reason: 'stop' }], + usage: { cost: 0.5, completion_tokens_details: { reasoning_tokens: 100 } }, + ...extra, +}); +const reply = (status, body) => ({ status, text: JSON.stringify(body), requestId: 'req' }); + +test('every model is asked at high reasoning with the same budget', () => { + assert.deepEqual(routed, { + model: 'vendor/model', + api: 'openrouter', + provider: 'vendor', + reasoning: 'high', + maxTokens: 64_000, + promptSha256: '4877dcbae3c85c37078fae7a2b6a7a87060550e5855eeed0eeee26296b4359c6', + transport: { attempts: 3, maxResponseBytes: 8 * 1024 * 1024 }, + }); + const body = requestBody(routed, []); + assert.deepEqual(body.provider, { + only: ['vendor'], + allow_fallbacks: false, + require_parameters: true, + data_collection: 'deny', + }); + assert.deepEqual(body.reasoning, { effort: 'high' }); + assert.equal(body.max_tokens, 64_000); + assert.equal('temperature' in body, false); + // Without --provider, OpenRouter chooses the provider, still without fallbacks. + assert.deepEqual(requestBody(generationSettings({ model: 'vendor/model' }), []).provider, { + allow_fallbacks: false, + require_parameters: true, + data_collection: 'deny', + }); +}); + +test('an OpenAI-compatible server gets a plain Chat Completions request', () => { + assert.equal(lab.baseUrl, 'https://models.example/v1'); + assert.equal(lab.provider, undefined); + assert.deepEqual(requestBody(lab, []), { + model: 'my-model', + messages: [], + max_completion_tokens: 64_000, + reasoning_effort: 'high', + stream: false, + }); + assert.throws(() => generationSettings({ model: 'm', baseUrl: 'https://u:p@host/v1' }), /credentials/); + assert.throws(() => generationSettings({ model: 'm', baseUrl: 'ftp://host/v1' }), /http/); + assert.throws( + () => generationSettings({ model: 'm', baseUrl: 'https://host/v1', provider: 'x' }), + /OpenRouter/, + ); + assert.throws(() => generationSettings({ model: 'm', provider: 'Bad Slug' }), /slug/); + assert.throws(() => generationSettings({}), /--model/); +}); + +test('a reply is accepted only from the requested model on the required provider', () => { + assert.equal(parseCompletion(completion('code'), routed).ok, true); + assert.equal(parseCompletion(completion(' '), routed).reason, 'empty-output'); + assert.equal( + parseCompletion(completion('code', { provider: 'Google' }), routed).reason, + 'served-provider-mismatch-or-missing', + ); + assert.equal( + parseCompletion(completion('code', { model: 'vendor/other' }), routed).reason, + 'served-model-mismatch-or-missing', + ); + const cut = completion('code'); + cut.choices[0].finish_reason = 'length'; + assert.equal(parseCompletion(cut, routed).reason, 'incomplete-output:length'); + assert.equal(parseCompletion({ error: { code: 400, message: 'bad' } }, routed).reason, 'provider-error'); + assert.equal(parseCompletion({ choices: [] }, routed).reason, 'invalid-provider-response'); + // An OpenAI-compatible server may name its model differently; the name is saved, not checked. + const served = parseCompletion(completion('code', { model: 'my-model-2026', provider: undefined }), lab); + assert.equal(served.ok, true); + assert.equal(served.model, 'my-model-2026'); +}); + +test('the request is re-sent only when the provider provably did not process it', async () => { + const sent = []; + const sequence = responses => async body => { + sent.push(body); + const next = responses.shift(); + if (next instanceof Error) throw next; + return next; + }; + const retried = await callModel({ + settings: routed, + messages: [], + dir: await scratch(), + key: 'k', + delay: noDelay, + send: sequence([new Error('ECONNREFUSED'), reply(429, {}), reply(200, completion('code'))]), + }); + assert.equal(retried.ok, true); + assert.equal(retried.calls.length, 3); + assert.equal(new Set(sent).size, 1); + + for (const failure of [reply(500, {}), new Error('ECONNRESET')]) { + const result = await callModel({ + settings: routed, + messages: [], + dir: await scratch(), + key: 'k', + delay: noDelay, + send: sequence([failure, reply(200, completion('code'))]), + }); + assert.equal(result.ok, false); + assert.equal(result.calls.length, 1); + } +}); + +test('a call leaves its request, responses and completion, and never reuses a directory', async () => { + const dir = await scratch(); + const send = async () => reply(200, completion('code')); + await callModel({ settings: routed, messages: [], dir, key: 'k', send }); + assert.deepEqual((await readdir(dir)).sort(), ['completion.json', 'request.json', 'response-1.json']); + await assert.rejects(callModel({ settings: routed, messages: [], dir, key: 'k', send }), /EEXIST/); +}); + +test('keys are required for OpenRouter and never sent over plain http to a remote host', async () => { + await assert.rejects( + callModel({ settings: routed, messages: [], dir: await scratch(), key: '' }), + /OPENROUTER_API_KEY/, + ); + const remote = generationSettings({ model: 'm', baseUrl: 'http://models.example/v1' }); + await assert.rejects( + callModel({ settings: remote, messages: [], dir: await scratch(), key: 'k' }), + /plain http/, + ); + const local = generationSettings({ model: 'm', baseUrl: 'http://127.0.0.1:8000/v1' }); + const send = async () => reply(200, completion('code', { model: 'm' })); + assert.equal( + (await callModel({ settings: local, messages: [], dir: await scratch(), key: 'k', send })).ok, + true, + ); +}); + +test('the reply is saved verbatim, including an empty or truncated one; no reply saves nothing', async () => { + const task = await loadTask('spiral-staircase'); + const fenced = '```ts\nexport default () => null;\n```'; + for (const [result, saved] of [ + [{ ok: true, reason: null, source: fenced }, fenced], + [{ ok: false, reason: 'empty-output', source: '' }, ''], + [{ ok: false, reason: 'incomplete-output:length', source: 'export default' }, 'export default'], + [{ ok: false, reason: 'http-500' }, null], + [{ ok: false, reason: 'served-model-mismatch-or-missing', source: 'code' }, null], + ]) { + const dir = await scratch(); + const call = async ({ messages, dir: target }) => { + assert.deepEqual(messages, requestMessages(task)); + await mkdir(target); + return result; + }; + const { replied } = await generateAttempt({ task, settings: routed, dir, call }); + assert.equal(replied, saved !== null, result.reason ?? 'ok'); + const text = await readFile(path.join(dir, 'asset.ts'), 'utf8').catch(() => null); + assert.equal(text, saved); + } +}); + +test('a rerun restores a reply saved before an interruption, and refuses a changed request', async () => { + const task = await loadTask('spiral-staircase'); + const dir = await scratch(); + assert.equal(await savedAttempt({ task, dir }), null); + const send = async () => reply(200, completion('export default () => null;')); + await callModel({ settings: routed, messages: requestMessages(task), dir, key: 'k', send }); + // The run stopped after completion.json and before asset.ts. + assert.deepEqual(await savedAttempt({ task, dir }), { replied: true, reason: null }); + assert.equal(await readFile(path.join(dir, 'asset.ts'), 'utf8'), 'export default () => null;'); + assert.equal((await stat(path.join(dir, 'asset.ts'))).mode & 0o777, 0o600); + // A second rerun leaves the saved reply as it is. + await writeFile(path.join(dir, 'asset.ts'), 'edited'); + await savedAttempt({ task, dir }); + assert.equal(await readFile(path.join(dir, 'asset.ts'), 'utf8'), 'edited'); + await assert.rejects( + savedAttempt({ task: { ...task, prompt: 'another brief' }, dir }), + /different request/, + ); +}); diff --git a/test/prompt.test.js b/test/prompt.test.js new file mode 100644 index 0000000..85b4539 --- /dev/null +++ b/test/prompt.test.js @@ -0,0 +1,31 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { listTasks, loadTask } from '../src/tasks.js'; +import { PROMPT_SHA256, SYSTEM_PROMPT, requestMessages } from '../src/prompt.js'; + +test('the system prompt is pinned', () => { + assert.equal(PROMPT_SHA256, '4877dcbae3c85c37078fae7a2b6a7a87060550e5855eeed0eeee26296b4359c6'); + assert.equal(createHash('sha256').update(SYSTEM_PROMPT).digest('hex'), PROMPT_SHA256); +}); + +test('the user message is the brief alone', async () => { + const task = await loadTask('spiral-staircase'); + assert.deepEqual(requestMessages(task), [ + { role: 'system', content: SYSTEM_PROMPT }, + { role: 'user', content: [{ type: 'text', text: task.prompt }] }, + ]); +}); + +test('every task loads, with a brief ending in no newline', async () => { + const names = await listTasks(); + assert.equal(names.length, 100); + for (const name of names) { + const task = await loadTask(name); + assert.ok(task.prompt.trim() && !task.prompt.endsWith('\n'), name); + } +}); + +test('an unknown task is refused', async () => { + await assert.rejects(loadTask('../tasks'), /Unknown task/); +}); diff --git a/tools/3jsbench.js b/tools/3jsbench.js new file mode 100755 index 0000000..f9a2396 --- /dev/null +++ b/tools/3jsbench.js @@ -0,0 +1,135 @@ +#!/usr/bin/env node +// 3jsbench command line: list the tasks, write their exact requests, generate attempts. +import { access, mkdir, readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { parseArgs } from 'node:util'; +import { listTasks, loadTask } from '../src/tasks.js'; +import { requestMessages } from '../src/prompt.js'; +import { generateAttempt, generationSettings, savedAttempt } from '../src/generation.js'; + +const HELP = `3jsbench: generate Three.js programs for the 3JSBench tasks. + + 3jsbench tasks + 3jsbench prompts --out DIR + 3jsbench generate --model MODEL --out DIR [--tasks all|a,b,...] [--attempts 3] [--concurrency 4] + [--provider SLUG | --base-url URL] + +prompts writes each task's exact request (system and user messages) to DIR/.json. +generate sends each request to MODEL and saves the reply verbatim as DIR///asset.ts, +beside the request, every response and the parsed completion. DIR/generation.json records the +settings. Rerunning generate skips finished attempts and never re-sends a request that may have +reached the provider; delete an attempt folder to send its request again. + +generate calls MODEL through OpenRouter (OPENROUTER_API_KEY); --provider requires one OpenRouter +provider. With --base-url, it calls an OpenAI-compatible server directly, with the +optional key THREEJSBENCH_ENDPOINT_KEY, never sent over plain http except to a loopback address.`; + +const OPTIONS = { + out: { type: 'string' }, + model: { type: 'string' }, + tasks: { type: 'string', default: 'all' }, + attempts: { type: 'string', default: '3' }, + concurrency: { type: 'string', default: '4' }, + provider: { type: 'string' }, + 'base-url': { type: 'string' }, + help: { type: 'boolean', short: 'h', default: false }, +}; + +function count(value, flag) { + const n = Number(value); + if (!Number.isInteger(n) || n < 1) throw new Error(flag + ' must be a positive integer'); + return n; +} + +async function selectTasks(list) { + const names = list === 'all' ? await listTasks() : list.split(','); + if (new Set(names).size !== names.length) throw new Error('--tasks names a task twice'); + return Promise.all(names.map(name => loadTask(name))); +} + +const exists = file => + access(file).then( + () => true, + () => false, + ); + +/** Runs `work` over `items`, at most `limit` at a time. */ +async function pool(items, limit, work) { + let next = 0; + const worker = async () => { + while (next < items.length) await work(items[next++]); + }; + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)); +} + +/** The settings a run directory records; a rerun must use the same ones. */ +async function recordSettings(out, settings) { + const file = path.join(out, 'generation.json'), + text = JSON.stringify(settings, null, 2) + '\n'; + await mkdir(out, { recursive: true }); + if (!(await exists(file))) return writeFile(file, text, { flag: 'wx' }); + if ((await readFile(file, 'utf8')) !== text) + throw new Error(`${file} records other settings; use another --out`); +} + +async function generate(v) { + if (!v.out) throw new Error('--out is required'); + const settings = generationSettings({ + model: v.model, + provider: v.provider ?? null, + baseUrl: v['base-url'] ?? null, + }); + const tasks = await selectTasks(v.tasks); + const attempts = count(v.attempts, '--attempts'), + concurrency = count(v.concurrency, '--concurrency'); + await recordSettings(v.out, settings); + const jobs = tasks.flatMap(task => + Array.from({ length: attempts }, (_, n) => ({ task, dir: path.join(v.out, task.id, String(n + 1)) })), + ); + const tally = { replied: 0, unanswered: 0, unfinished: 0 }; + await pool(jobs, concurrency, async ({ task, dir }) => { + const name = path.relative(v.out, dir); + const saved = await savedAttempt({ task, dir }); + if (saved) return tally[saved.replied ? 'replied' : 'unanswered']++; + if (await exists(dir)) { + // A call was sent but never finished; it may have been billed. + console.error(`${name}: a previous call did not finish and may have reached the provider`); + return tally.unfinished++; + } + await mkdir(path.dirname(dir), { recursive: true }); + const { replied, reason } = await generateAttempt({ task, settings, dir }); + tally[replied ? 'replied' : 'unanswered']++; + console.error(`${name}: ${replied ? 'reply saved' : 'no reply'}${reason ? ` (${reason})` : ''}`); + }); + const unfinished = tally.unfinished ? `, ${tally.unfinished} unfinished` : ''; + console.log( + `${jobs.length} attempts: ${tally.replied} with a reply, ${tally.unanswered} without${unfinished}.`, + ); + if (tally.unanswered || tally.unfinished) + console.log('To send a request again, delete its attempt folder and rerun.'); +} + +async function main() { + const [command, ...args] = process.argv.slice(2); + if (command === '--help' || command === '-h') return console.log(HELP); + const { values: v } = parseArgs({ args, options: OPTIONS, strict: true }); + if (!command || v.help) return console.log(HELP); + if (command === 'tasks') return console.log((await listTasks()).join('\n')); + if (command === 'prompts') { + if (!v.out) throw new Error('--out is required'); + await mkdir(v.out, { recursive: true }); + for (const task of await selectTasks('all')) + await writeFile( + path.join(v.out, task.id + '.json'), + JSON.stringify({ messages: requestMessages(task) }, null, 2) + '\n', + ); + return; + } + if (command === 'generate') return generate(v); + throw new Error('Unknown command: ' + command + '\n\n' + HELP); +} + +main().catch(error => { + console.error(error.message); + process.exitCode = 1; +});