Add generation environment and source

.gitattributes

@@ -0,0 +1,2 @@
+# Task prompts are sent byte for byte; never convert their line endings.
+tasks/** text eol=lf

.github/workflows/ci.yml

@@ -0,0 +1,20 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-node@v6
+        with:
+          node-version: '22.12.0'
+      - run: npm test

.gitignore

@@ -0,0 +1,7 @@
+node_modules/
+# Generated attempts and prompts are outputs, not source.
+runs/
+prompts/
+.DS_Store
+.env
+.env.*

.prettierrc.json

@@ -0,0 +1,5 @@
+{
+  "printWidth": 110,
+  "singleQuote": true,
+  "arrowParens": "avoid"
+}

README.md

@@ -1,21 +1,54 @@
 # 3JSBench
 
-3JSBench evaluates LLMs on procedural text-to-3D generation of [Three.js](https://github.com/mrdoob/three.js) objects.
+3JSBench evaluates LLMs on procedural text-to-3D generation of [Three.js](https://github.com/mrdoob/three.js) objects. Each task is a short brief, such as:
 
 > a cozy italian restaurant storefront with the restaurant's name on the sign above the door, warm lit letters over a striped awning
 
-| | GPT-6 Astra | Gemini 3.8 Flash | Grok 4.6 |
-| :--- | :---: | :---: | :---: |
-| | <img src="assets/gpt-6-astra.jpg" width="240" alt="GPT-6 Astra"> | <img src="assets/gemini-3.8-flash.jpg" width="240" alt="Gemini 3.8 Flash"> | <img src="assets/grok-4.6.jpg" width="240" alt="Grok 4.6"> |
-| Restaurant name is readable | ✓ | ✗ | ✗ |
-| Awning is level or slopes down outward | ✓ | ✓ | ✗ |
-| Awning is connected | ✓ | ✗ | ✗ |
-| Sign touches the building facade | ✓ | ✓ | ✓ |
-| Awning and sign don't intersect | ✓ | ✓ | ✓ |
-| No flickering surfaces | ✓ | ✗ | ✗ |
-| **Result** | **Pass** (16/16) | **Fail** (11/16) | **Fail** (12/16) |
+Website: [3jsbench.com](https://3jsbench.com)
 
-`tasks/<name>/prompt.md` holds the brief for each of the 100 tasks. The grader
-source is coming soon.
+This repository holds the 100 tasks and the code that generates attempts. We
+will release the full benchmark soon, including the grader that renders each
+attempt, maps its parts and runs the checks.
 
-Website: [3jsbench.com](https://3jsbench.com)
+- `tasks/<name>/prompt.md`: the brief.
+- [`src/`](src/) and [`tools/3jsbench.js`](tools/3jsbench.js): generation.
+
+## Generate
+
+Requires Node 22.12+. There are no package dependencies.
+
+```sh
+node tools/3jsbench.js generate --model vendor/model-name --out runs/my-model
+```
+
+This calls any OpenRouter model id with `OPENROUTER_API_KEY` and makes three
+attempts per task by default. To generate with your own model, give the base
+URL of an OpenAI-compatible server:
+
+```sh
+node tools/3jsbench.js generate --model my-model --base-url https://my-server/v1 --out runs/my-model
+```
+
+Each attempt folder, `runs/<model>/<task>/<n>/`, holds the request, every
+response, the parsed completion (served model and provider, tokens, cost) and
+the reply, saved verbatim as `asset.ts`. To send the requests yourself,
+`node tools/3jsbench.js prompts --out prompts/` writes each task's exact
+messages. Run `node tools/3jsbench.js --help` for all options.
+
+## The request
+
+Each attempt is one request and one reply, with no tools and no second turn.
+Every model gets the same system prompt ([`src/prompt.js`](src/prompt.js)),
+then the task's brief as the user message:
+
+> Write a Three.js 0.184.0 module that builds the object in the brief. Default-export a function that takes no arguments and returns a new THREE.Group. Each call must return a new copy of the same object. Import only from "three", and do not load textures, files or other external resources. Use at most 1,000 meshes and 200,000 triangles in total, counting each instance separately. +Y is up, +Z is the front and +X is to the right. Unless the brief says otherwise, the lowest point is at y=0. Add a floor or other scenery only if the brief asks for it. Return only complete raw TypeScript, saved verbatim as asset.ts: no Markdown, explanation or patch.
+
+| Setting | Value |
+| --- | --- |
+| Reasoning | high |
+| Output budget | 64,000 tokens |
+| Temperature | not set |
+| Route | OpenRouter, no fallbacks, data collection denied; `--provider` requires one provider |
+| Retries | up to 3 sends, only when the provider did not process the request (rate limit, unavailable gateway, connection refused) |
+
+An empty or truncated reply is still saved as the attempt.

package.json

@@ -0,0 +1,15 @@
+{
+  "name": "3jsbench",
+  "private": true,
+  "type": "module",
+  "description": "A benchmark of 3D assets that language models write as Three.js programs.",
+  "bin": {
+    "3jsbench": "tools/3jsbench.js"
+  },
+  "engines": {
+    "node": ">=22.12.0"
+  },
+  "scripts": {
+    "test": "node --test \"test/**/*.test.js\""
+  }
+}

src/generation.js

@@ -0,0 +1,86 @@
+// How an attempt is generated: one request, one reply, no tools and no repair
+// turn. Every model is asked the same request (prompt.js) at high reasoning
+// with the same token budget. The settings are written beside the attempts
+// they produced.
+import { readFile, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { PROMPT_SHA256, requestMessages } from './prompt.js';
+import { callModel } from './model-call.js';
+
+/** An OpenAI-compatible server's base URL, such as https://host/v1: http(s), no credentials, query or fragment. */
+function validBaseUrl(value) {
+  let url;
+  try {
+    url = new URL(value);
+  } catch {
+    throw new Error('--base-url must be an http(s) URL');
+  }
+  if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || /[?#]/.test(value))
+    throw new Error('--base-url must be an http(s) URL without credentials, query or fragment');
+  return url.href.replace(/\/+$/, '');
+}
+
+/**
+ * The settings for one model. Through OpenRouter (the default), `provider`
+ * requires one backend, so every attempt of a model is served by the same one;
+ * otherwise OpenRouter chooses per request. With `baseUrl`, the request goes
+ * directly to an OpenAI-compatible server.
+ */
+export function generationSettings({ model, provider = null, baseUrl = null }) {
+  if (typeof model !== 'string' || !model.trim()) throw new Error('--model is required');
+  if (baseUrl !== null && provider !== null) throw new Error('--provider applies only to OpenRouter');
+  if (provider !== null && !/^[a-z0-9][a-z0-9-]*$/.test(provider))
+    throw new Error('--provider must be an OpenRouter provider slug');
+  return {
+    model,
+    api: baseUrl !== null ? 'openai-compatible' : 'openrouter',
+    ...(baseUrl !== null ? { baseUrl: validBaseUrl(baseUrl) } : {}),
+    ...(provider !== null ? { provider } : {}),
+    reasoning: 'high',
+    maxTokens: 64_000,
+    promptSha256: PROMPT_SHA256,
+    // No local deadline: a request that timed out may still have been billed.
+    transport: { attempts: 3, maxResponseBytes: 8 * 1024 * 1024 },
+  };
+}
+
+// Failures that are the model's own: it replied with nothing, or its reply was
+// cut off. Its reply is still the attempt. Any other failure produced no reply.
+const isModelFailure = reason => reason === 'empty-output' || String(reason).startsWith('incomplete-output:');
+
+/** Saves a completion's reply verbatim as asset.ts, unless it is already saved. */
+async function saveReply(dir, completion) {
+  const replied = completion.ok || isModelFailure(completion.reason);
+  if (replied)
+    await writeFile(path.join(dir, 'asset.ts'), completion.source ?? '', { flag: 'wx', mode: 0o600 }).catch(
+      error => {
+        if (error.code !== 'EEXIST') throw error;
+      },
+    );
+  return { replied, reason: completion.reason };
+}
+
+/** Generates one attempt into `dir`, which must not exist yet. */
+export async function generateAttempt({ task, settings, dir, call = callModel }) {
+  return saveReply(dir, await call({ settings, messages: requestMessages(task), dir }));
+}
+
+/**
+ * An attempt a previous run finished in `dir`, read from its saved files without
+ * sending anything; null if it has no completion. A reply a run saved in
+ * completion.json but was stopped before writing to asset.ts is written now. An
+ * attempt sent with a different request, such as an edited brief, is refused.
+ */
+export async function savedAttempt({ task, dir }) {
+  let completion;
+  try {
+    completion = JSON.parse(await readFile(path.join(dir, 'completion.json'), 'utf8'));
+  } catch (error) {
+    if (error.code === 'ENOENT') return null;
+    throw error;
+  }
+  const request = JSON.parse(await readFile(path.join(dir, 'request.json'), 'utf8'));
+  if (JSON.stringify(request.messages) !== JSON.stringify(requestMessages(task)))
+    throw new Error(`${dir} was sent a different request; use another --out`);
+  return saveReply(dir, completion);
+}