feat(readers): open .jsonl.gz + behavior_v2 run logs everywhere via shared js/runlog-format.js (PR 3 of 3) — Studio v0.73

CLAUDE.md

@@ -296,8 +296,11 @@ fix flows to every page automatically; two hand-written HTML pages never will.
   `commitRunLog` gzips the bridge export (`GH.gzipBytes`) and commits
   `runlogs/<bench>/<name>.jsonl.gz` via `GH.commitFile`, which routes >30 MiB payloads
   through the Git Database API (`GH.directCommitLarge`) because the Contents API
-  rejects ~35 MiB+ files. Every reader (dashboard, replay viewer, adapter) must inflate
-  on the gzip magic and accept both `behavior_v1` and `behavior_v2` line formats. The
+  rejects ~35 MiB+ files. Every reader must go through **`js/runlog-format.js`**
+  (`readRunlogText` to inflate on the gzip magic, `createNormalizer().normalize(rec)` per
+  parsed line so `behavior_v2` `["a",…]` echoes become the v1 `arena_command` object) —
+  the dashboard uses an exact vendored copy at `dashboard/data-browser/vendor/`, and
+  `tests/test-runlog-format.js` fails when the copies diverge (re-copy after editing). The
   log level is a runtime setting (File ▾ → Run logging, localStorage `studio_log_level`,
   default `behavior_v2`); the runner asserts it via `log_control` and the bridge ACKS
   the level it will actually write (`bridge.waitForLogLevelAck`) — a pre-3.0 bridge

arena_studio.html

@@ -2918,7 +2918,7 @@ <h2 id="modalTitle">Import error</h2>
 </div>
 
 <footer id="footer">
-  <span class="foot-left">Arena Studio v0.72 | 2026-09-06 17:54 ET · <a href="https://github.com/reiserlab/webDisplayTools" target="_blank" rel="noopener" title="webDisplayTools on GitHub: source, issues, and release notes (docs/development/arena-studio-release-notes.md)">GitHub</a></span>
+  <span class="foot-left">Arena Studio v0.73 | 2026-09-06 18:29 ET · <a href="https://github.com/reiserlab/webDisplayTools" target="_blank" rel="noopener" title="webDisplayTools on GitHub: source, issues, and release notes (docs/development/arena-studio-release-notes.md)">GitHub</a></span>
   <!-- Course-repo quick-links: open protocols / logs / patterns in a new tab.
        Hrefs built from the configured repo + bench id (updateGhQuickLinks). -->
   <span id="ghQuickLinks" title="Open the course repo on GitHub (new tab)">
@@ -2950,7 +2950,8 @@ <h2 id="modalTitle">Import error</h2>
 <script src="js/studio-runlog-adapter.js"></script>
 <script src="js/kinematics.js"></script> <!-- shared scope/dashboard kinematic derivations -->
 <script src="js/studio-github.js"></script>
-<script src="js/runlog-replay.js?v=20260712-stackfix"></script>
+<script src="js/runlog-format.js?v=20260906-1829"></script>
+<script src="js/runlog-replay.js?v=20260906-1829"></script>
 <script src="js/runtime-controls.js"></script>
 <script src="js/arena-replay-viewer-protocol.js"></script>
 

dashboard/data-browser/README.md

@@ -197,6 +197,11 @@ value still pre-fills the local-server path.
 - `plot-specs.js`: protocol adapters and Plotly figure/CSV specifications
 - `github-repo.js`: PAT storage and private GitHub Contents API reads
 - `vendor/kinematics.js`: unchanged shared Arena Studio FicTrac math
+- `vendor/runlog-format.js`: exact copy of `webDisplayTools/js/runlog-format.js` — run-log
+  FILE decoding shared with the Studio's replay: gzip (`.jsonl.gz`, Studio v0.72+) and the
+  `behavior_v2` compact arena echoes, expanded to the v1 `arena_command` object at parse
+  time so every metric sees one shape. `tests/test-runlog-format.js` (repo root) fails if
+  the two copies diverge.
 
 ## Validation
 
@@ -209,6 +214,18 @@ node dashboard/data-browser/tests/test-github-client.js
 
 The analysis test parses live p0, p1, p2, current P3, and legacy P3 fixtures;
 validates stimulus alignment, P2 occupancy, P3 phase normalization, logged LED
-settings, and skipped-frame QC; builds every protocol page; and checks two-fly
-aggregation. The GitHub test verifies that the token appears only in the
-Authorization header.
+settings, and skipped-frame QC; builds every protocol page; checks two-fly
+aggregation; and re-reads the P3 fixture as `behavior_v2` + gzip asserting identical
+frames, events, preference indices and page CSV rows. The GitHub test verifies that
+the token appears only in the Authorization header.
+
+```bash
+node dashboard/data-browser/tests/corpus-v2-parity.js [/path/to/cshl-2026-course]
+```
+
+runs that v1-vs-v2.gz comparison over EVERY log in a course-repo clone (the
+behavior_v2 corpus gate; prints a Markdown table).
+
+Both `.jsonl` and `.jsonl.gz` open from the repo, a local server, a URL or a dropped
+file; the catalog shows the committed (compressed) size, with the inflated size and
+line format in the hover once a run is loaded.

dashboard/data-browser/analysis-core.js

@@ -4,6 +4,14 @@
     const K =
         global.Kinematics ||
         (typeof require === 'function' ? require('./vendor/kinematics.js') : null);
+    // Run-log FILE format (gzip + behavior_v1/v2 line formats): the shared
+    // js/runlog-format.js, vendored byte-identical like kinematics. parseJsonl
+    // takes TEXT — loaders inflate with F.readRunlogText first — and normalizes
+    // every line through F.createNormalizer() so behavior_v2's compact arena
+    // echoes reach every consumer as the v1 `arena_command` object.
+    const F =
+        global.RunlogFormat ||
+        (typeof require === 'function' ? require('./vendor/runlog-format.js') : null);
     const DEFAULT_BALL_DIAMETER_MM = 9;
     const DEFAULT_SMOOTH_WINDOW_S = 0.5;
     const ANALOG_OFF_FLOOR_MV = 4900;
@@ -175,7 +183,8 @@
 
     function parseFilename(sourceName) {
         const fileName = safeText(sourceName).split('/').pop() || 'runlog.jsonl';
-        const stem = fileName.replace(/\.jsonl$/i, '');
+        // `.jsonl.gz` (Studio v0.72+) and `.jsonl` share the same stem grammar.
+        const stem = fileName.replace(/\.gz$/i, '').replace(/\.jsonl$/i, '');
         const fields = stem.split('__');
         return {
             fileName,
@@ -719,13 +728,17 @@
         let schema = [];
         let metadata = {};
         let sessionStartMs = NaN;
+        const normalizer = F ? F.createNormalizer() : null;
 
         for (let index = 0; index < lines.length; index += 1) {
             const line = lines[index].trim();
             if (!line) continue;
             let rec;
             try {
                 rec = JSON.parse(line);
+                // behavior_v2: ["a", …] arena echoes → the v1 arena_command object
+                // (needs the v2 frame_schema's t0, which precedes them in the file).
+                if (normalizer) rec = normalizer.normalize(rec);
             } catch (error) {
                 parseErrors.push({ lineNumber: index + 1, message: error.message });
                 continue;
@@ -763,7 +776,10 @@
             events,
             steps,
             parseErrors,
-            sessionStartMs
+            sessionStartMs,
+            // 'behavior_v2' | 'behavior_v1' | 'full' | 'legacy' | 'unknown'
+            logFormat: normalizer ? normalizer.format : 'unknown',
+            rawBytes: text ? text.length : 0
         };
         deriveSignals(run, options);
         assignFramesToSteps(run);

dashboard/data-browser/app.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const A = window.DashboardAnalysis;
+const F = window.RunlogFormat; // run-log file format: gzip + behavior_v1/v2 (vendor/runlog-format.js)
 const P = window.DashboardPlots;
 const G = window.DashboardGitHub;
 const ANALYSIS_AXES_KEY = 'dashboard_analysis_axes';
@@ -501,7 +502,7 @@ const CATALOG_COLUMNS = [
         narrowHide: true,
         defaultHidden: true,
         cell: (d) =>
-            `<span class="run-size">${Number.isFinite(d.size) ? (d.size / 1048576).toFixed(1) + ' MB' : ''}</span>`,
+            `<span class="run-size" title="${escapeHtml(sizeTitle(d))}">${escapeHtml(sizeLabel(d))}</span>`,
         sort: (d) => d.size || 0
     }
 ];
@@ -631,6 +632,28 @@ function renderCatalog() {
     renderFocusOptions();
 }
 
+// Catalog size column: the committed size (gzip for `.jsonl.gz`); the hover adds
+// the inflated size once the run has been loaded.
+function formatBytes(n) {
+    if (!Number.isFinite(n) || n <= 0) return '';
+    return n >= 1e6 ? `${(n / 1e6).toFixed(1)} MB` : `${Math.max(1, Math.round(n / 1e3))} KB`;
+}
+function sizeLabel(descriptor) {
+    const label = formatBytes(descriptor.size);
+    if (!label) return '';
+    return /\.gz$/i.test(descriptor.path || descriptor.sourceName || '') ? `${label} gz` : label;
+}
+function sizeTitle(descriptor) {
+    const gz = /\.gz$/i.test(descriptor.path || descriptor.sourceName || '');
+    const run = state.runs.get(descriptor.key);
+    const parts = [];
+    if (Number.isFinite(descriptor.size))
+        parts.push(`${gz ? 'compressed (gzip)' : 'file'} size ${formatBytes(descriptor.size)}`);
+    if (run && run.rawBytes) parts.push(`${formatBytes(run.rawBytes)} of JSONL text`);
+    if (run && run.logFormat) parts.push(`format ${run.logFormat}`);
+    return parts.join(' · ');
+}
+
 function renderFocusOptions() {
     if (!state.catalog.length) {
         els.focusRunSelect.innerHTML = '<option value="">No runs available</option>';
@@ -716,7 +739,8 @@ async function ensureRun(descriptor) {
         setStatus('', `Fetching ${descriptor.runId}`);
         const response = await fetch(descriptor.url);
         if (!response.ok) throw new Error(`${response.status} ${response.statusText}`);
-        text = await response.text();
+        // bytes, not text: `.jsonl.gz` inflates on the gzip magic
+        text = await F.readRunlogText(new Uint8Array(await response.arrayBuffer()));
     } else {
         throw new Error(`No loader for ${descriptor.runId}`);
     }
@@ -729,11 +753,12 @@ async function loadFiles(files) {
     let firstKey = '';
     for (const file of files) {
         const key = `file:${file.name}:${file.size}:${file.lastModified}`;
-        const text = await file.text();
+        const text = await F.readRunlogText(file); // inflates `.jsonl.gz`
         const result = await parseAndAddText(text, file.name, {
             key,
             path: file.name,
-            sourceType: 'file'
+            sourceType: 'file',
+            size: file.size
         });
         if (!firstKey) firstKey = result.descriptor.key;
     }
@@ -745,7 +770,8 @@ async function loadUrl(url) {
     setStatus('', `Fetching ${url}`);
     const response = await fetch(url);
     if (!response.ok) throw new Error(`${response.status} ${response.statusText}`);
-    const text = await response.text();
+    const bytes = new Uint8Array(await response.arrayBuffer());
+    const text = await F.readRunlogText(bytes); // inflates `.jsonl.gz`
     const sourceName = decodeURIComponent(
         new URL(url, window.location.href).pathname.split('/').pop() || 'runlog.jsonl'
     );
@@ -754,7 +780,8 @@ async function loadUrl(url) {
         key,
         path: url,
         url,
-        sourceType: 'url'
+        sourceType: 'url',
+        size: bytes.length
     });
     await focusDescriptor(result.descriptor.key, true);
     setStatus('ok', `Loaded ${result.descriptor.runId}`);
@@ -836,7 +863,7 @@ async function browseGithub() {
     setStatus('', `Indexing ${state.github.selectedFolders.join(', ')}`);
     try {
         const directFiles = state.github.rootItems.filter(
-            (item) => item.type === 'file' && item.name.toLowerCase().endsWith('.jsonl')
+            (item) => item.type === 'file' && F.isRunlogName(item.name)
         );
         const directoryFiles = await G.mapLimit(directories, 4, async (directory, index) => {
             setStatus('', `Indexing runlog folders ${index + 1}/${directories.length}`);
@@ -846,7 +873,7 @@ async function browseGithub() {
         const files = [
             ...directFiles.map((item) => ({ ...item, rigFolder: 'runlogs root' })),
             ...directoryFiles.flat()
-        ].filter((item) => item.type === 'file' && item.name.toLowerCase().endsWith('.jsonl'));
+        ].filter((item) => item.type === 'file' && F.isRunlogName(item.name));
         // Per-folder index.json → start / duration / end state without downloading
         // logs (browsers can't Range-read a tail from GitHub — CORS preflight 403).
         const indexByFolder = new Map();
@@ -949,16 +976,29 @@ async function listDirectoryLinks(directoryUrl) {
 async function fetchUrlPrefix(url, bytes) {
     const response = await fetch(url, { headers: { Range: `bytes=0-${bytes - 1}` } });
     if (!response.ok) throw new Error(`${response.status} ${response.statusText}`);
-    if (!response.body || !response.body.getReader) return (await response.text()).slice(0, bytes);
+    if (!response.body || !response.body.getReader) {
+        const all = new Uint8Array(await response.arrayBuffer());
+        return (await F.readRunlogPrefixText(all.subarray(0, bytes))).slice(0, bytes);
+    }
+    // Stream bytes; a plain file stops at the run_metadata line, a `.jsonl.gz`
+    // prefix is inflated (truncation-tolerant) once the byte budget is read.
     const reader = response.body.getReader();
     const decoder = new TextDecoder();
+    const chunks = [];
+    let total = 0;
     let text = '';
+    let gzip = null;
     try {
-        while (text.length < bytes) {
+        while (total < bytes) {
             const part = await reader.read();
             if (part.done) break;
-            text += decoder.decode(part.value, { stream: true });
-            if (text.includes('"run_metadata"')) break;
+            chunks.push(part.value);
+            total += part.value.length;
+            if (gzip === null) gzip = F.isGzip(part.value);
+            if (!gzip) {
+                text += decoder.decode(part.value, { stream: true });
+                if (text.includes('"run_metadata"')) break;
+            }
         }
     } finally {
         try {
@@ -967,7 +1007,14 @@ async function fetchUrlPrefix(url, bytes) {
             /* already complete */
         }
     }
-    return text;
+    if (!gzip) return text;
+    const all = new Uint8Array(total);
+    let at = 0;
+    for (const c of chunks) {
+        all.set(c, at);
+        at += c.length;
+    }
+    return F.readRunlogPrefixText(all);
 }
 
 async function scanLocal() {
@@ -977,13 +1024,11 @@ async function scanLocal() {
     setStatus('', `Scanning ${base.pathname}`);
     try {
         const first = await listDirectoryLinks(base);
-        const direct = first.filter((url) => url.pathname.toLowerCase().endsWith('.jsonl'));
+        const direct = first.filter((url) => F.isRunlogName(url.pathname));
         const directories = first.filter((url) => url.pathname.endsWith('/'));
         const childFiles = await Promise.all(
             directories.map(async (directory) =>
-                (await listDirectoryLinks(directory)).filter((url) =>
-                    url.pathname.toLowerCase().endsWith('.jsonl')
-                )
+                (await listDirectoryLinks(directory)).filter((url) => F.isRunlogName(url.pathname))
             )
         );
         const files = [...direct, ...childFiles.flat()];
@@ -1953,8 +1998,10 @@ els.scopeAutoY.addEventListener('click', () => {
 document.body.addEventListener('dragover', (event) => event.preventDefault());
 document.body.addEventListener('drop', async (event) => {
     event.preventDefault();
-    const files = [...((event.dataTransfer && event.dataTransfer.files) || [])].filter((file) =>
-        /\.(jsonl|ndjson|json)$/i.test(file.name)
+    // Same acceptance as the file input (`accept=".jsonl,.ndjson,.json,…"`): a
+    // dropped `.json` is a user's choice; directory listings use F.isRunlogName alone.
+    const files = [...((event.dataTransfer && event.dataTransfer.files) || [])].filter(
+        (file) => F.isRunlogName(file.name) || /\.json$/i.test(file.name)
     );
     if (!files.length) return;
     try {

dashboard/data-browser/github-repo.js

@@ -8,6 +8,12 @@
     const BENCH_KEY = 'studio_bench_id';
     const FOLDERS_KEY_PREFIX = 'dashboard_runlog_folders:';
     const DEFAULT_REPO = 'reiserlab/cshl-2026-course';
+    // Run logs are committed as `.jsonl.gz` since Studio v0.72; fetchRaw reads
+    // BYTES and lets the shared format module inflate (magic-detected, so a raw
+    // `.jsonl` still works). Vendored byte-identical copy of js/runlog-format.js.
+    const F =
+        global.RunlogFormat ||
+        (typeof require === 'function' ? require('./vendor/runlog-format.js') : null);
 
     function currentToken() {
         return sessionStorage.getItem(TOKEN_KEY) || localStorage.getItem(TOKEN_KEY) || '';
@@ -175,21 +181,37 @@
         if (prefixBytes) requestHeaders.Range = `bytes=0-${Math.max(1023, prefixBytes - 1)}`;
         const response = await fetch(contentsUrl(repo, path, ref), { headers: requestHeaders });
         if (!response.ok) throw new Error(`GitHub HTTP ${response.status}`);
-        if (!prefixBytes || !response.body || !response.body.getReader) return response.text();
+        const inflateAll = (bytes) =>
+            F ? F.readRunlogText(bytes) : new TextDecoder().decode(bytes);
+        if (!prefixBytes || !response.body || !response.body.getReader) {
+            return inflateAll(new Uint8Array(await response.arrayBuffer()));
+        }
 
+        // Prefix read (catalog metadata): stream bytes until the budget is spent, or
+        // — for a plain-text file — until a complete run_metadata line is in hand.
+        // A gzip prefix cannot be scanned as text; it is inflated (truncation-
+        // tolerant) once the budget is read, which still yields the file's head.
         const reader = response.body.getReader();
         const decoder = new TextDecoder();
+        const chunks = [];
+        let total = 0;
         let text = '';
+        let gzip = null;
         try {
-            while (text.length < prefixBytes) {
+            while (total < prefixBytes) {
                 const part = await reader.read();
                 if (part.done) break;
-                text += decoder.decode(part.value, { stream: true });
-                if (
-                    text.includes('"run_metadata"') &&
-                    text.split(/\r?\n/).some((line) => line.includes('"run_metadata"'))
-                )
-                    break;
+                chunks.push(part.value);
+                total += part.value.length;
+                if (gzip === null) gzip = F ? F.isGzip(part.value) : false;
+                if (!gzip) {
+                    text += decoder.decode(part.value, { stream: true });
+                    if (
+                        text.includes('"run_metadata"') &&
+                        text.split(/\r?\n/).some((line) => line.includes('"run_metadata"'))
+                    )
+                        break;
+                }
             }
         } finally {
             try {
@@ -198,7 +220,14 @@
                 /* response may already be complete */
             }
         }
-        return text;
+        if (!gzip) return text;
+        const all = new Uint8Array(total);
+        let at = 0;
+        for (const c of chunks) {
+            all.set(c, at);
+            at += c.length;
+        }
+        return F.readRunlogPrefixText(all);
     }
 
     function fetchPrefix(repoValue, path, ref, bytes) {