Skip to content

updating leaderboard ui to be comparison based #57

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
js2552 merged 1 commit into from
Aug 16, 2026
Merged

updating leaderboard ui to be comparison based #57

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 51 additions & 21 deletions scripts/generate-datasets.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,20 @@ function isFiniteNumber(value) {
return typeof value === 'number' && Number.isFinite(value);
}

// Mirrors isRecord in src/lib/performance.ts — keep in sync. `typeof x === 'object'` alone is
// true for arrays too, which previously let a stray `"finetune": []` be misclassified as a
// fine-tuned run here while the browser-side isRecord() correctly rejected it.
function isRecord(value) {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}

// Mirrors toText in src/lib/performance.ts — keep in sync.
function toText(value) {
if (typeof value !== 'string') return null;
const trimmed = value.trim();
return trimmed ? trimmed : null;
}

function resolveMetricKey(task, metrics) {
const candidates = TASK_METRIC_KEYS[task] ?? [];
for (const key of candidates) {
Expand Down Expand Up @@ -119,27 +133,39 @@ function computeCategoryScores(metrics) {
return { f1, map, precision, recall };
}

// Mirrors buildRunNote in src/lib/performance.ts — keep in sync. entry.notes carries the run's
// full methodology text (prompt used, parsing rules, metric definitions, etc.) straight from the
// benchmark script, appended after the generated summary line rather than discarded.
function buildRunNote(entry) {
const parts = [];
if (isFiniteNumber(entry.num_samples)) parts.push(`${entry.num_samples} samples`);
if (typeof entry.device === 'string' && entry.device.trim()) parts.push(entry.device.trim());
return parts.length ? parts.join(' · ') : null;
const parts = ['Zero-shot'];
if (isFiniteNumber(entry.num_samples)) parts.push(`evaluated on ${entry.num_samples} images`);
const summary = parts.join(' · ');
const detail = toText(entry.notes);
return detail ? `${summary}\n\n${detail}` : summary;
}

function buildFinetuneNote(finetune) {
if (!finetune || typeof finetune !== 'object') return null;
const parts = [];
// Mirrors buildFinetuneNote in src/lib/performance.ts — keep in sync.
function buildFinetuneNote(finetune, entryNotes) {
if (!isRecord(finetune)) return null;
const parts = ['Fine-tuned'];
if (isFiniteNumber(finetune.train_samples)) parts.push(`trained on ${finetune.train_samples} images from this dataset`);
if (isFiniteNumber(finetune.val_samples)) parts.push(`validated on ${finetune.val_samples} images`);
if (isFiniteNumber(finetune.epochs)) parts.push(`${finetune.epochs} epochs`);
if (isFiniteNumber(finetune.train_samples)) parts.push(`${finetune.train_samples} train samples`);
return parts.length ? parts.join(' · ') : null;
if (isFiniteNumber(finetune.lr)) parts.push(`lr=${finetune.lr}`);
if (isFiniteNumber(finetune.weight_decay)) parts.push(`weight decay=${finetune.weight_decay}`);
if (isFiniteNumber(finetune.split_seed)) parts.push(`seed=${finetune.split_seed}`);
if (isFiniteNumber(finetune.train_ratio)) parts.push(`train ratio=${finetune.train_ratio}`);
const summary = parts.join(' · ');
const detail = toText(entryNotes);
return detail ? `${summary}\n\n${detail}` : summary;
}

// Rendered as train/test/val percentages in that fixed order, e.g. "10/80/10" — mirrors
// buildSplitBreakdown in src/lib/performance.ts, kept in sync for the same reason as above.
function buildSplitBreakdown(entry, finetune) {
const testSamples = isFiniteNumber(entry.num_samples) ? entry.num_samples : null;
const trainSamples = finetune != null && typeof finetune === 'object' && isFiniteNumber(finetune.train_samples) ? finetune.train_samples : null;
const valSamples = finetune != null && typeof finetune === 'object' && isFiniteNumber(finetune.val_samples) ? finetune.val_samples : null;
const trainSamples = isRecord(finetune) && isFiniteNumber(finetune.train_samples) ? finetune.train_samples : null;
const valSamples = isRecord(finetune) && isFiniteNumber(finetune.val_samples) ? finetune.val_samples : null;
const total = (trainSamples ?? 0) + (testSamples ?? 0) + (valSamples ?? 0);
if (!total) return null;
const toShare = (value) => (value != null ? ((value / total) * 100).toFixed(0) : '-');
Expand All @@ -161,17 +187,17 @@ function makeLeaderboardRow(entry, metricKey, variant) {
categoryScores: computeCategoryScores(entry.metrics),
benchmarkId: isFiniteNumber(entry.benchmark_id) ? entry.benchmark_id : null,
variant,
date: typeof entry.timestamp === 'string' ? entry.timestamp.slice(0, 10) : null,
date: toText(entry.timestamp)?.slice(0, 10) ?? null,
submitted_by: null,
link: null,
notes: variant === 'fine-tuned' ? buildFinetuneNote(finetune) : buildRunNote(entry),
optimized: isOptimized(entry.optimized) || (finetune != null && typeof finetune === 'object' && isOptimized(finetune.optimized)),
platform: typeof entry.device === 'string' && entry.device.trim() ? entry.device.trim() : null,
notes: variant === 'fine-tuned' ? buildFinetuneNote(finetune, entry.notes) : buildRunNote(entry),
optimized: isOptimized(entry.optimized) || (isRecord(finetune) && isOptimized(finetune.optimized)),
platform: toText(entry.device),
splitBreakdown: buildSplitBreakdown(entry, finetune),
// dataset_config is the result set's own config field (raw / augmented) — no fallback to
// `split`, which is a different concept (the run's data split, e.g. "train"). Absent config
// means the run used the dataset's raw (unaugmented) form.
datasetConfig: typeof entry.dataset_config === 'string' && entry.dataset_config.trim() ? entry.dataset_config.trim() : 'raw',
datasetConfig: toText(entry.dataset_config) ?? 'raw',
};
}

Expand All @@ -190,7 +216,7 @@ function buildLeaderboardFromRawResults(rawResults) {
entry,
metricKey,
score: entry.metrics[metricKey],
isFinetune: entry.finetune != null && typeof entry.finetune === 'object',
isFinetune: isRecord(entry.finetune),
});
}

Expand Down Expand Up @@ -279,10 +305,14 @@ function buildGlobalPerformanceRecords(performanceDatasets, metadataLookup) {
machine_learning_task: meta.machine_learning_task,
benchmarkId: entry.benchmarkId ?? null,
variant: entry.variant === 'zero-shot' || entry.variant === 'fine-tuned' ? entry.variant : null,
optimized: Boolean(entry.optimized),
platform: typeof entry.platform === 'string' && entry.platform.trim() ? entry.platform.trim() : null,
splitBreakdown: entry.splitBreakdown ?? null,
datasetConfig: entry.datasetConfig ?? null,
// isOptimized(), not Boolean() — a pre-built {leaderboard:[...]} entry may still encode
// this as the raw "yes"/"no" string convention (see isOptimized above), and Boolean("no")
// is true.
optimized: isOptimized(entry.optimized),
platform: toText(entry.platform),
splitBreakdown: toText(entry.splitBreakdown),
datasetConfig: toText(entry.datasetConfig),
notes: toText(entry.notes),
});
});
}
Expand Down
60 changes: 13 additions & 47 deletions src/components/LeaderboardDetailModal.module.css
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,53 +29,6 @@
margin-bottom: 1.1rem;
}

.badgeRow {
display: flex;
gap: 0.4rem;
flex-wrap: wrap;
align-items: center;
margin-bottom: 0.5rem;
}

.taskBadge {
font-family: var(--ifm-code-font-family);
font-size: 0.65rem;
padding: 0.15rem 0.5rem;
border-radius: 4px;
display: inline-block;
white-space: nowrap;
}

.badgeClassification {
background: var(--agml-badge-classification-bg);
color: var(--agml-badge-classification-fg);
}

.badgeDetection {
background: var(--agml-badge-detection-bg);
color: var(--agml-badge-detection-fg);
}

.badgeSegmentation {
background: var(--agml-badge-segmentation-bg);
color: var(--agml-badge-segmentation-fg);
}

.badgeOther {
background: var(--agml-badge-other-bg);
color: var(--agml-badge-other-fg);
}

.resultTypeBadge {
font-family: var(--ifm-code-font-family);
font-size: 0.65rem;
padding: 0.15rem 0.5rem;
border-radius: 4px;
background: var(--agml-tag-bg);
color: var(--agml-tag-fg);
display: inline-block;
}

.title {
font-family: var(--ifm-code-font-family);
font-weight: 700;
Expand Down Expand Up @@ -243,3 +196,16 @@
line-height: 1.5;
color: var(--agml-muted);
}

.notesMeta {
margin: 0 0 0.4rem;
}

.notesText {
margin: 0;
color: var(--agml-text);
overflow-wrap: anywhere;
white-space: pre-wrap;
max-height: 220px;
overflow-y: auto;
}
98 changes: 34 additions & 64 deletions src/components/LeaderboardDetailModal.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,57 +1,29 @@
import { Fragment, useEffect, useState } from 'react';
import Link from '@docusaurus/Link';
import type { GlobalLeaderboardEntry } from '../lib/performance';
import { formatGlobalResultTypeKey, globalResultTypeKey, METRIC_LABELS } from '../lib/performance';
import type { ModelComparisonDetail } from '../lib/performance';
import { toDisplayLabel } from '../lib/labelOverrides';
import styles from './LeaderboardDetailModal.module.css';

function toLabel(value: string) {
return value.replace(/_/g, ' ');
}

// 1st, 2nd, 3rd, 4th, ..., 11th-13th stay "th" (the exception the mod-10 rule alone gets wrong).
function ordinal(value: number): string {
const rounded = Math.round(value);
const mod100 = rounded % 100;
if (mod100 >= 11 && mod100 <= 13) return `${rounded}th`;
switch (rounded % 10) {
case 1:
return `${rounded}st`;
case 2:
return `${rounded}nd`;
case 3:
return `${rounded}rd`;
default:
return `${rounded}th`;
}
}

function formatPercentile(value: number | null) {
return value == null ? null : `${ordinal(value)} pctl`;
function formatRank(rank: number | null, totalModels: number) {
return rank == null ? null : `#${rank} of ${totalModels}`;
}

function formatScore(value: number) {
return value.toFixed(3);
}

function formatResultType(entry: { variant: 'zero-shot' | 'fine-tuned' | null; optimized: boolean }) {
const base = entry.variant === 'fine-tuned' ? 'Fine-tuned' : entry.variant === 'zero-shot' ? 'Zero-shot' : '—';
if (base === '—') return base;
return entry.optimized ? `${base} (optimized)` : base;
}

function taskBadgeClass(task: string | null): string {
if (!task) return styles.badgeOther;
if (task.includes('classif')) return styles.badgeClassification;
if (task.includes('detect')) return styles.badgeDetection;
if (task.includes('segment')) return styles.badgeSegmentation;
return styles.badgeOther;
const key = globalResultTypeKey(entry);
return key ? formatGlobalResultTypeKey(key) : '—';
}

export function LeaderboardDetailModal({
entry,
detail,
open,
onClose,
}: {
entry: GlobalLeaderboardEntry | null;
detail: ModelComparisonDetail | null;
open: boolean;
onClose: () => void;
}) {
Expand All @@ -73,7 +45,7 @@ export function LeaderboardDetailModal({
}, [open, onClose]);

const [expandedKeys, setExpandedKeys] = useState<Set<string>>(new Set());
useEffect(() => setExpandedKeys(new Set()), [entry]);
useEffect(() => setExpandedKeys(new Set()), [detail]);
const toggleExpanded = (key: string) => {
setExpandedKeys((current) => {
const next = new Set(current);
Expand All @@ -83,7 +55,7 @@ export function LeaderboardDetailModal({
});
};

if (!open || entry == null) return null;
if (!open || detail == null) return null;

return (
<div className={styles.backdrop} role="presentation" onClick={onClose}>
Expand All @@ -97,35 +69,30 @@ export function LeaderboardDetailModal({
<div className={styles.header}>
<div>
<h2 id="leaderboard-detail-title" className={styles.title}>
{entry.model}
{detail.model}
{detail.task ? ` (${toDisplayLabel(detail.task)})` : ''}
</h2>
<div className={styles.badgeRow}>
<span className={`${styles.taskBadge} ${taskBadgeClass(entry.machineLearningTask)}`}>
{entry.machineLearningTask ? toLabel(entry.machineLearningTask) : 'Unknown task'}
</span>
<span className={styles.resultTypeBadge}>{entry.resultType}</span>
</div>
<p className={styles.summaryLine}>
{entry.appearances} result{entry.appearances === 1 ? '' : 's'} across {entry.datasets.length} dataset
{entry.datasets.length === 1 ? '' : 's'}
Ranked by {METRIC_LABELS[detail.metric]} across {detail.datasets.length} dataset
{detail.datasets.length === 1 ? '' : 's'} shared with all {detail.totalModels} compared models
</p>
</div>
<button type="button" className={styles.closeButton} onClick={onClose} aria-label="Close model details">
×
</button>
</div>

<p className={styles.sectionTitle}>Datasets included ({entry.datasetDetails.length})</p>
<p className={styles.sectionTitle}>Datasets contributing to this average ({detail.datasets.length})</p>
<div className={styles.tableWrap}>
<div className={styles.table} role="table">
<div className={styles.tableRow} role="row">
<span role="columnheader">Dataset</span>
<span role="columnheader">Split %</span>
<span role="columnheader">Config</span>
<span role="columnheader">Scores (percentile)</span>
<span role="columnheader">Scores (rank)</span>
</div>
{entry.datasetDetails.map((detail, index) => {
const rowKey = `${detail.dataset}-${index}`;
{detail.datasets.map((datasetDetail, index) => {
const rowKey = `${datasetDetail.dataset}-${index}`;
const isExpanded = expandedKeys.has(rowKey);
const scoreLines = (
[
Expand All @@ -136,12 +103,12 @@ export function LeaderboardDetailModal({
] as [string, 'f1' | 'map' | 'precision' | 'recall'][]
)
.map(([label, category]) => {
const score = detail.scores[category];
const pctl = detail.percentiles[category];
if (score == null || pctl == null) return null;
return [label, formatScore(score), formatPercentile(pctl)] as [string, string, string];
const score = datasetDetail.scores[category];
const rank = formatRank(datasetDetail.ranks[category], detail.totalModels);
if (score == null) return null;
return [label, formatScore(score), rank] as [string, string, string | null];
})
.filter((line): line is [string, string, string] => line != null);
.filter((line): line is [string, string, string | null] => line != null);

return (
<Fragment key={rowKey}>
Expand All @@ -161,34 +128,34 @@ export function LeaderboardDetailModal({
<span role="cell">
<div className={styles.datasetCell}>
<button type="button" className={styles.datasetName}>
{toLabel(detail.dataset)}
{toDisplayLabel(datasetDetail.dataset)}
<span className={styles.expandChevron} aria-hidden>
{isExpanded ? '▲' : '▾'}
</span>
</button>
<Link
className={styles.viewDatasetLink}
to={`/datasets?dataset=${encodeURIComponent(detail.dataset)}`}
to={`/datasets?dataset=${encodeURIComponent(datasetDetail.dataset)}`}
onClick={(event) => event.stopPropagation()}
>
view dataset
</Link>
</div>
</span>
<span role="cell" className={styles.simpleCell}>
{detail.splitBreakdown ?? '—'}
{datasetDetail.splitBreakdown ?? '—'}
</span>
<span role="cell" className={styles.simpleCell}>
{detail.datasetConfig ?? '—'}
{datasetDetail.datasetConfig ?? '—'}
</span>
<span role="cell">
{scoreLines.length === 0 ? (
'—'
) : (
<div className={styles.scoresCell}>
{scoreLines.map(([label, score, pctl]) => (
{scoreLines.map(([label, score, rank]) => (
<span key={label}>
{label} {score} <span>({pctl})</span>
{label} {score} {rank && <span>({rank})</span>}
</span>
))}
</div>
Expand All @@ -197,7 +164,10 @@ export function LeaderboardDetailModal({
</div>
{isExpanded && (
<div className={styles.notesRow}>
{formatResultType(detail)} · {detail.platform ?? 'unknown platform'}
<p className={styles.notesMeta}>
{formatResultType(datasetDetail)} · {datasetDetail.platform ?? 'unknown platform'}
</p>
<p className={styles.notesText}>{datasetDetail.notes ?? 'No additional notes for this result.'}</p>
</div>
)}
</Fragment>
Expand Down
Loading
Loading