From f1a7867a4cbebb38e7668adfd1b2e9b4f4d9cfac Mon Sep 17 00:00:00 2001 From: Mark Saroufim Date: Sun, 26 Jul 2026 08:43:26 -0700 Subject: [PATCH 1/4] feat: show application validation badges --- frontend/src/api/api.ts | 7 ++ .../pages/leaderboard/Leaderboard.test.tsx | 56 ++++++++++++++ .../leaderboard/components/RankingLists.tsx | 77 ++++++++++++++++++- kernelboard/api/leaderboard.py | 36 ++++++++- tests/api/test_leaderboard_api.py | 48 ++++++++++++ tests/conftest.py | 3 +- tests/data.sql | 18 +++++ 7 files changed, 240 insertions(+), 5 deletions(-) diff --git a/frontend/src/api/api.ts b/frontend/src/api/api.ts index e0491f03..fcb88a7d 100644 --- a/frontend/src/api/api.ts +++ b/frontend/src/api/api.ts @@ -36,6 +36,13 @@ export interface LeaderboardDetail { user_name: string; submission_id: number; line_count?: number; + validation_status?: "completed" | "failed" | null; + validation_shapes_passed?: number | null; + validation_shapes_total?: number | null; + validation_fully_validated?: boolean | null; + validation_geomean_speedup?: number | null; + validation_contract_version?: string | null; + validation_checked_at?: string | null; }> >; } diff --git a/frontend/src/pages/leaderboard/Leaderboard.test.tsx b/frontend/src/pages/leaderboard/Leaderboard.test.tsx index 7ec63dfe..8c2e0cad 100644 --- a/frontend/src/pages/leaderboard/Leaderboard.test.tsx +++ b/frontend/src/pages/leaderboard/Leaderboard.test.tsx @@ -108,6 +108,62 @@ describe("Leaderboard", () => { expect(screen.getByText(/custom_kernel/)).toBeInTheDocument(); }); + it("shows full and partial application validation badges beside submitters", () => { + const mockData = { + deadline: mockDeadline, + description: mockDescription, + name: mockName, + reference: mockReference, + starter: mockStarter, + gpu_types: ["B200"], + rankings: { + B200: [ + { + file_name: "fast.py", + prev_score: 0, + rank: 1, + score: 3.25, + user_name: "stable-user", + submission_id: 101, + validation_status: "completed", + validation_shapes_passed: 8, + validation_shapes_total: 8, + validation_fully_validated: true, + validation_geomean_speedup: 1.4, + validation_contract_version: "v1", + }, + { + file_name: "partial.py", + prev_score: 0.1, + rank: 2, + score: 3.5, + user_name: "partial-user", + submission_id: 102, + validation_status: "completed", + validation_shapes_passed: 6, + validation_shapes_total: 8, + validation_fully_validated: false, + validation_geomean_speedup: 1.1, + validation_contract_version: "v1", + }, + ], + }, + }; + + (apiHook.fetcherApiCallback as ReturnType).mockReturnValue({ + data: mockData, + loading: false, + error: null, + errorStatus: null, + call: mockCall, + }); + + renderWithRouter(); + + expect(screen.getByText("VALIDATED")).toBeInTheDocument(); + expect(screen.getByText("6/8 VALIDATED")).toBeInTheDocument(); + }); + it("shows loading state", () => { (apiHook.fetcherApiCallback as ReturnType).mockReturnValue({ data: null, diff --git a/frontend/src/pages/leaderboard/components/RankingLists.tsx b/frontend/src/pages/leaderboard/components/RankingLists.tsx index 24b8b2cc..b45a5dea 100644 --- a/frontend/src/pages/leaderboard/components/RankingLists.tsx +++ b/frontend/src/pages/leaderboard/components/RankingLists.tsx @@ -2,8 +2,10 @@ import { useState } from "react"; import { Box, Button, + Chip, Grid, Stack, + Tooltip, type SxProps, type Theme, Typography, @@ -30,6 +32,13 @@ interface RankingItem { submission_count?: number; submission_time?: string; line_count?: number; + validation_status?: "completed" | "failed" | null; + validation_shapes_passed?: number | null; + validation_shapes_total?: number | null; + validation_fully_validated?: boolean | null; + validation_geomean_speedup?: number | null; + validation_contract_version?: string | null; + validation_checked_at?: string | null; } interface RankingsListProps { @@ -97,6 +106,58 @@ const styles: Record> = { }, }; +function ValidationBadge({ item }: { item: RankingItem }) { + if (!item.validation_status) return null; + + const passed = item.validation_shapes_passed ?? 0; + const total = item.validation_shapes_total ?? 0; + const fullyValidated = + item.validation_status === "completed" && + item.validation_fully_validated === true && + total > 0 && + passed === total; + const failed = item.validation_status === "failed"; + const label = fullyValidated + ? "VALIDATED" + : failed + ? "VALIDATION ERROR" + : `${passed}/${total} VALIDATED`; + const speedup = + typeof item.validation_geomean_speedup === "number" + ? ` Geomean synchronized-wall speedup across measured shapes: ${item.validation_geomean_speedup.toFixed(2)}×.` + : ""; + const contract = item.validation_contract_version + ? ` Contract: ${item.validation_contract_version}.` + : ""; + const checked = item.validation_checked_at + ? ` Checked ${new Date(item.validation_checked_at).toLocaleString()}.` + : ""; + const tooltip = fullyValidated + ? `All ${passed}/${total} training shapes passed the convergence, numerical, fallback, and speed gates.${speedup}${contract}${checked}` + : failed + ? `The validation job failed before it could complete.${contract}${checked}` + : `${passed}/${total} training shapes passed the convergence, numerical, fallback, and speed gates.${speedup}${contract}${checked}`; + + return ( + + + + ); +} + export default function RankingsList({ rankings, leaderboardId, @@ -205,9 +266,19 @@ export default function RankingsList({ {item.rank}. - - {item.user_name} {getMedalIcon(item.rank)} - + + + {item.user_name} {getMedalIcon(item.rank)} + + + diff --git a/kernelboard/api/leaderboard.py b/kernelboard/api/leaderboard.py index 4f0b4ab2..59b6434c 100644 --- a/kernelboard/api/leaderboard.py +++ b/kernelboard/api/leaderboard.py @@ -265,11 +265,38 @@ def _get_query(): s.file_name AS file_name, r.submission_id AS submission_id, COALESCE(sc.submission_count, 0) AS submission_count, + validation.status AS validation_status, + validation.passed_shapes AS validation_shapes_passed, + validation.total_shapes AS validation_shapes_total, + validation.fully_validated AS validation_fully_validated, + validation.geomean_sync_wall_speedup AS validation_geomean_speedup, + validation.contract_version AS validation_contract_version, + validation.checked_at AS validation_checked_at, RANK() OVER (PARTITION BY r.runner, u.id ORDER BY r.score ASC) AS rank FROM leaderboard.runs r JOIN leaderboard.submission s ON r.submission_id = s.id LEFT JOIN leaderboard.user_info u ON s.user_id = u.id LEFT JOIN submission_counts sc ON s.user_id = sc.user_id AND r.runner = sc.runner + LEFT JOIN LATERAL ( + SELECT + status, + passed_shapes, + total_shapes, + fully_validated, + geomean_sync_wall_speedup, + contract_version, + checked_at + FROM leaderboard.submission_validation + WHERE submission_id = s.id + AND gpu_type = r.runner + AND contract_version = ( + SELECT task->'validation'->>'version' + FROM leaderboard.leaderboard + WHERE id = %(leaderboard_id)s + ) + ORDER BY checked_at DESC + LIMIT 1 + ) validation ON TRUE WHERE NOT r.secret AND r.score IS NOT NULL AND r.passed @@ -312,7 +339,14 @@ def _get_query(): 'file_name', r.file_name, 'submission_id', r.submission_id, 'submission_count', r.submission_count, - 'submission_time', r.submission_time + 'submission_time', r.submission_time, + 'validation_status', r.validation_status, + 'validation_shapes_passed', r.validation_shapes_passed, + 'validation_shapes_total', r.validation_shapes_total, + 'validation_fully_validated', r.validation_fully_validated, + 'validation_geomean_speedup', r.validation_geomean_speedup, + 'validation_contract_version', r.validation_contract_version, + 'validation_checked_at', r.validation_checked_at ) ORDER BY r.score ASC ) diff --git a/tests/api/test_leaderboard_api.py b/tests/api/test_leaderboard_api.py index 7bcfadcb..ab46d8e6 100644 --- a/tests/api/test_leaderboard_api.py +++ b/tests/api/test_leaderboard_api.py @@ -49,6 +49,54 @@ def test_active_leaderboard_omits_line_counts(client, app): assert all("line_count" not in item for item in ranked_items) +def test_leaderboard_includes_latest_validation_summary(client, app): + initial = client.get("/api/leaderboard/339").get_json() + gpu_type, rankings = next(iter(initial["data"]["rankings"].items())) + submission_id = rankings[0]["submission_id"] + + with app.app_context(): + conn = get_db_connection() + with conn.cursor() as cur: + cur.execute( + """ + UPDATE leaderboard.leaderboard + SET task = jsonb_set( + task, + '{validation}', + '{"version": "v1"}'::jsonb + ) + WHERE id = 339 + """ + ) + cur.execute( + """ + INSERT INTO leaderboard.submission_validation ( + submission_id, gpu_type, contract_name, contract_version, + status, passed_shapes, total_shapes, fully_validated, + geomean_sync_wall_speedup, result + ) + VALUES (%s, %s, 'natural-gradient-training', 'v1', + 'completed', 8, 8, TRUE, 1.25, '{}') + """, + (submission_id, gpu_type), + ) + conn.commit() + + response = client.get("/api/leaderboard/339") + assert response.status_code == 200 + validated = next( + item + for item in response.get_json()["data"]["rankings"][gpu_type] + if item["submission_id"] == submission_id + ) + assert validated["validation_status"] == "completed" + assert validated["validation_shapes_passed"] == 8 + assert validated["validation_shapes_total"] == 8 + assert validated["validation_fully_validated"] is True + assert validated["validation_geomean_speedup"] == 1.25 + assert validated["validation_contract_version"] == "v1" + + def test_leaderboard_line_counts_support_bytea_code_storage(client, app): with app.app_context(): conn = get_db_connection() diff --git a/tests/conftest.py b/tests/conftest.py index 3170a8b2..d7e0d20e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import os import random import secrets import string @@ -131,7 +132,7 @@ def db_server(): "-f", "tests/data.sql", ], - env={"PGPASSWORD": password}, + env={**os.environ, "PGPASSWORD": password}, ) if result.returncode != 0: diff --git a/tests/data.sql b/tests/data.sql index c99155e1..ec31f12e 100644 --- a/tests/data.sql +++ b/tests/data.sql @@ -9219,6 +9219,24 @@ CREATE TABLE IF NOT EXISTS leaderboard.submission_job_status ( CONSTRAINT uq_submission_job_status_submission_id UNIQUE (submission_id) -- one-to-one with submission ); + +CREATE TABLE IF NOT EXISTS leaderboard.submission_validation ( + id BIGSERIAL PRIMARY KEY, + submission_id INTEGER NOT NULL + REFERENCES leaderboard.submission(id) ON DELETE CASCADE, + gpu_type TEXT NOT NULL, + contract_name TEXT NOT NULL, + contract_version TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('completed', 'failed')), + passed_shapes INTEGER NOT NULL DEFAULT 0, + total_shapes INTEGER NOT NULL DEFAULT 0, + fully_validated BOOLEAN NOT NULL DEFAULT FALSE, + geomean_sync_wall_speedup DOUBLE PRECISION, + result JSONB NOT NULL DEFAULT '{}'::jsonb, + error TEXT, + checked_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (submission_id, gpu_type, contract_version) +); -- -- PostgreSQL database dump complete -- From 0e77a5922bd9d34177d7bc0177a93b1c9ba0788a Mon Sep 17 00:00:00 2001 From: Mark Saroufim Date: Sun, 26 Jul 2026 09:05:08 -0700 Subject: [PATCH 2/4] fix: repair leaderboard validation coverage --- .../pages/leaderboard/Leaderboard.test.tsx | 25 +++++++++++-------- .../src/pages/leaderboard/Leaderboard.tsx | 2 +- .../leaderboard/components/RankingLists.tsx | 6 ++--- tests/api/test_leaderboard_api.py | 4 +-- tests/data.sql | 1 - 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/frontend/src/pages/leaderboard/Leaderboard.test.tsx b/frontend/src/pages/leaderboard/Leaderboard.test.tsx index 8c2e0cad..c409695f 100644 --- a/frontend/src/pages/leaderboard/Leaderboard.test.tsx +++ b/frontend/src/pages/leaderboard/Leaderboard.test.tsx @@ -9,6 +9,10 @@ vi.mock("../../lib/hooks/useApi", () => ({ fetcherApiCallback: vi.fn(), })); +vi.mock("./components/UserTrendChart", () => ({ + default: () =>
, +})); + // Mutable auth state for mocking useAuthStore per test type AuthState = { me: null | { authenticated: boolean; user?: { identity?: string } }; @@ -174,7 +178,7 @@ describe("Leaderboard", () => { }); renderWithRouter(); - expect(screen.getByText(/Summoning/i)).toBeInTheDocument(); + expect(screen.getByRole("progressbar")).toBeInTheDocument(); }); it("shows error message", () => { @@ -320,17 +324,18 @@ describe("Leaderboard", () => { const btn = screen.queryByTestId("ranking-show-all-button-0"); expect(btn).toBeInTheDocument(); - // By default only 3 rows shown - expect(screen.queryAllByTestId("ranking-0-row")).toHaveLength(3); - - // Click to show all - fireEvent.click(btn!); + // Rankings start expanded. expect(screen.queryAllByTestId("ranking-0-row")).toHaveLength(4); expect(within(btn!).getByText(/Hide/i)).toBeInTheDocument(); - // Click to hide again + // Hide the rows after the top three. fireEvent.click(btn!); expect(screen.queryAllByTestId("ranking-0-row")).toHaveLength(3); + expect(within(btn!).getByText(/Show all/i)).toBeInTheDocument(); + + // Expand again. + fireEvent.click(btn!); + expect(screen.queryAllByTestId("ranking-0-row")).toHaveLength(4); }); // -------------------- Starter codeblock -------------------- @@ -566,9 +571,9 @@ describe("Leaderboard", () => { // Switch to the Submission tab explicitly fireEvent.click(screen.getByRole("tab", { name: /Submission/i })); - const submit_btn = screen.getByTestId("leaderboard-submit-btn"); - expect(submit_btn).toBeInTheDocument(); - expect(submit_btn).toBeDisabled(); + expect( + screen.queryByTestId("leaderboard-submit-btn"), + ).not.toBeInTheDocument(); const deadline_txt = screen.getByTestId("deadline-passed-text"); expect( diff --git a/frontend/src/pages/leaderboard/Leaderboard.tsx b/frontend/src/pages/leaderboard/Leaderboard.tsx index 205a97e2..699910e6 100644 --- a/frontend/src/pages/leaderboard/Leaderboard.tsx +++ b/frontend/src/pages/leaderboard/Leaderboard.tsx @@ -178,8 +178,8 @@ const LeaderboardContent = memo(function LeaderboardContent() { findTopUsers(); }, [id, data?.rankings]); - if (loading || !data) return ; if (error) return ; + if (loading || !data) return ; if (!data) return null; const toDeadlineLocal = (raw: string) => { diff --git a/frontend/src/pages/leaderboard/components/RankingLists.tsx b/frontend/src/pages/leaderboard/components/RankingLists.tsx index b45a5dea..2fccfac1 100644 --- a/frontend/src/pages/leaderboard/components/RankingLists.tsx +++ b/frontend/src/pages/leaderboard/components/RankingLists.tsx @@ -175,7 +175,7 @@ export default function RankingsList({ const toggleExpanded = (field: string) => { setExpanded((prev) => ({ ...prev, - [field]: !prev[field], + [field]: !(prev[field] ?? true), })); }; @@ -293,7 +293,7 @@ export default function RankingsList({ - )} - + {showLineCount && ( diff --git a/tests/api/test_leaderboard_api.py b/tests/api/test_leaderboard_api.py index ab46d8e6..d1c627c1 100644 --- a/tests/api/test_leaderboard_api.py +++ b/tests/api/test_leaderboard_api.py @@ -71,11 +71,11 @@ def test_leaderboard_includes_latest_validation_summary(client, app): cur.execute( """ INSERT INTO leaderboard.submission_validation ( - submission_id, gpu_type, contract_name, contract_version, + submission_id, gpu_type, contract_version, status, passed_shapes, total_shapes, fully_validated, geomean_sync_wall_speedup, result ) - VALUES (%s, %s, 'natural-gradient-training', 'v1', + VALUES (%s, %s, 'v1', 'completed', 8, 8, TRUE, 1.25, '{}') """, (submission_id, gpu_type), diff --git a/tests/data.sql b/tests/data.sql index ec31f12e..94df9410 100644 --- a/tests/data.sql +++ b/tests/data.sql @@ -9225,7 +9225,6 @@ CREATE TABLE IF NOT EXISTS leaderboard.submission_validation ( submission_id INTEGER NOT NULL REFERENCES leaderboard.submission(id) ON DELETE CASCADE, gpu_type TEXT NOT NULL, - contract_name TEXT NOT NULL, contract_version TEXT NOT NULL, status TEXT NOT NULL CHECK (status IN ('completed', 'failed')), passed_shapes INTEGER NOT NULL DEFAULT 0, From 820996511749f1e7883aaf6282beef130afd6261 Mon Sep 17 00:00:00 2001 From: Mark Saroufim Date: Sun, 26 Jul 2026 09:13:43 -0700 Subject: [PATCH 3/4] fix: describe validation gates accurately --- frontend/src/pages/leaderboard/components/RankingLists.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/pages/leaderboard/components/RankingLists.tsx b/frontend/src/pages/leaderboard/components/RankingLists.tsx index 2fccfac1..957851e3 100644 --- a/frontend/src/pages/leaderboard/components/RankingLists.tsx +++ b/frontend/src/pages/leaderboard/components/RankingLists.tsx @@ -133,10 +133,10 @@ function ValidationBadge({ item }: { item: RankingItem }) { ? ` Checked ${new Date(item.validation_checked_at).toLocaleString()}.` : ""; const tooltip = fullyValidated - ? `All ${passed}/${total} training shapes passed the convergence, numerical, fallback, and speed gates.${speedup}${contract}${checked}` + ? `All ${passed}/${total} training shapes passed the convergence, numerical, and speed gates.${speedup}${contract}${checked}` : failed ? `The validation job failed before it could complete.${contract}${checked}` - : `${passed}/${total} training shapes passed the convergence, numerical, fallback, and speed gates.${speedup}${contract}${checked}`; + : `${passed}/${total} training shapes passed the convergence, numerical, and speed gates.${speedup}${contract}${checked}`; return ( From 816da9b96d69c45e5fdf6a6c569b08ad27f8e084 Mon Sep 17 00:00:00 2001 From: Mark Saroufim Date: Sun, 26 Jul 2026 09:26:07 -0700 Subject: [PATCH 4/4] test: keep unvalidated entries ranked --- tests/api/test_leaderboard_api.py | 40 ++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/api/test_leaderboard_api.py b/tests/api/test_leaderboard_api.py index d1c627c1..fce17306 100644 --- a/tests/api/test_leaderboard_api.py +++ b/tests/api/test_leaderboard_api.py @@ -51,6 +51,10 @@ def test_active_leaderboard_omits_line_counts(client, app): def test_leaderboard_includes_latest_validation_summary(client, app): initial = client.get("/api/leaderboard/339").get_json() + initial_ids = { + gpu: [item["submission_id"] for item in items] + for gpu, items in initial["data"]["rankings"].items() + } gpu_type, rankings = next(iter(initial["data"]["rankings"].items())) submission_id = rankings[0]["submission_id"] @@ -84,9 +88,14 @@ def test_leaderboard_includes_latest_validation_summary(client, app): response = client.get("/api/leaderboard/339") assert response.status_code == 200 + returned_rankings = response.get_json()["data"]["rankings"] + assert { + gpu: [item["submission_id"] for item in items] + for gpu, items in returned_rankings.items() + } == initial_ids validated = next( item - for item in response.get_json()["data"]["rankings"][gpu_type] + for item in returned_rankings[gpu_type] if item["submission_id"] == submission_id ) assert validated["validation_status"] == "completed" @@ -96,6 +105,35 @@ def test_leaderboard_includes_latest_validation_summary(client, app): assert validated["validation_geomean_speedup"] == 1.25 assert validated["validation_contract_version"] == "v1" + with app.app_context(): + conn = get_db_connection() + with conn.cursor() as cur: + cur.execute( + """ + UPDATE leaderboard.submission_validation + SET status = 'failed', + passed_shapes = 0, + total_shapes = 0, + fully_validated = FALSE + WHERE submission_id = %s AND gpu_type = %s + """, + (submission_id, gpu_type), + ) + conn.commit() + + failed_response = client.get("/api/leaderboard/339") + failed_rankings = failed_response.get_json()["data"]["rankings"] + assert { + gpu: [item["submission_id"] for item in items] + for gpu, items in failed_rankings.items() + } == initial_ids + failed = next( + item + for item in failed_rankings[gpu_type] + if item["submission_id"] == submission_id + ) + assert failed["validation_status"] == "failed" + def test_leaderboard_line_counts_support_bytea_code_storage(client, app): with app.app_context():