From 0a317f856094eff855c3c320bbe3ec56d93d507c Mon Sep 17 00:00:00 2001 From: Rafael Haenel Date: Wed, 11 Nov 2026 10:13:12 -0700 Subject: [PATCH] chore(dev): pin tsim-bench with call sizes that fit a GPU tsim-bench's worker capped a call at 2^28 / outputs shots, ignoring bit-packing and postselection, so on a GPU the per-call overhead dominated (512M instead of 1.9G shots/s on end2end_d5_dsurface15_p0.001). The new pin bounds a call by the bytes it returns and warms the timed size. Served rounds now calibrate once per device, since a GPU's call would take minutes on a CPU. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/benchmarking-tsim/SKILL.md | 5 ++-- dev/tsim_dev/bench.py | 29 +++++++++++++++-------- dev/tsim_dev/corpus.py | 2 +- test/unit/test_dev_cli.py | 6 +++++ 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/.claude/skills/benchmarking-tsim/SKILL.md b/.claude/skills/benchmarking-tsim/SKILL.md index 9aa77409..e2dfb6de 100644 --- a/.claude/skills/benchmarking-tsim/SKILL.md +++ b/.claude/skills/benchmarking-tsim/SKILL.md @@ -98,8 +98,9 @@ uv run tsim-dev bench --side .:threads=4 --side origin/main:threads=4 freshly compiled ones (the cached case is what users get). A worker whose warmup compiled a kernel is started again. - The first 3 rounds' processes start at once, so their imports and compiles - overlap; each then times calls (calibrated once, about 0.3 s, best of 2) when its - round comes. With `--ref` the rounds stop after 3 when every round agrees on the + overlap; each then times calls (calibrated once per device, about 0.3 s, best of + 2) when its round comes. A call returns at most about 256 MB of rows (bit-packed, + kept rows only), so a GPU's per-call overhead does not dominate. With `--ref` the rounds stop after 3 when every round agrees on the verdict, else they go on to `--rounds` (5). - `--thorough` runs each round's process from start to end on its own, every round, with 0.5 s calls (best of 3) and a prime run first: about 4x slower. Use it for the diff --git a/dev/tsim_dev/bench.py b/dev/tsim_dev/bench.py index 6ecc5ed8..6dee368b 100644 --- a/dev/tsim_dev/bench.py +++ b/dev/tsim_dev/bench.py @@ -540,9 +540,9 @@ def _served_rounds( `MIN_ROUNDS` rounds' workers start at once, so their imports and compiles overlap (one pair at a time for wide circuits, to bound the memory); later rounds start their own pair. The first worker calibrates the call (shots per call, calls per - timing), and every round times the same calls. With two sides the rounds stop - after `MIN_ROUNDS` once more rounds could not change the verdict, else after - `args.rounds`. + timing), and every round times the same calls; a side on another device + calibrates its own. With two sides the rounds stop after `MIN_ROUNDS` once more + rounds could not change the verdict, else after `args.rounds`. """ pairs: list[dict[str, Worker]] = [] try: @@ -550,12 +550,19 @@ def _served_rounds( error = _start_pairs(case, sides, args, pairs, 1 if wide else MIN_ROUNDS) if error: return error - try: - first = pairs[0][sides[0].describe()] - call = first.request(f"calibrate {args.min_seconds}", "SHOTS") - except InfraError as error: - return f"{sides[0].describe()}: {error}" - request = f"time {call['shots']} {call['calls']} {REPEATS}" + requests: dict[str, str] = {} + calibrated: dict[str, str] = {} + for side in sides: + # NOTE: a GPU's call is far larger than a CPU's, so each device calibrates. + device = side.device or args.device + if device not in calibrated: + try: + worker = pairs[0][side.describe()] + call = worker.request(f"calibrate {args.min_seconds}", "SHOTS") + except InfraError as error: + return f"{side.describe()}: {error}" + calibrated[device] = f"time {call['shots']} {call['calls']} {REPEATS}" + requests[side.describe()] = calibrated[device] for index in range(args.rounds): if index == len(pairs): error = _start_pairs(case, sides, args, pairs, 1) @@ -565,7 +572,9 @@ def _served_rounds( order = sides if index % 2 == 0 else sides[::-1] for side in order: try: - result = workers[side.describe()].request(request, "RESULT") + result = workers[side.describe()].request( + requests[side.describe()], "RESULT" + ) except InfraError as error: return f"{side.describe()}: {error}" misses = workers[side.describe()].ready.get("cache_misses", 0) diff --git a/dev/tsim_dev/corpus.py b/dev/tsim_dev/corpus.py index 01ca8d2e..1f37cea1 100644 --- a/dev/tsim_dev/corpus.py +++ b/dev/tsim_dev/corpus.py @@ -24,7 +24,7 @@ URL = "https://github.com/rafaelha/tsim-bench" # Bump together with any change that needs newer circuits or manifests. -REV = "eb4a6d1140b19bcebf1d10f9248c5a8f4de239ef" +REV = "19804f26c51ba42b9a7d7f53721685406eeff83c" COMPLETE = ".tsim-dev-complete" diff --git a/test/unit/test_dev_cli.py b/test/unit/test_dev_cli.py index d0d13adf..1dc483f7 100644 --- a/test/unit/test_dev_cli.py +++ b/test/unit/test_dev_cli.py @@ -569,3 +569,9 @@ def fake_measure(side, case, args, mode="bench"): assert len(record["rounds"]["new"]) == 5 # NOTE: every round has a pair of its own; rounds past the first batch start one. assert len(starts) == 2 * (6 + 2) + # NOTE: a side on another device calibrates its own call. + sides[0] = bench.Side("new", Path("python"), device="gpu") + rates.update(new=[1e6] * 5, ref=[1e6] * 5) + requests.clear() + bench.run_case(case, sides, args) + assert requests[:2] == [("new", "calibrate 0.3"), ("ref", "calibrate 0.3")]