chore(dev): pin tsim-bench with call sizes that fit a GPU

.claude/skills/benchmarking-tsim/SKILL.md

@@ -98,8 +98,9 @@ uv run tsim-dev bench --side .:threads=4 --side origin/main:threads=4
   freshly compiled ones (the cached case is what users get). A worker whose warmup
   compiled a kernel is started again.
 - The first 3 rounds' processes start at once, so their imports and compiles
-  overlap; each then times calls (calibrated once, about 0.3 s, best of 2) when its
-  round comes. With `--ref` the rounds stop after 3 when every round agrees on the
+  overlap; each then times calls (calibrated once per device, about 0.3 s, best of
+  2) when its round comes. A call returns at most about 256 MB of rows (bit-packed,
+  kept rows only), so a GPU's per-call overhead does not dominate. With `--ref` the rounds stop after 3 when every round agrees on the
   verdict, else they go on to `--rounds` (5).
 - `--thorough` runs each round's process from start to end on its own, every round,
   with 0.5 s calls (best of 3) and a prime run first: about 4x slower. Use it for the

dev/tsim_dev/bench.py

@@ -540,22 +540,29 @@ def _served_rounds(
     `MIN_ROUNDS` rounds' workers start at once, so their imports and compiles overlap
     (one pair at a time for wide circuits, to bound the memory); later rounds start
     their own pair. The first worker calibrates the call (shots per call, calls per
-    timing), and every round times the same calls. With two sides the rounds stop
-    after `MIN_ROUNDS` once more rounds could not change the verdict, else after
-    `args.rounds`.
+    timing), and every round times the same calls; a side on another device
+    calibrates its own. With two sides the rounds stop after `MIN_ROUNDS` once more
+    rounds could not change the verdict, else after `args.rounds`.
     """
     pairs: list[dict[str, Worker]] = []
     try:
         wide = case.tsim_width is not None and case.tsim_width > PARALLEL_WIDTH
         error = _start_pairs(case, sides, args, pairs, 1 if wide else MIN_ROUNDS)
         if error:
             return error
-        try:
-            first = pairs[0][sides[0].describe()]
-            call = first.request(f"calibrate {args.min_seconds}", "SHOTS")
-        except InfraError as error:
-            return f"{sides[0].describe()}: {error}"
-        request = f"time {call['shots']} {call['calls']} {REPEATS}"
+        requests: dict[str, str] = {}
+        calibrated: dict[str, str] = {}
+        for side in sides:
+            # NOTE: a GPU's call is far larger than a CPU's, so each device calibrates.
+            device = side.device or args.device
+            if device not in calibrated:
+                try:
+                    worker = pairs[0][side.describe()]
+                    call = worker.request(f"calibrate {args.min_seconds}", "SHOTS")
+                except InfraError as error:
+                    return f"{side.describe()}: {error}"
+                calibrated[device] = f"time {call['shots']} {call['calls']} {REPEATS}"
+            requests[side.describe()] = calibrated[device]
         for index in range(args.rounds):
             if index == len(pairs):
                 error = _start_pairs(case, sides, args, pairs, 1)
@@ -565,7 +572,9 @@ def _served_rounds(
             order = sides if index % 2 == 0 else sides[::-1]
             for side in order:
                 try:
-                    result = workers[side.describe()].request(request, "RESULT")
+                    result = workers[side.describe()].request(
+                        requests[side.describe()], "RESULT"
+                    )
                 except InfraError as error:
                     return f"{side.describe()}: {error}"
                 misses = workers[side.describe()].ready.get("cache_misses", 0)

dev/tsim_dev/corpus.py

@@ -24,7 +24,7 @@
 
 URL = "https://github.com/rafaelha/tsim-bench"
 # Bump together with any change that needs newer circuits or manifests.
-REV = "eb4a6d1140b19bcebf1d10f9248c5a8f4de239ef"
+REV = "19804f26c51ba42b9a7d7f53721685406eeff83c"
 COMPLETE = ".tsim-dev-complete"
 
 

test/unit/test_dev_cli.py

@@ -569,3 +569,9 @@ def fake_measure(side, case, args, mode="bench"):
     assert len(record["rounds"]["new"]) == 5
     # NOTE: every round has a pair of its own; rounds past the first batch start one.
     assert len(starts) == 2 * (6 + 2)
+    # NOTE: a side on another device calibrates its own call.
+    sides[0] = bench.Side("new", Path("python"), device="gpu")
+    rates.update(new=[1e6] * 5, ref=[1e6] * 5)
+    requests.clear()
+    bench.run_case(case, sides, args)
+    assert requests[:2] == [("new", "calibrate 0.3"), ("ref", "calibrate 0.3")]