diff --git a/dirigo-results/0.2.13-port/intercept/trace/refilldefer/ladder-defer.json b/dirigo-results/0.2.13-port/intercept/trace/refilldefer/ladder-defer.json new file mode 100644 index 00000000000..3125c00be81 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/trace/refilldefer/ladder-defer.json @@ -0,0 +1,272 @@ +[ + { + "target": 75, + "ttft_ms": 553.4, + "wall_ms": 621.8, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 532, + "prefill_tps": 236.8, + "decode_ms": 87, + "park_tokens": 96, + "park_ms": 98.6, + "kv_hit_pct": 93.1, + "kv_mib_from_ram": 1.2, + "server_ms": 21.4, + "label": "defer" + }, + { + "target": 75, + "ttft_ms": 537.2, + "wall_ms": 606.1, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 516, + "prefill_tps": 244.1, + "decode_ms": 87, + "park_tokens": 133, + "park_ms": 96.7, + "kv_hit_pct": 93.0, + "kv_mib_from_ram": 1.6, + "server_ms": 21.2, + "label": "defer" + }, + { + "target": 75, + "ttft_ms": 3068.7, + "wall_ms": 16588.4, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 356, + "prefill_tps": 354.3, + "decode_ms": 16230, + "park_tokens": 133, + "park_ms": 99.9, + "server_ms": 2712.7, + "label": "defer" + }, + { + "target": 75, + "ttft_ms": 2943.6, + "wall_ms": 16430.2, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 250, + "prefill_tps": 503.9, + "decode_ms": 16177, + "kv_hit_pct": 92.64, + "kv_mib_from_ram": 1.6, + "server_ms": 2693.6, + "label": "defer" + }, + { + "target": 75, + "ttft_ms": 2947.1, + "wall_ms": 19129.9, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 252, + "prefill_tps": 500.6, + "decode_ms": 18875, + "kv_hit_pct": 92.64, + "kv_mib_from_ram": 1.6, + "server_ms": 2695.1, + "label": "defer" + }, + { + "target": 75, + "ttft_ms": 2947.4, + "wall_ms": 16441.6, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 252, + "prefill_tps": 500.1, + "decode_ms": 16187, + "kv_hit_pct": 92.64, + "kv_mib_from_ram": 1.6, + "server_ms": 2695.4, + "label": "defer" + }, + { + "target": 512, + "ttft_ms": 3246.0, + "wall_ms": 16727.6, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 553, + "prefill_tps": 824.8, + "decode_ms": 16171, + "server_ms": 2693.0, + "label": "defer" + }, + { + "target": 512, + "ttft_ms": 3124.4, + "wall_ms": 16618.9, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 431, + "prefill_tps": 1058.8, + "decode_ms": 16185, + "kv_hit_pct": 92.76, + "kv_mib_from_ram": 5.7, + "server_ms": 2693.4, + "label": "defer" + }, + { + "target": 512, + "ttft_ms": 3130.4, + "wall_ms": 16596.6, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 433, + "prefill_tps": 1053.9, + "decode_ms": 16161, + "kv_hit_pct": 93.68, + "kv_mib_from_ram": 5.7, + "server_ms": 2697.4, + "label": "defer" + }, + { + "target": 512, + "ttft_ms": 3126.6, + "wall_ms": 16609.1, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 432, + "prefill_tps": 1054.6, + "decode_ms": 16173, + "kv_hit_pct": 92.76, + "kv_mib_from_ram": 5.7, + "server_ms": 2694.6, + "label": "defer" + }, + { + "target": 512, + "ttft_ms": 3125.2, + "wall_ms": 16611.9, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 431, + "prefill_tps": 1057.3, + "decode_ms": 16177, + "kv_hit_pct": 92.76, + "kv_mib_from_ram": 5.7, + "server_ms": 2694.2, + "label": "defer" + }, + { + "target": 512, + "ttft_ms": 3120.7, + "wall_ms": 16599.3, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 430, + "prefill_tps": 1060.6, + "decode_ms": 16166, + "kv_hit_pct": 93.68, + "kv_mib_from_ram": 5.7, + "server_ms": 2690.7, + "label": "defer" + }, + { + "target": 2048, + "ttft_ms": 4033.7, + "wall_ms": 20174.9, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1338, + "prefill_tps": 1201.5, + "decode_ms": 18831, + "kv_hit_pct": 91.65, + "kv_mib_from_ram": 19.5, + "server_ms": 2695.7, + "label": "defer" + }, + { + "target": 2048, + "ttft_ms": 3944.0, + "wall_ms": 20094.5, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1243, + "prefill_tps": 1293.7, + "decode_ms": 18846, + "kv_hit_pct": 91.65, + "kv_mib_from_ram": 19.5, + "server_ms": 2701.0, + "label": "defer" + }, + { + "target": 2048, + "ttft_ms": 3954.1, + "wall_ms": 20126.0, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1255, + "prefill_tps": 1281.0, + "decode_ms": 18865, + "server_ms": 2699.1, + "label": "defer" + }, + { + "target": 2048, + "ttft_ms": 3952.1, + "wall_ms": 20117.4, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1252, + "prefill_tps": 1284.4, + "decode_ms": 18860, + "kv_hit_pct": 91.65, + "kv_mib_from_ram": 19.5, + "server_ms": 2700.1, + "label": "defer" + }, + { + "target": 2048, + "ttft_ms": 3955.1, + "wall_ms": 20125.0, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1258, + "prefill_tps": 1278.0, + "decode_ms": 18861, + "kv_hit_pct": 91.65, + "kv_mib_from_ram": 19.5, + "server_ms": 2697.1, + "label": "defer" + }, + { + "target": 2048, + "ttft_ms": 3947.1, + "wall_ms": 20109.5, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1255, + "prefill_tps": 1281.6, + "decode_ms": 18850, + "kv_hit_pct": 91.65, + "kv_mib_from_ram": 19.5, + "server_ms": 2692.1, + "label": "defer" + } +] \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/trace/refilldefer/ladder-refill.json b/dirigo-results/0.2.13-port/intercept/trace/refilldefer/ladder-refill.json new file mode 100644 index 00000000000..8034145bf98 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/trace/refilldefer/ladder-refill.json @@ -0,0 +1,262 @@ +[ + { + "target": 75, + "ttft_ms": 555.4, + "wall_ms": 623.6, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 534, + "prefill_tps": 235.9, + "decode_ms": 87, + "park_tokens": 96, + "park_ms": 97.9, + "server_ms": 21.4, + "label": "refill" + }, + { + "target": 75, + "ttft_ms": 538.0, + "wall_ms": 606.6, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 517, + "prefill_tps": 243.8, + "decode_ms": 87, + "park_tokens": 133, + "park_ms": 96.8, + "server_ms": 21.0, + "label": "refill" + }, + { + "target": 75, + "ttft_ms": 449.1, + "wall_ms": 517.2, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 428, + "prefill_tps": 294.6, + "decode_ms": 87, + "park_tokens": 133, + "park_ms": 97.3, + "kv_hit_pct": 93.0, + "kv_mib_from_ram": 1.6, + "server_ms": 21.1, + "label": "refill" + }, + { + "target": 75, + "ttft_ms": 347.1, + "wall_ms": 415.3, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 326, + "prefill_tps": 386.9, + "decode_ms": 87, + "kv_hit_pct": 90.58, + "kv_mib_from_ram": 1.6, + "server_ms": 21.1, + "label": "refill" + }, + { + "target": 75, + "ttft_ms": 346.6, + "wall_ms": 445.2, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 325, + "prefill_tps": 387.4, + "decode_ms": 118, + "kv_hit_pct": 90.58, + "kv_mib_from_ram": 1.6, + "server_ms": 21.6, + "label": "refill" + }, + { + "target": 75, + "ttft_ms": 347.0, + "wall_ms": 415.3, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 326, + "prefill_tps": 387.0, + "decode_ms": 87, + "server_ms": 21.0, + "label": "refill" + }, + { + "target": 512, + "ttft_ms": 590.1, + "wall_ms": 693.7, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 567, + "prefill_tps": 803.7, + "decode_ms": 123, + "kv_hit_pct": 90.58, + "kv_mib_from_ram": 1.6, + "server_ms": 23.1, + "label": "refill" + }, + { + "target": 512, + "ttft_ms": 539.2, + "wall_ms": 607.0, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 516, + "prefill_tps": 883.1, + "decode_ms": 87, + "kv_hit_pct": 92.27, + "kv_mib_from_ram": 5.6, + "server_ms": 23.2, + "label": "refill" + }, + { + "target": 512, + "ttft_ms": 536.1, + "wall_ms": 603.9, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 513, + "prefill_tps": 888.1, + "decode_ms": 87, + "kv_hit_pct": 90.86, + "kv_mib_from_ram": 5.6, + "server_ms": 23.1, + "label": "refill" + }, + { + "target": 512, + "ttft_ms": 539.1, + "wall_ms": 606.9, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 516, + "prefill_tps": 883.1, + "decode_ms": 87, + "kv_hit_pct": 90.86, + "kv_mib_from_ram": 5.6, + "server_ms": 23.1, + "label": "refill" + }, + { + "target": 512, + "ttft_ms": 536.3, + "wall_ms": 627.2, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 514, + "prefill_tps": 887.9, + "decode_ms": 110, + "server_ms": 22.3, + "label": "refill" + }, + { + "target": 512, + "ttft_ms": 538.1, + "wall_ms": 629.4, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 515, + "prefill_tps": 884.7, + "decode_ms": 111, + "server_ms": 23.1, + "label": "refill" + }, + { + "target": 2048, + "ttft_ms": 1486.5, + "wall_ms": 1572.3, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1459, + "prefill_tps": 1102.5, + "decode_ms": 108, + "kv_hit_pct": 92.27, + "kv_mib_from_ram": 5.6, + "server_ms": 27.5, + "label": "refill" + }, + { + "target": 2048, + "ttft_ms": 1465.7, + "wall_ms": 1551.2, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1439, + "prefill_tps": 1117.7, + "decode_ms": 108, + "kv_hit_pct": 90.89, + "kv_mib_from_ram": 19.5, + "server_ms": 26.7, + "label": "refill" + }, + { + "target": 2048, + "ttft_ms": 1465.9, + "wall_ms": 1551.4, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1438, + "prefill_tps": 1117.9, + "decode_ms": 108, + "server_ms": 27.9, + "label": "refill" + }, + { + "target": 2048, + "ttft_ms": 1452.6, + "wall_ms": 1521.3, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1425, + "prefill_tps": 1128.4, + "decode_ms": 90, + "server_ms": 27.6, + "label": "refill" + }, + { + "target": 2048, + "ttft_ms": 1476.1, + "wall_ms": 1544.9, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1449, + "prefill_tps": 1109.8, + "decode_ms": 90, + "kv_hit_pct": 90.89, + "kv_mib_from_ram": 19.5, + "server_ms": 27.1, + "label": "refill" + }, + { + "target": 2048, + "ttft_ms": 1469.2, + "wall_ms": 1537.9, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1442, + "prefill_tps": 1115.3, + "decode_ms": 90, + "server_ms": 27.2, + "label": "refill" + } +] \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/trace/refilldefer/patch-STRATA_REFILL_DEFER.diff b/dirigo-results/0.2.13-port/intercept/trace/refilldefer/patch-STRATA_REFILL_DEFER.diff new file mode 100644 index 00000000000..39093b39df5 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/trace/refilldefer/patch-STRATA_REFILL_DEFER.diff @@ -0,0 +1,14 @@ +--- a/sycl/src/program/generate.cpp ++++ b/sycl/src/program/generate.cpp +@@ -11264,7 +11264,13 @@ (in the serve prompt path, after the read loop) + } +- if (!refill(err)) { ++ // STRATA_REFILL_DEFER=1: keep the slots the prompt lent instead of handing them back before the first ++ // token. The refill is one PCIe copy per lent slot (0.319 ms/slot: 72 ms at 127 tokens, 188 ms at 1.6 k) ++ // and it runs inside prompt_ms, so the first token waits for it - while the decode that follows needs ++ // exactly the experts the prompt just put there. OFF by default. ++ static const bool refill_defer = std::getenv("STRATA_REFILL_DEFER") != nullptr; ++ if (!refill_defer && !refill(err)) { + std::printf("ERR refilling a lent slot failed: %s\n", err.c_str()); + return 1; + } diff --git a/dirigo-results/0.2.13-port/intercept/trace/refilldefer/strata-refilldefer-ab.sh b/dirigo-results/0.2.13-port/intercept/trace/refilldefer/strata-refilldefer-ab.sh new file mode 100755 index 00000000000..7764589edc0 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/trace/refilldefer/strata-refilldefer-ab.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# strata-refilldefer-ab.sh - does skipping the post-prompt slot refill cost anything? +# +# The refill hands the lent expert slots back before the first token (0.319 ms/slot, 72-188 ms). STRATA_REFILL_DEFER=1 +# skips it, leaving the prompt's experts in the cache - which is what the decode that follows wants. The risk is the +# NEXT request: its prompt lends new slots and may find fewer of its own resident. +# +# So the ladder runs six samples per size in one window: the first sample pays for the previous request's leftovers, and +# a rising trend across samples is that cost; comparison between arms at the same sample index is the benefit. +# +# usage: strata-refilldefer-ab.sh TAG [--defer] +# UNDO: rm this file and ~/strata-refilldefer-ab +set -uo pipefail +TAG=${1:?tag} +shift || true +DEFER="" +for a in "$@"; do [ "$a" = "--defer" ] && DEFER=1; done +S=/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731 +TREE=$S/strata-0.2.13 +CONF=$TREE/strata-coder-iq1_m-sycl-tuned.json +OUT=$HOME/strata-refilldefer-ab +mkdir -p "$OUT" + +echo "[$(date +%H:%M:%S)] ===== refilldefer-$TAG defer=${DEFER:-0}" +env STRATA_TRACE=1 ${DEFER:+STRATA_REFILL_DEFER=1} \ + ~/bin/strata-card-window.sh "$TREE" 8097 --conf "$CONF" -- bash -c " + LOG=\$(python3 -c \"import json;print(json.load(open('$CONF'))['log'])\") + curl -s -m 5 http://127.0.0.1:\$PORT/v1/status | python3 -c 'import json,sys;d=json.load(sys.stdin);print(\" engine:\",d[\"engine\"])' + BEFORE=\$(stat -c%s \"\$LOG\") + /usr/bin/python3 \$HOME/bin/strata-intercept-probe.py --port \$PORT --log \"\$LOG\" --out $OUT/ladder-$TAG.json \ + --label $TAG --sizes 75,512,2048 --samples 6 + tail -c +\$((BEFORE + 1)) \"\$LOG\" > $OUT/engine-$TAG.log +" 2>&1 | grep -aE '=====|engine:|wrote|restore' +echo "[$(date +%H:%M:%S)] done" diff --git a/dirigo-results/0.2.13-port/intercept/trace/refilldefer/strata-refilldefer-analyse.py b/dirigo-results/0.2.13-port/intercept/trace/refilldefer/strata-refilldefer-analyse.py new file mode 100644 index 00000000000..804ae66459b --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/trace/refilldefer/strata-refilldefer-analyse.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Compare two refilldefer arms: TTFT, the refill line, and decode speed. + + ~/bin/strata-refilldefer-analyse.py DIR TAG_A TAG_B + +Each arm is a ladder JSON (from strata-intercept-probe.py) plus the engine log slice its window wrote. The interesting +question is not only the median: with STRATA_REFILL_DEFER the cache keeps the prompt's experts, so the NEXT request may +pay for it. That shows up as a rising trend across the six samples per size in one arm and not the other. +""" +import json +import re +import statistics as st +import sys +from collections import defaultdict + +RE_PROMPT = re.compile(r"strata serve: prompt (\d+) tokens = (\d+) reused \+ (\d+) read in (\d+) ms \(([0-9.]+) tok/s\), " + r"(\d+) generated in ([\d.]+) ms \(([0-9.]+) tok/s\)") +RE_REFILL = re.compile(r"strata trace: refilled (\d+) slots on \d+ stage\(s\) in ([\d.]+) ms") + + +def load(dirpath, tag): + rows = json.load(open(f"{dirpath}/ladder-{tag}.json")) + # pair the engine log's prompt lines with the ladder rows in order + gen = [] + try: + for line in open(f"{dirpath}/engine-{tag}.log", errors="replace"): + m = RE_PROMPT.search(line) + if m: + gen.append(dict(tokens=int(m.group(1)), read_n=int(m.group(3)), prefill_ms=float(m.group(4)), + decode_tok_s=float(m.group(8)), generated=int(m.group(6)))) + m = RE_REFILL.search(line) + if m and gen: + gen[-1]["refill_ms"] = float(m.group(2)) + gen[-1]["refill_slots"] = int(m.group(1)) + except FileNotFoundError: + pass + for i, r in enumerate(rows): + if i < len(gen): + r["decode_tok_s"] = gen[i].get("decode_tok_s") + r["refill_ms"] = gen[i].get("refill_ms") + r["refill_slots"] = gen[i].get("refill_slots") + return rows + + +def report(title, rows): + by = defaultdict(list) + for r in rows: + by[r["prompt_tokens"]].append(r) + print(f"\n{title}") + print(f" {'tokens':>7} {'n':>3} {'ttft med':>9} {'ttft by sample (s)':<28} {'refill':>7} {'decode tok/s':>13}") + out = {} + for size in sorted(by): + g = by[size] + seq = " ".join(f"{r['ttft_ms']/1000:.3f}" for r in g) + ref = [r["refill_ms"] for r in g if r.get("refill_ms") is not None] + dec = [r["decode_tok_s"] for r in g if r.get("decode_tok_s")] + ref_txt = f"{st.median(ref):.1f}" if ref else "none" + dec_txt = f"{st.median(dec):.1f}" if dec else "-" + out[size] = dict(ttft=st.median([r["ttft_ms"] for r in g]), + refill=st.median(ref) if ref else None, + decode=st.median(dec) if dec else None, seq=seq) + print(f" {size:>7} {len(g):>3} {out[size]['ttft']:>9.0f} {seq:<28} {ref_txt:>7} {dec_txt:>13}") + return out + + +def main(): + d, a, b = sys.argv[1], sys.argv[2], sys.argv[3] + ra, rb = load(d, a), load(d, b) + A, B = report(f"{a} (refill kept)", ra), report(f"{b}", rb) + print("\nper-size medians") + print(f" {'tokens':>7} {'A ttft':>8} {'B ttft':>8} {'delta':>8} {'A refill':>9} {'B refill':>9} {'A dec':>7} {'B dec':>7}") + for size in sorted(set(A) & set(B)): + x, y = A[size], B[size] + f = lambda v, u="": f"{v:.1f}{u}" if v else "-" + print(f" {size:>7} {x['ttft']:>8.0f} {y['ttft']:>8.0f} {y['ttft']-x['ttft']:>+8.0f} " + f"{f(x['refill']):>9} {f(y['refill']):>9} {f(x['decode']):>7} {f(y['decode']):>7}") + + +if __name__ == "__main__": + main() diff --git a/docs/PERF-NEXT.md b/docs/PERF-NEXT.md index c5577467462..61f1b5b4cae 100644 --- a/docs/PERF-NEXT.md +++ b/docs/PERF-NEXT.md @@ -51,15 +51,16 @@ fixed term. | # | idea | evidence it could pay | cost | first test | | --- | --- | --- | --- | --- | -| 1 | **defer the slot refill past the first token** | §9: the refill runs between the prompt read and `prompt done` at **0.319 ms per lent slot** — 72 ms on a 127-token prompt, 188 ms on a 1.6 k one, on the TTFT path. Decode needs its own experts, not the slots being refilled | engine change, a few lines | the same ladder with the refill moved; watch for PCIe contention with decode | -| 2 | ~~ship `conv-route-batch-after` 1–2~~ | **closed, keep 4.** §7: 1 buys 0.52 s across the first six single-shot requests and costs **1.88 s** on the next conversation turn (`C2` 2.465 s against 0.587) — break-even at ~21 single-shots per conversation re-open | — | done; the config key that would have set it was also dead (PR #16) | -| 3 | ~~engine-side per-request timer~~ | **done, no build**: `STRATA_TRACE` already splits the prompt path — §9 names 109 ms of read-side fixed work, 72 ms of refill and 56 ms of bookkeeping | — | done | -| 4 | ~~a one-shot mode that also drops the cache slots~~ | **closed, §8**: no saving over the router's cache-off path at matched sizes (+2 ms at 456 tokens, +8 ms at 1,608), and a 0-slot start severs conversation reuse for good — every turn re-prefilled | — | done | -| 5 | residency-biased router on SYCL | `route_prior.cu` is CUDA-only; here **12,288 of 24,576** experts are resident, so half of every token's experts cross PCIe. Upstream measured +15 % / +37 % on a 12 GB NVIDIA card | engine port, days | decode lanes, 5 interleaved pairs | -| 6 | prefill chunk at 4k/8k/12k with intervals | 16k/32k were tested, but 8,192 was never shown to be the knee | ~20 min | paired, three prompt sizes, per-prompt medians | -| 7 | CPU pool workers and affinity on the 7700X | 15 workers is a 16-thread default; p-core pinning was **+27 %** decode on the 13700K | 15 min | workers 8/12/15/16, `--pool-affinity` on/off | -| 8 | dispatch overhead on SYCL | the Vulkan analysis found ~88 % of per-dispatch cost is fixed; nobody has counted dispatches per token on SYCL | a day | `unitrace` kernel count per token, then fuse the five most frequent small ops | -| 9 | DFlash2 block drafter on SYCL | upstream's headline decode lever; 7 drafts per pass | weeks | only after 1–3 | +| 1 | ~~defer the slot refill past the first token~~ | **closed, measured, refuted (§9)**: the flag shipped in a probe binary and TTFT went 398 → 2,945 ms while decode fell from 92 to **0.4 tok/s**. The borrowed slots are the decode working set, so the refill is a prefetch for the tokens about to be generated, not a tax | — | done | +| 2 | **make the refill cheaper, not later** | the refill is right to exist but 0.319 ms/slot is a PCIe copy of experts decode will touch; only the ones the first N decode positions actually route are worth pre-loading. Shrink the set, keep the timing | engine change, days | the same ladder with a reduced refill set, decode watched | +| 3 | ~~ship `conv-route-batch-after` 1–2~~ | **closed, keep 4.** §7: 1 buys 0.52 s across the first six single-shot requests and costs **1.88 s** on the next conversation turn (`C2` 2.465 s against 0.587) — break-even at ~21 single-shots per conversation re-open | — | done; the config key that would have set it was also dead (PR #16) | +| 4 | ~~engine-side per-request timer~~ | **done, no build**: `STRATA_TRACE` already splits the prompt path — §9 names 109 ms of read-side fixed work, 72 ms of refill and 56 ms of bookkeeping | — | done | +| 5 | ~~a one-shot mode that also drops the cache slots~~ | **closed, §8**: no saving over the router's cache-off path at matched sizes (+2 ms at 456 tokens, +8 ms at 1,608), and a 0-slot start severs conversation reuse for good — every turn re-prefilled | — | done | +| 6 | residency-biased router on SYCL | `route_prior.cu` is CUDA-only; here **12,288 of 24,576** experts are resident, so half of every token's experts cross PCIe. Upstream measured +15 % / +37 % on a 12 GB NVIDIA card | engine port, days | decode lanes, 5 interleaved pairs | +| 7 | prefill chunk at 4k/8k/12k with intervals | 16k/32k were tested, but 8,192 was never shown to be the knee | ~20 min | paired, three prompt sizes, per-prompt medians | +| 8 | CPU pool workers and affinity on the 7700X | 15 workers is a 16-thread default; p-core pinning was **+27 %** decode on the 13700K | 15 min | workers 8/12/15/16, `--pool-affinity` on/off | +| 9 | dispatch overhead on SYCL | the Vulkan analysis found ~88 % of per-dispatch cost is fixed; nobody has counted dispatches per token on SYCL | a day | `unitrace` kernel count per token, then fuse the five most frequent small ops | +| 10 | DFlash2 block drafter on SYCL | upstream's headline decode lever; 7 drafts per pass | weeks | only after 1–3 | ## 5. The PLE sweep, run 2026-11-14 — closed @@ -206,14 +207,30 @@ Ladder with the deployed router, six samples per size, medians: raises `read + refill` by only 7–29 ms: the cache-on cost is checkpoint work inside the engine's prompt span, not in the prompt read. Both measurements were right about different parts of the same 200 ms. -**The lever this exposes:** the refill does not have to be synchronous with the answer. It reclaims slots the *next* -request will lend again, and decode for the current request needs its own experts, not those slots. Moving it after the -first token would take 72–188 ms off TTFT — the largest remaining cheap win, and it is an engine change of a few lines -rather than a research project. +**The lever this looked like — and how it was refuted.** It reads like a tax to defer: the slots go back to the +profile's hottest experts, and decode "needs its own experts". Built as `STRATA_REFILL_DEFER=1` (source patch, one +`if`, probe binary, default unchanged) and measured on the same ladder: + +| real tokens | refill kept | refill deferred | decode, kept | decode, deferred | +| --- | --- | --- | --- | --- | +| 126 | 398 ms | **2,945 ms** | 92.0 tok/s | 30.9 tok/s | +| 456 | 539 ms | **3,126 ms** | 91.8 tok/s | **0.5 tok/s** | +| 1,608 | 1,468 ms | **3,953 ms** | 74.3 tok/s | **0.4 tok/s** | + +The per-sample sequence says why: at 126 tokens the first two requests are normal (0.553, 0.537 s) and the third is +3.069 s — the collapse begins once the cache holds the previous prompt's experts instead of the profile's. **The borrowed +slots are the decode working set**: the refill is a prefetch for the tokens about to be generated, so skipping it moves +the cost from one 72–188 ms transfer per request into a streamed expert fetch per token. 0.32 ms/slot buys 92 tok/s +instead of 0.5. + +What survives is a narrower question: not *when* to refill, but *how much* — only the experts the first decode positions +actually route are worth pre-loading. That is item 2 of §4 now. Artifacts: [`0.2.13-port/intercept/trace/`](../dirigo-results/0.2.13-port/intercept/trace/) — both ladders, the engine logs with every trace line, the parser and the window driver, and [`trace/q2_0/`](../dirigo-results/0.2.13-port/intercept/trace/q2_0/) for the second weight set: its ladder, its config, its engine log, and both calibration attempts. +[`trace/refilldefer/`](../dirigo-results/0.2.13-port/intercept/trace/refilldefer/) holds the refutation: both ladders, +both engine logs, the source patch as a diff, the A/B driver and the analysis script. ### Cross-weight check: it ran, and the fix was the profile