diff --git a/dirigo-results/0.2.13-port/intercept/conf-base.json b/dirigo-results/0.2.13-port/intercept/conf-base.json new file mode 100644 index 00000000000..522f818e182 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/conf-base.json @@ -0,0 +1,63 @@ +{ + "exe": "/home/bob/bin/strata-sycl-host.sh", + "args": [ + "--pack", + "/home/bob/XeStrata-data/packs/coder-iq1_m", + "--native", + "/home/bob/strata-models/IQ1_M/Qwen3.8-Flash-Next-GSQ-RCO-IQ1_M-00001-of-00002.gguf", + "--ple-gguf", + "/home/bob/strata-models/IQ1_M/Qwen3.8-Flash-Next-GSQ-RCO-IQ1_M-00002-of-00002.gguf", + "--expert-profile", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/data/expert-profile-mine.bin", + "--expert-cache", + "auto", + "--stream-experts", + "--prefill", + "auto", + "--prompt-cache", + "6", + "--prompt-cache-every", + "4096", + "--batch", + "0", + "--spec", + "6", + "--spec-min-p", + "0.5", + "--mtp", + "/home/bob/XeStrata-data/mtp/rt", + "--mtp-draft-vocab", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/data/draft_vocab_en.bin", + "--mtp-window", + "8192", + "--max-context", + "131072", + "--kv", + "int8", + "--kv-resident", + "32768", + "--vram-reserve-mib", + "700", + "--conversation-cache-mib", + "2048" + ], + "cwd": "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13", + "tokenizer": "/home/bob/XeStrata-data/packs/coder-iq1_m/tokenizer", + "model_name": "qwen3.8-flash-next-coder-iq1_m", + "log": "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/strata-coder-iq1_m-sycl-tuned.log", + "port": 8095, + "backend": "sycl", + "open_browser": false, + "thinking_policy": "on", + "conv_route": "on", + "conv_route_batch_after": 4, + "short_answer": { + "spec": 0, + "max_tokens": 16 + }, + "spec_window": 0, + "spec_window_needle_min": 6000, + "code_tune": { + "mode": "off" + } +} \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/conf-base2.json b/dirigo-results/0.2.13-port/intercept/conf-base2.json new file mode 100644 index 00000000000..522f818e182 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/conf-base2.json @@ -0,0 +1,63 @@ +{ + "exe": "/home/bob/bin/strata-sycl-host.sh", + "args": [ + "--pack", + "/home/bob/XeStrata-data/packs/coder-iq1_m", + "--native", + "/home/bob/strata-models/IQ1_M/Qwen3.8-Flash-Next-GSQ-RCO-IQ1_M-00001-of-00002.gguf", + "--ple-gguf", + "/home/bob/strata-models/IQ1_M/Qwen3.8-Flash-Next-GSQ-RCO-IQ1_M-00002-of-00002.gguf", + "--expert-profile", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/data/expert-profile-mine.bin", + "--expert-cache", + "auto", + "--stream-experts", + "--prefill", + "auto", + "--prompt-cache", + "6", + "--prompt-cache-every", + "4096", + "--batch", + "0", + "--spec", + "6", + "--spec-min-p", + "0.5", + "--mtp", + "/home/bob/XeStrata-data/mtp/rt", + "--mtp-draft-vocab", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/data/draft_vocab_en.bin", + "--mtp-window", + "8192", + "--max-context", + "131072", + "--kv", + "int8", + "--kv-resident", + "32768", + "--vram-reserve-mib", + "700", + "--conversation-cache-mib", + "2048" + ], + "cwd": "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13", + "tokenizer": "/home/bob/XeStrata-data/packs/coder-iq1_m/tokenizer", + "model_name": "qwen3.8-flash-next-coder-iq1_m", + "log": "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/strata-coder-iq1_m-sycl-tuned.log", + "port": 8095, + "backend": "sycl", + "open_browser": false, + "thinking_policy": "on", + "conv_route": "on", + "conv_route_batch_after": 4, + "short_answer": { + "spec": 0, + "max_tokens": 16 + }, + "spec_window": 0, + "spec_window_needle_min": 6000, + "code_tune": { + "mode": "off" + } +} \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/conf-nocache.json b/dirigo-results/0.2.13-port/intercept/conf-nocache.json new file mode 100644 index 00000000000..e65b1f9139f --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/conf-nocache.json @@ -0,0 +1,65 @@ +{ + "exe": "/home/bob/bin/strata-sycl-host.sh", + "args": [ + "--pack", + "/home/bob/XeStrata-data/packs/coder-iq1_m", + "--native", + "/home/bob/strata-models/IQ1_M/Qwen3.8-Flash-Next-GSQ-RCO-IQ1_M-00001-of-00002.gguf", + "--ple-gguf", + "/home/bob/strata-models/IQ1_M/Qwen3.8-Flash-Next-GSQ-RCO-IQ1_M-00002-of-00002.gguf", + "--expert-profile", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/data/expert-profile-mine.bin", + "--expert-cache", + "auto", + "--stream-experts", + "--prefill", + "auto", + "--prompt-cache", + "6", + "--prompt-cache-every", + "4096", + "--batch", + "0", + "--spec", + "6", + "--spec-min-p", + "0.5", + "--mtp", + "/home/bob/XeStrata-data/mtp/rt", + "--mtp-draft-vocab", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/data/draft_vocab_en.bin", + "--mtp-window", + "8192", + "--max-context", + "131072", + "--kv", + "int8", + "--kv-resident", + "32768", + "--vram-reserve-mib", + "700", + "--conversation-cache-mib", + "2048", + "--prompt-cache", + "0" + ], + "cwd": "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13", + "tokenizer": "/home/bob/XeStrata-data/packs/coder-iq1_m/tokenizer", + "model_name": "qwen3.8-flash-next-coder-iq1_m", + "log": "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/strata-coder-iq1_m-sycl-tuned.log", + "port": 8095, + "backend": "sycl", + "open_browser": false, + "thinking_policy": "on", + "conv_route": "on", + "conv_route_batch_after": 4, + "short_answer": { + "spec": 0, + "max_tokens": 16 + }, + "spec_window": 0, + "spec_window_needle_min": 6000, + "code_tune": { + "mode": "off" + } +} \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/conf-nospec.json b/dirigo-results/0.2.13-port/intercept/conf-nospec.json new file mode 100644 index 00000000000..1021cd90dfe --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/conf-nospec.json @@ -0,0 +1,65 @@ +{ + "exe": "/home/bob/bin/strata-sycl-host.sh", + "args": [ + "--pack", + "/home/bob/XeStrata-data/packs/coder-iq1_m", + "--native", + "/home/bob/strata-models/IQ1_M/Qwen3.8-Flash-Next-GSQ-RCO-IQ1_M-00001-of-00002.gguf", + "--ple-gguf", + "/home/bob/strata-models/IQ1_M/Qwen3.8-Flash-Next-GSQ-RCO-IQ1_M-00002-of-00002.gguf", + "--expert-profile", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/data/expert-profile-mine.bin", + "--expert-cache", + "auto", + "--stream-experts", + "--prefill", + "auto", + "--prompt-cache", + "6", + "--prompt-cache-every", + "4096", + "--batch", + "0", + "--spec", + "6", + "--spec-min-p", + "0.5", + "--mtp", + "/home/bob/XeStrata-data/mtp/rt", + "--mtp-draft-vocab", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/data/draft_vocab_en.bin", + "--mtp-window", + "8192", + "--max-context", + "131072", + "--kv", + "int8", + "--kv-resident", + "32768", + "--vram-reserve-mib", + "700", + "--conversation-cache-mib", + "2048", + "--spec", + "0" + ], + "cwd": "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13", + "tokenizer": "/home/bob/XeStrata-data/packs/coder-iq1_m/tokenizer", + "model_name": "qwen3.8-flash-next-coder-iq1_m", + "log": "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/strata-coder-iq1_m-sycl-tuned.log", + "port": 8095, + "backend": "sycl", + "open_browser": false, + "thinking_policy": "on", + "conv_route": "on", + "conv_route_batch_after": 4, + "short_answer": { + "spec": 0, + "max_tokens": 16 + }, + "spec_window": 0, + "spec_window_needle_min": 6000, + "code_tune": { + "mode": "off" + } +} \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/conf-on.json b/dirigo-results/0.2.13-port/intercept/conf-on.json new file mode 100644 index 00000000000..522f818e182 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/conf-on.json @@ -0,0 +1,63 @@ +{ + "exe": "/home/bob/bin/strata-sycl-host.sh", + "args": [ + "--pack", + "/home/bob/XeStrata-data/packs/coder-iq1_m", + "--native", + "/home/bob/strata-models/IQ1_M/Qwen3.8-Flash-Next-GSQ-RCO-IQ1_M-00001-of-00002.gguf", + "--ple-gguf", + "/home/bob/strata-models/IQ1_M/Qwen3.8-Flash-Next-GSQ-RCO-IQ1_M-00002-of-00002.gguf", + "--expert-profile", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/data/expert-profile-mine.bin", + "--expert-cache", + "auto", + "--stream-experts", + "--prefill", + "auto", + "--prompt-cache", + "6", + "--prompt-cache-every", + "4096", + "--batch", + "0", + "--spec", + "6", + "--spec-min-p", + "0.5", + "--mtp", + "/home/bob/XeStrata-data/mtp/rt", + "--mtp-draft-vocab", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/data/draft_vocab_en.bin", + "--mtp-window", + "8192", + "--max-context", + "131072", + "--kv", + "int8", + "--kv-resident", + "32768", + "--vram-reserve-mib", + "700", + "--conversation-cache-mib", + "2048" + ], + "cwd": "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13", + "tokenizer": "/home/bob/XeStrata-data/packs/coder-iq1_m/tokenizer", + "model_name": "qwen3.8-flash-next-coder-iq1_m", + "log": "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/strata-coder-iq1_m-sycl-tuned.log", + "port": 8095, + "backend": "sycl", + "open_browser": false, + "thinking_policy": "on", + "conv_route": "on", + "conv_route_batch_after": 4, + "short_answer": { + "spec": 0, + "max_tokens": 16 + }, + "spec_window": 0, + "spec_window_needle_min": 6000, + "code_tune": { + "mode": "off" + } +} \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/conf-shadow.json b/dirigo-results/0.2.13-port/intercept/conf-shadow.json new file mode 100644 index 00000000000..7bd3fc5006f --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/conf-shadow.json @@ -0,0 +1,63 @@ +{ + "exe": "/home/bob/bin/strata-sycl-host.sh", + "args": [ + "--pack", + "/home/bob/XeStrata-data/packs/coder-iq1_m", + "--native", + "/home/bob/strata-models/IQ1_M/Qwen3.8-Flash-Next-GSQ-RCO-IQ1_M-00001-of-00002.gguf", + "--ple-gguf", + "/home/bob/strata-models/IQ1_M/Qwen3.8-Flash-Next-GSQ-RCO-IQ1_M-00002-of-00002.gguf", + "--expert-profile", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/data/expert-profile-mine.bin", + "--expert-cache", + "auto", + "--stream-experts", + "--prefill", + "auto", + "--prompt-cache", + "6", + "--prompt-cache-every", + "4096", + "--batch", + "0", + "--spec", + "6", + "--spec-min-p", + "0.5", + "--mtp", + "/home/bob/XeStrata-data/mtp/rt", + "--mtp-draft-vocab", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/data/draft_vocab_en.bin", + "--mtp-window", + "8192", + "--max-context", + "131072", + "--kv", + "int8", + "--kv-resident", + "32768", + "--vram-reserve-mib", + "700", + "--conversation-cache-mib", + "2048" + ], + "cwd": "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13", + "tokenizer": "/home/bob/XeStrata-data/packs/coder-iq1_m/tokenizer", + "model_name": "qwen3.8-flash-next-coder-iq1_m", + "log": "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/strata-coder-iq1_m-sycl-tuned.log", + "port": 8095, + "backend": "sycl", + "open_browser": false, + "thinking_policy": "on", + "conv_route": "shadow", + "conv_route_batch_after": 4, + "short_answer": { + "spec": 0, + "max_tokens": 16 + }, + "spec_window": 0, + "spec_window_needle_min": 6000, + "code_tune": { + "mode": "off" + } +} \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/probe-base.json b/dirigo-results/0.2.13-port/intercept/probe-base.json new file mode 100644 index 00000000000..d7de38aded1 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/probe-base.json @@ -0,0 +1,96 @@ +[ + { + "target": 75, + "ttft_ms": 558.1, + "wall_ms": 626.8, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 537, + "prefill_tps": 234.7, + "decode_ms": 87, + "park_tokens": 96, + "park_ms": 100.4, + "kv_hit_pct": 93.1, + "kv_mib_from_ram": 1.2, + "other_ms": -79.3, + "label": "base" + }, + { + "target": 75, + "ttft_ms": 546.6, + "wall_ms": 605.7, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 525, + "prefill_tps": 239.8, + "decode_ms": 78, + "park_tokens": 133, + "park_ms": 101.9, + "kv_hit_pct": 91.21, + "kv_mib_from_ram": 1.6, + "other_ms": -80.3, + "label": "base" + }, + { + "target": 75, + "ttft_ms": 456.1, + "wall_ms": 524.3, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 435, + "prefill_tps": 289.9, + "decode_ms": 87, + "park_tokens": 133, + "park_ms": 101.6, + "other_ms": -80.5, + "label": "base" + }, + { + "target": 75, + "ttft_ms": 349.8, + "wall_ms": 465.3, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 328, + "prefill_tps": 383.7, + "decode_ms": 134, + "kv_hit_pct": 90.58, + "kv_mib_from_ram": 1.6, + "other_ms": 21.8, + "label": "base" + }, + { + "target": 75, + "ttft_ms": 348.2, + "wall_ms": 409.3, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 327, + "prefill_tps": 385.6, + "decode_ms": 80, + "kv_hit_pct": 93.0, + "kv_mib_from_ram": 1.7, + "other_ms": 21.2, + "label": "base" + }, + { + "target": 75, + "ttft_ms": 349.1, + "wall_ms": 447.9, + "prompt_tokens": 126, + "reused": 0, + "fresh": 126, + "prefill_ms": 328, + "prefill_tps": 384.5, + "decode_ms": 117, + "kv_hit_pct": 88.51, + "kv_mib_from_ram": 1.6, + "other_ms": 21.1, + "label": "base" + } +] \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/probe-base2.json b/dirigo-results/0.2.13-port/intercept/probe-base2.json new file mode 100644 index 00000000000..54479683209 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/probe-base2.json @@ -0,0 +1,90 @@ +[ + { + "target": 75, + "ttft_ms": 557.8, + "wall_ms": 616.7, + "prompt_tokens": 127, + "reused": 0, + "fresh": 127, + "prefill_ms": 537, + "prefill_tps": 236.7, + "decode_ms": 78, + "park_tokens": 96, + "park_ms": 99.4, + "kv_hit_pct": 93.1, + "kv_mib_from_ram": 1.2, + "other_ms": -78.6, + "label": "base2" + }, + { + "target": 75, + "ttft_ms": 540.6, + "wall_ms": 599.3, + "prompt_tokens": 127, + "reused": 0, + "fresh": 127, + "prefill_ms": 520, + "prefill_tps": 244.4, + "decode_ms": 78, + "park_tokens": 134, + "park_ms": 97.5, + "other_ms": -76.9, + "label": "base2" + }, + { + "target": 75, + "ttft_ms": 460.8, + "wall_ms": 518.8, + "prompt_tokens": 127, + "reused": 0, + "fresh": 127, + "prefill_ms": 439, + "prefill_tps": 289.0, + "decode_ms": 77, + "park_tokens": 134, + "park_ms": 108.3, + "other_ms": -86.5, + "label": "base2" + }, + { + "target": 75, + "ttft_ms": 346.7, + "wall_ms": 423.9, + "prompt_tokens": 127, + "reused": 0, + "fresh": 127, + "prefill_ms": 325, + "prefill_tps": 390.5, + "decode_ms": 96, + "other_ms": 21.7, + "label": "base2" + }, + { + "target": 75, + "ttft_ms": 346.5, + "wall_ms": 422.9, + "prompt_tokens": 127, + "reused": 0, + "fresh": 127, + "prefill_ms": 325, + "prefill_tps": 390.5, + "decode_ms": 95, + "kv_hit_pct": 91.94, + "kv_mib_from_ram": 1.7, + "other_ms": 21.5, + "label": "base2" + }, + { + "target": 75, + "ttft_ms": 347.0, + "wall_ms": 423.0, + "prompt_tokens": 127, + "reused": 0, + "fresh": 127, + "prefill_ms": 325, + "prefill_tps": 390.4, + "decode_ms": 95, + "other_ms": 22.0, + "label": "base2" + } +] \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/probe-nocache.json b/dirigo-results/0.2.13-port/intercept/probe-nocache.json new file mode 100644 index 00000000000..7761934c7b1 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/probe-nocache.json @@ -0,0 +1,88 @@ +[ + { + "target": 75, + "ttft_ms": 360.7, + "wall_ms": 439.2, + "prompt_tokens": 127, + "reused": 0, + "fresh": 127, + "prefill_ms": 339, + "prefill_tps": 374.5, + "decode_ms": 97, + "kv_hit_pct": 90.77, + "kv_mib_from_ram": 1.2, + "other_ms": 21.7, + "label": "nocache" + }, + { + "target": 75, + "ttft_ms": 345.0, + "wall_ms": 413.4, + "prompt_tokens": 127, + "reused": 0, + "fresh": 127, + "prefill_ms": 324, + "prefill_tps": 392.4, + "decode_ms": 87, + "other_ms": 21.0, + "label": "nocache" + }, + { + "target": 75, + "ttft_ms": 345.8, + "wall_ms": 413.4, + "prompt_tokens": 127, + "reused": 0, + "fresh": 127, + "prefill_ms": 324, + "prefill_tps": 391.8, + "decode_ms": 86, + "kv_hit_pct": 90.66, + "kv_mib_from_ram": 1.6, + "other_ms": 21.8, + "label": "nocache" + }, + { + "target": 75, + "ttft_ms": 344.9, + "wall_ms": 412.6, + "prompt_tokens": 127, + "reused": 0, + "fresh": 127, + "prefill_ms": 324, + "prefill_tps": 392.4, + "decode_ms": 86, + "kv_hit_pct": 90.66, + "kv_mib_from_ram": 1.6, + "other_ms": 20.9, + "label": "nocache" + }, + { + "target": 75, + "ttft_ms": 344.5, + "wall_ms": 411.9, + "prompt_tokens": 127, + "reused": 0, + "fresh": 127, + "prefill_ms": 323, + "prefill_tps": 393.3, + "decode_ms": 86, + "other_ms": 21.5, + "label": "nocache" + }, + { + "target": 75, + "ttft_ms": 344.3, + "wall_ms": 411.9, + "prompt_tokens": 127, + "reused": 0, + "fresh": 127, + "prefill_ms": 323, + "prefill_tps": 393.2, + "decode_ms": 86, + "kv_hit_pct": 90.66, + "kv_mib_from_ram": 1.6, + "other_ms": 21.3, + "label": "nocache" + } +] \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/probe-on-byteprefix.json b/dirigo-results/0.2.13-port/intercept/probe-on-byteprefix.json new file mode 100644 index 00000000000..3cb134d7a92 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/probe-on-byteprefix.json @@ -0,0 +1,294 @@ +[ + { + "target": 4, + "ttft_ms": 0.9, + "wall_ms": 606.4, + "prompt_tokens": 74, + "reused": 0, + "fresh": 74, + "prefill_ms": 495, + "prefill_tps": 149.5, + "decode_ms": 109, + "park_tokens": 96, + "park_ms": 99.8, + "other_ms": -593.9, + "label": "on" + }, + { + "target": 4, + "ttft_ms": 0.8, + "wall_ms": 582.0, + "prompt_tokens": 74, + "reused": 0, + "fresh": 74, + "prefill_ms": 487, + "prefill_tps": 152.0, + "decode_ms": 93, + "park_tokens": 84, + "park_ms": 101.2, + "kv_hit_pct": 93.31, + "kv_mib_from_ram": 1.0, + "other_ms": -587.4, + "label": "on" + }, + { + "target": 8, + "ttft_ms": 1.0, + "wall_ms": 495.1, + "prompt_tokens": 75, + "reused": 0, + "fresh": 75, + "prefill_ms": 400, + "prefill_tps": 187.7, + "decode_ms": 93, + "park_tokens": 84, + "park_ms": 99.8, + "kv_hit_pct": 93.31, + "kv_mib_from_ram": 1.0, + "other_ms": -498.8, + "label": "on" + }, + { + "target": 8, + "ttft_ms": 0.8, + "wall_ms": 386.9, + "prompt_tokens": 75, + "reused": 0, + "fresh": 75, + "prefill_ms": 291, + "prefill_tps": 257.4, + "decode_ms": 93, + "kv_hit_pct": 90.98, + "kv_mib_from_ram": 1.1, + "other_ms": -290.2, + "label": "on" + }, + { + "target": 16, + "ttft_ms": 1.0, + "wall_ms": 430.8, + "prompt_tokens": 83, + "reused": 0, + "fresh": 83, + "prefill_ms": 302, + "prefill_tps": 275.2, + "decode_ms": 126, + "kv_hit_pct": 90.98, + "kv_mib_from_ram": 1.1, + "other_ms": -301.0, + "label": "on" + }, + { + "target": 16, + "ttft_ms": 0.9, + "wall_ms": 401.0, + "prompt_tokens": 83, + "reused": 0, + "fresh": 83, + "prefill_ms": 300, + "prefill_tps": 276.7, + "decode_ms": 99, + "kv_hit_pct": 93.73, + "kv_mib_from_ram": 1.2, + "other_ms": -299.1, + "label": "on" + }, + { + "target": 32, + "ttft_ms": 0.8, + "wall_ms": 406.6, + "prompt_tokens": 95, + "reused": 0, + "fresh": 95, + "prefill_ms": 308, + "prefill_tps": 308.0, + "decode_ms": 95, + "other_ms": -307.2, + "label": "on" + }, + { + "target": 32, + "ttft_ms": 0.9, + "wall_ms": 401.6, + "prompt_tokens": 95, + "reused": 0, + "fresh": 95, + "prefill_ms": 307, + "prefill_tps": 309.8, + "decode_ms": 92, + "other_ms": -306.1, + "label": "on" + }, + { + "target": 64, + "ttft_ms": 0.9, + "wall_ms": 424.9, + "prompt_tokens": 120, + "reused": 0, + "fresh": 120, + "prefill_ms": 336, + "prefill_tps": 357.5, + "decode_ms": 87, + "kv_hit_pct": 91.12, + "kv_mib_from_ram": 1.3, + "other_ms": -335.1, + "label": "on" + }, + { + "target": 64, + "ttft_ms": 1.0, + "wall_ms": 409.4, + "prompt_tokens": 120, + "reused": 0, + "fresh": 120, + "prefill_ms": 320, + "prefill_tps": 375.0, + "decode_ms": 87, + "kv_hit_pct": 90.72, + "kv_mib_from_ram": 1.5, + "other_ms": -319.0, + "label": "on" + }, + { + "target": 128, + "ttft_ms": 1.0, + "wall_ms": 453.0, + "prompt_tokens": 167, + "reused": 0, + "fresh": 167, + "prefill_ms": 364, + "prefill_tps": 459.4, + "decode_ms": 87, + "other_ms": -363.0, + "label": "on" + }, + { + "target": 128, + "ttft_ms": 1.0, + "wall_ms": 429.6, + "prompt_tokens": 167, + "reused": 0, + "fresh": 167, + "prefill_ms": 340, + "prefill_tps": 490.9, + "decode_ms": 87, + "other_ms": -339.0, + "label": "on" + }, + { + "target": 256, + "ttft_ms": 1.2, + "wall_ms": 548.7, + "prompt_tokens": 264, + "reused": 0, + "fresh": 264, + "prefill_ms": 459, + "prefill_tps": 575.0, + "decode_ms": 87, + "kv_hit_pct": 90.72, + "kv_mib_from_ram": 2.1, + "other_ms": -457.8, + "label": "on" + }, + { + "target": 256, + "ttft_ms": 1.5, + "wall_ms": 509.1, + "prompt_tokens": 264, + "reused": 0, + "fresh": 264, + "prefill_ms": 419, + "prefill_tps": 629.6, + "decode_ms": 87, + "kv_hit_pct": 90.82, + "kv_mib_from_ram": 3.3, + "other_ms": -417.5, + "label": "on" + }, + { + "target": 512, + "ttft_ms": 1.8, + "wall_ms": 646.4, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 533, + "prefill_tps": 855.9, + "decode_ms": 110, + "other_ms": -531.2, + "label": "on" + }, + { + "target": 512, + "ttft_ms": 1.6, + "wall_ms": 630.6, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 517, + "prefill_tps": 882.0, + "decode_ms": 110, + "kv_hit_pct": 92.27, + "kv_mib_from_ram": 5.6, + "other_ms": -515.4, + "label": "on" + }, + { + "target": 1024, + "ttft_ms": 2.6, + "wall_ms": 1036.4, + "prompt_tokens": 842, + "reused": 0, + "fresh": 842, + "prefill_ms": 921, + "prefill_tps": 914.7, + "decode_ms": 111, + "kv_hit_pct": 92.27, + "kv_mib_from_ram": 5.6, + "other_ms": -918.4, + "label": "on" + }, + { + "target": 1024, + "ttft_ms": 2.5, + "wall_ms": 1027.3, + "prompt_tokens": 842, + "reused": 0, + "fresh": 842, + "prefill_ms": 911, + "prefill_tps": 923.8, + "decode_ms": 111, + "kv_hit_pct": 92.27, + "kv_mib_from_ram": 10.3, + "other_ms": -908.5, + "label": "on" + }, + { + "target": 2048, + "ttft_ms": 4.8, + "wall_ms": 1565.1, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1432, + "prefill_tps": 1122.6, + "decode_ms": 127, + "kv_hit_pct": 90.89, + "kv_mib_from_ram": 19.5, + "other_ms": -1427.2, + "label": "on" + }, + { + "target": 2048, + "ttft_ms": 3.9, + "wall_ms": 1548.3, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1452, + "prefill_tps": 1107.2, + "decode_ms": 90, + "other_ms": -1448.1, + "label": "on" + } +] \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/probe-on.json b/dirigo-results/0.2.13-port/intercept/probe-on.json new file mode 100644 index 00000000000..f5e1694460a --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/probe-on.json @@ -0,0 +1,304 @@ +[ + { + "target": 4, + "ttft_ms": 505.4, + "wall_ms": 596.7, + "prompt_tokens": 74, + "reused": 0, + "fresh": 74, + "prefill_ms": 484, + "prefill_tps": 152.8, + "decode_ms": 109, + "park_tokens": 96, + "park_ms": 98.1, + "kv_hit_pct": 93.1, + "kv_mib_from_ram": 1.2, + "other_ms": -76.7, + "label": "on" + }, + { + "target": 4, + "ttft_ms": 506.0, + "wall_ms": 580.6, + "prompt_tokens": 74, + "reused": 0, + "fresh": 74, + "prefill_ms": 485, + "prefill_tps": 152.5, + "decode_ms": 93, + "park_tokens": 84, + "park_ms": 101.8, + "other_ms": -80.8, + "label": "on" + }, + { + "target": 8, + "ttft_ms": 423.1, + "wall_ms": 497.3, + "prompt_tokens": 75, + "reused": 0, + "fresh": 75, + "prefill_ms": 402, + "prefill_tps": 186.6, + "decode_ms": 93, + "park_tokens": 84, + "park_ms": 101.9, + "kv_hit_pct": 93.31, + "kv_mib_from_ram": 1.0, + "other_ms": -80.8, + "label": "on" + }, + { + "target": 8, + "ttft_ms": 311.1, + "wall_ms": 416.5, + "prompt_tokens": 75, + "reused": 0, + "fresh": 75, + "prefill_ms": 290, + "prefill_tps": 258.6, + "decode_ms": 124, + "kv_hit_pct": 90.98, + "kv_mib_from_ram": 1.1, + "other_ms": 21.1, + "label": "on" + }, + { + "target": 16, + "ttft_ms": 320.6, + "wall_ms": 399.9, + "prompt_tokens": 83, + "reused": 0, + "fresh": 83, + "prefill_ms": 299, + "prefill_tps": 277.3, + "decode_ms": 98, + "kv_hit_pct": 93.29, + "kv_mib_from_ram": 1.1, + "other_ms": 21.6, + "label": "on" + }, + { + "target": 16, + "ttft_ms": 319.6, + "wall_ms": 384.4, + "prompt_tokens": 83, + "reused": 0, + "fresh": 83, + "prefill_ms": 298, + "prefill_tps": 278.1, + "decode_ms": 83, + "kv_hit_pct": 88.44, + "kv_mib_from_ram": 1.1, + "other_ms": 21.6, + "label": "on" + }, + { + "target": 32, + "ttft_ms": 329.5, + "wall_ms": 407.3, + "prompt_tokens": 95, + "reused": 0, + "fresh": 95, + "prefill_ms": 308, + "prefill_tps": 308.3, + "decode_ms": 97, + "kv_hit_pct": 89.64, + "kv_mib_from_ram": 1.1, + "other_ms": 21.5, + "label": "on" + }, + { + "target": 32, + "ttft_ms": 326.8, + "wall_ms": 400.2, + "prompt_tokens": 95, + "reused": 0, + "fresh": 95, + "prefill_ms": 306, + "prefill_tps": 310.7, + "decode_ms": 92, + "kv_hit_pct": 88.5, + "kv_mib_from_ram": 1.3, + "other_ms": 20.8, + "label": "on" + }, + { + "target": 64, + "ttft_ms": 356.5, + "wall_ms": 423.9, + "prompt_tokens": 120, + "reused": 0, + "fresh": 120, + "prefill_ms": 335, + "prefill_tps": 358.0, + "decode_ms": 86, + "kv_hit_pct": 91.12, + "kv_mib_from_ram": 1.3, + "other_ms": 21.5, + "label": "on" + }, + { + "target": 64, + "ttft_ms": 342.9, + "wall_ms": 410.4, + "prompt_tokens": 120, + "reused": 0, + "fresh": 120, + "prefill_ms": 322, + "prefill_tps": 373.1, + "decode_ms": 86, + "kv_hit_pct": 90.72, + "kv_mib_from_ram": 1.5, + "other_ms": 20.9, + "label": "on" + }, + { + "target": 128, + "ttft_ms": 385.4, + "wall_ms": 452.9, + "prompt_tokens": 167, + "reused": 0, + "fresh": 167, + "prefill_ms": 364, + "prefill_tps": 459.1, + "decode_ms": 86, + "kv_hit_pct": 90.72, + "kv_mib_from_ram": 1.5, + "other_ms": 21.4, + "label": "on" + }, + { + "target": 128, + "ttft_ms": 361.8, + "wall_ms": 446.7, + "prompt_tokens": 167, + "reused": 0, + "fresh": 167, + "prefill_ms": 340, + "prefill_tps": 490.6, + "decode_ms": 104, + "kv_hit_pct": 90.72, + "kv_mib_from_ram": 2.1, + "other_ms": 21.8, + "label": "on" + }, + { + "target": 256, + "ttft_ms": 481.9, + "wall_ms": 566.3, + "prompt_tokens": 264, + "reused": 0, + "fresh": 264, + "prefill_ms": 460, + "prefill_tps": 574.1, + "decode_ms": 104, + "kv_hit_pct": 90.82, + "kv_mib_from_ram": 3.3, + "other_ms": 21.9, + "label": "on" + }, + { + "target": 256, + "ttft_ms": 441.3, + "wall_ms": 508.9, + "prompt_tokens": 264, + "reused": 0, + "fresh": 264, + "prefill_ms": 419, + "prefill_tps": 629.5, + "decode_ms": 86, + "other_ms": 22.3, + "label": "on" + }, + { + "target": 512, + "ttft_ms": 552.8, + "wall_ms": 637.6, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 530, + "prefill_tps": 860.1, + "decode_ms": 105, + "kv_hit_pct": 90.82, + "kv_mib_from_ram": 3.3, + "other_ms": 22.8, + "label": "on" + }, + { + "target": 512, + "ttft_ms": 537.8, + "wall_ms": 622.2, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 515, + "prefill_tps": 885.1, + "decode_ms": 104, + "kv_hit_pct": 90.86, + "kv_mib_from_ram": 5.6, + "other_ms": 22.8, + "label": "on" + }, + { + "target": 1024, + "ttft_ms": 939.5, + "wall_ms": 1007.6, + "prompt_tokens": 842, + "reused": 0, + "fresh": 842, + "prefill_ms": 915, + "prefill_tps": 920.0, + "decode_ms": 88, + "kv_hit_pct": 90.86, + "kv_mib_from_ram": 5.6, + "other_ms": 24.5, + "label": "on" + }, + { + "target": 1024, + "ttft_ms": 935.8, + "wall_ms": 1027.1, + "prompt_tokens": 842, + "reused": 0, + "fresh": 842, + "prefill_ms": 911, + "prefill_tps": 923.8, + "decode_ms": 111, + "kv_hit_pct": 90.86, + "kv_mib_from_ram": 10.3, + "other_ms": 24.8, + "label": "on" + }, + { + "target": 2048, + "ttft_ms": 1469.2, + "wall_ms": 1537.8, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1442, + "prefill_tps": 1115.2, + "decode_ms": 90, + "kv_hit_pct": 92.27, + "kv_mib_from_ram": 10.3, + "other_ms": 27.2, + "label": "on" + }, + { + "target": 2048, + "ttft_ms": 1475.0, + "wall_ms": 1566.9, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1443, + "prefill_tps": 1114.0, + "decode_ms": 113, + "kv_hit_pct": 90.89, + "kv_mib_from_ram": 19.5, + "other_ms": 32.0, + "label": "on" + } +] \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/probe-shadow-byteprefix.json b/dirigo-results/0.2.13-port/intercept/probe-shadow-byteprefix.json new file mode 100644 index 00000000000..650ec1c971c --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/probe-shadow-byteprefix.json @@ -0,0 +1,332 @@ +[ + { + "target": 4, + "ttft_ms": 0.9, + "wall_ms": 605.6, + "prompt_tokens": 74, + "reused": 0, + "fresh": 74, + "prefill_ms": 493, + "prefill_tps": 150.2, + "decode_ms": 110, + "park_tokens": 97, + "park_ms": 98.1, + "other_ms": -590.2, + "label": "shadow" + }, + { + "target": 4, + "ttft_ms": 0.8, + "wall_ms": 617.6, + "prompt_tokens": 74, + "reused": 0, + "fresh": 74, + "prefill_ms": 489, + "prefill_tps": 151.4, + "decode_ms": 126, + "park_tokens": 82, + "park_ms": 100.4, + "other_ms": -588.6, + "label": "shadow" + }, + { + "target": 8, + "ttft_ms": 0.9, + "wall_ms": 624.3, + "prompt_tokens": 75, + "reused": 0, + "fresh": 75, + "prefill_ms": 498, + "prefill_tps": 150.7, + "decode_ms": 124, + "park_tokens": 84, + "park_ms": 105.3, + "kv_hit_pct": 94.93, + "kv_mib_from_ram": 1.0, + "other_ms": -602.4, + "label": "shadow" + }, + { + "target": 8, + "ttft_ms": 0.8, + "wall_ms": 617.3, + "prompt_tokens": 75, + "reused": 0, + "fresh": 75, + "prefill_ms": 491, + "prefill_tps": 152.7, + "decode_ms": 124, + "park_tokens": 86, + "park_ms": 102.0, + "kv_hit_pct": 94.53, + "kv_mib_from_ram": 1.1, + "other_ms": -592.2, + "label": "shadow" + }, + { + "target": 16, + "ttft_ms": 0.9, + "wall_ms": 615.4, + "prompt_tokens": 83, + "reused": 0, + "fresh": 83, + "prefill_ms": 511, + "prefill_tps": 162.3, + "decode_ms": 101, + "park_tokens": 86, + "park_ms": 101.0, + "kv_hit_pct": 94.53, + "kv_mib_from_ram": 1.1, + "other_ms": -611.1, + "label": "shadow" + }, + { + "target": 16, + "ttft_ms": 0.8, + "wall_ms": 595.8, + "prompt_tokens": 83, + "reused": 0, + "fresh": 83, + "prefill_ms": 510, + "prefill_tps": 162.9, + "decode_ms": 84, + "park_tokens": 90, + "park_ms": 100.9, + "kv_hit_pct": 92.43, + "kv_mib_from_ram": 1.1, + "other_ms": -610.1, + "label": "shadow" + }, + { + "target": 32, + "ttft_ms": 0.9, + "wall_ms": 617.4, + "prompt_tokens": 95, + "reused": 0, + "fresh": 95, + "prefill_ms": 519, + "prefill_tps": 183.0, + "decode_ms": 95, + "park_tokens": 90, + "park_ms": 100.0, + "kv_hit_pct": 92.43, + "kv_mib_from_ram": 1.1, + "other_ms": -618.1, + "label": "shadow" + }, + { + "target": 32, + "ttft_ms": 0.8, + "wall_ms": 643.6, + "prompt_tokens": 95, + "reused": 0, + "fresh": 95, + "prefill_ms": 522, + "prefill_tps": 182.0, + "decode_ms": 119, + "park_tokens": 105, + "park_ms": 102.2, + "kv_hit_pct": 93.63, + "kv_mib_from_ram": 1.3, + "other_ms": -623.4, + "label": "shadow" + }, + { + "target": 64, + "ttft_ms": 0.9, + "wall_ms": 636.0, + "prompt_tokens": 120, + "reused": 0, + "fresh": 120, + "prefill_ms": 546, + "prefill_tps": 219.6, + "decode_ms": 87, + "park_tokens": 107, + "park_ms": 101.2, + "kv_hit_pct": 94.35, + "kv_mib_from_ram": 1.3, + "other_ms": -646.3, + "label": "shadow" + }, + { + "target": 64, + "ttft_ms": 1.0, + "wall_ms": 622.4, + "prompt_tokens": 120, + "reused": 0, + "fresh": 120, + "prefill_ms": 533, + "prefill_tps": 225.2, + "decode_ms": 87, + "park_tokens": 127, + "park_ms": 100.4, + "kv_hit_pct": 93.1, + "kv_mib_from_ram": 1.5, + "other_ms": -632.4, + "label": "shadow" + }, + { + "target": 128, + "ttft_ms": 1.0, + "wall_ms": 666.8, + "prompt_tokens": 167, + "reused": 0, + "fresh": 167, + "prefill_ms": 577, + "prefill_tps": 289.4, + "decode_ms": 87, + "park_tokens": 127, + "park_ms": 101.7, + "kv_hit_pct": 93.1, + "kv_mib_from_ram": 1.5, + "other_ms": -677.7, + "label": "shadow" + }, + { + "target": 128, + "ttft_ms": 1.0, + "wall_ms": 664.7, + "prompt_tokens": 167, + "reused": 0, + "fresh": 167, + "prefill_ms": 558, + "prefill_tps": 299.1, + "decode_ms": 104, + "park_tokens": 174, + "park_ms": 100.7, + "kv_hit_pct": 93.12, + "kv_mib_from_ram": 2.1, + "other_ms": -657.7, + "label": "shadow" + }, + { + "target": 256, + "ttft_ms": 1.3, + "wall_ms": 761.2, + "prompt_tokens": 264, + "reused": 0, + "fresh": 264, + "prefill_ms": 672, + "prefill_tps": 393.0, + "decode_ms": 86, + "park_tokens": 174, + "park_ms": 102.0, + "other_ms": -772.7, + "label": "shadow" + }, + { + "target": 256, + "ttft_ms": 1.3, + "wall_ms": 724.8, + "prompt_tokens": 264, + "reused": 0, + "fresh": 264, + "prefill_ms": 635, + "prefill_tps": 415.9, + "decode_ms": 87, + "park_tokens": 271, + "park_ms": 99.3, + "kv_hit_pct": 93.23, + "kv_mib_from_ram": 3.3, + "other_ms": -733.0, + "label": "shadow" + }, + { + "target": 512, + "ttft_ms": 1.7, + "wall_ms": 861.4, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 748, + "prefill_tps": 609.3, + "decode_ms": 110, + "park_tokens": 271, + "park_ms": 101.2, + "other_ms": -847.5, + "label": "shadow" + }, + { + "target": 512, + "ttft_ms": 1.7, + "wall_ms": 860.2, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 730, + "prefill_tps": 624.4, + "decode_ms": 127, + "park_tokens": 463, + "park_ms": 101.4, + "kv_hit_pct": 94.07, + "kv_mib_from_ram": 5.6, + "other_ms": -829.7, + "label": "shadow" + }, + { + "target": 1024, + "ttft_ms": 2.6, + "wall_ms": 1273.2, + "prompt_tokens": 842, + "reused": 0, + "fresh": 842, + "prefill_ms": 1144, + "prefill_tps": 736.2, + "decode_ms": 125, + "park_tokens": 463, + "park_ms": 100.7, + "other_ms": -1242.1, + "label": "shadow" + }, + { + "target": 1024, + "ttft_ms": 2.5, + "wall_ms": 1268.7, + "prompt_tokens": 842, + "reused": 0, + "fresh": 842, + "prefill_ms": 1139, + "prefill_tps": 739.2, + "decode_ms": 125, + "park_tokens": 849, + "park_ms": 102.1, + "kv_hit_pct": 95.21, + "kv_mib_from_ram": 10.3, + "other_ms": -1238.6, + "label": "shadow" + }, + { + "target": 2048, + "ttft_ms": 3.8, + "wall_ms": 1769.9, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1676, + "prefill_tps": 959.3, + "decode_ms": 88, + "park_tokens": 849, + "park_ms": 104.8, + "kv_hit_pct": 95.21, + "kv_mib_from_ram": 10.3, + "other_ms": -1777.0, + "label": "shadow" + }, + { + "target": 2048, + "ttft_ms": 3.9, + "wall_ms": 1772.1, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1678, + "prefill_tps": 958.1, + "decode_ms": 88, + "park_tokens": 1615, + "park_ms": 110.8, + "kv_hit_pct": 93.32, + "kv_mib_from_ram": 19.5, + "other_ms": -1784.9, + "label": "shadow" + } +] \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/probe-shadow.json b/dirigo-results/0.2.13-port/intercept/probe-shadow.json new file mode 100644 index 00000000000..fe996de0c83 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/probe-shadow.json @@ -0,0 +1,328 @@ +[ + { + "target": 4, + "ttft_ms": 515.6, + "wall_ms": 631.9, + "prompt_tokens": 74, + "reused": 0, + "fresh": 74, + "prefill_ms": 495, + "prefill_tps": 149.6, + "decode_ms": 134, + "park_tokens": 97, + "park_ms": 101.7, + "kv_hit_pct": 91.69, + "kv_mib_from_ram": 1.2, + "other_ms": -81.1, + "label": "shadow" + }, + { + "target": 4, + "ttft_ms": 512.7, + "wall_ms": 633.3, + "prompt_tokens": 74, + "reused": 0, + "fresh": 74, + "prefill_ms": 492, + "prefill_tps": 150.5, + "decode_ms": 139, + "park_tokens": 84, + "park_ms": 104.6, + "kv_hit_pct": 95.39, + "kv_mib_from_ram": 1.0, + "other_ms": -83.9, + "label": "shadow" + }, + { + "target": 8, + "ttft_ms": 513.2, + "wall_ms": 588.1, + "prompt_tokens": 75, + "reused": 0, + "fresh": 75, + "prefill_ms": 492, + "prefill_tps": 152.4, + "decode_ms": 93, + "park_tokens": 84, + "park_ms": 100.6, + "kv_hit_pct": 94.67, + "kv_mib_from_ram": 1.0, + "other_ms": -79.4, + "label": "shadow" + }, + { + "target": 8, + "ttft_ms": 510.3, + "wall_ms": 615.3, + "prompt_tokens": 75, + "reused": 0, + "fresh": 75, + "prefill_ms": 489, + "prefill_tps": 153.3, + "decode_ms": 123, + "park_tokens": 85, + "park_ms": 102.1, + "kv_hit_pct": 94.53, + "kv_mib_from_ram": 1.1, + "other_ms": -80.8, + "label": "shadow" + }, + { + "target": 16, + "ttft_ms": 530.6, + "wall_ms": 607.5, + "prompt_tokens": 83, + "reused": 0, + "fresh": 83, + "prefill_ms": 510, + "prefill_tps": 162.8, + "decode_ms": 95, + "park_tokens": 86, + "park_ms": 101.3, + "other_ms": -80.7, + "label": "shadow" + }, + { + "target": 16, + "ttft_ms": 532.6, + "wall_ms": 639.2, + "prompt_tokens": 83, + "reused": 0, + "fresh": 83, + "prefill_ms": 512, + "prefill_tps": 162.3, + "decode_ms": 125, + "park_tokens": 93, + "park_ms": 101.8, + "other_ms": -81.2, + "label": "shadow" + }, + { + "target": 32, + "ttft_ms": 541.0, + "wall_ms": 618.1, + "prompt_tokens": 95, + "reused": 0, + "fresh": 95, + "prefill_ms": 520, + "prefill_tps": 182.7, + "decode_ms": 95, + "park_tokens": 94, + "park_ms": 102.3, + "kv_hit_pct": 94.84, + "kv_mib_from_ram": 1.2, + "other_ms": -81.3, + "label": "shadow" + }, + { + "target": 32, + "ttft_ms": 540.4, + "wall_ms": 617.4, + "prompt_tokens": 95, + "reused": 0, + "fresh": 95, + "prefill_ms": 519, + "prefill_tps": 182.9, + "decode_ms": 95, + "park_tokens": 105, + "park_ms": 101.7, + "kv_hit_pct": 93.63, + "kv_mib_from_ram": 1.3, + "other_ms": -80.3, + "label": "shadow" + }, + { + "target": 64, + "ttft_ms": 567.6, + "wall_ms": 635.8, + "prompt_tokens": 120, + "reused": 0, + "fresh": 120, + "prefill_ms": 547, + "prefill_tps": 219.6, + "decode_ms": 87, + "park_tokens": 105, + "park_ms": 100.3, + "kv_hit_pct": 93.63, + "kv_mib_from_ram": 1.3, + "other_ms": -79.7, + "label": "shadow" + }, + { + "target": 64, + "ttft_ms": 554.4, + "wall_ms": 623.1, + "prompt_tokens": 120, + "reused": 0, + "fresh": 120, + "prefill_ms": 533, + "prefill_tps": 225.0, + "decode_ms": 87, + "park_tokens": 127, + "park_ms": 99.9, + "other_ms": -78.5, + "label": "shadow" + }, + { + "target": 128, + "ttft_ms": 597.7, + "wall_ms": 666.2, + "prompt_tokens": 167, + "reused": 0, + "fresh": 167, + "prefill_ms": 577, + "prefill_tps": 289.6, + "decode_ms": 87, + "park_tokens": 127, + "park_ms": 101.0, + "kv_hit_pct": 93.12, + "kv_mib_from_ram": 2.1, + "other_ms": -80.3, + "label": "shadow" + }, + { + "target": 128, + "ttft_ms": 580.4, + "wall_ms": 648.9, + "prompt_tokens": 167, + "reused": 0, + "fresh": 167, + "prefill_ms": 559, + "prefill_tps": 298.6, + "decode_ms": 87, + "park_tokens": 174, + "park_ms": 102.6, + "kv_hit_pct": 93.12, + "kv_mib_from_ram": 2.1, + "other_ms": -81.2, + "label": "shadow" + }, + { + "target": 256, + "ttft_ms": 696.7, + "wall_ms": 764.8, + "prompt_tokens": 264, + "reused": 0, + "fresh": 264, + "prefill_ms": 675, + "prefill_tps": 390.9, + "decode_ms": 86, + "park_tokens": 174, + "park_ms": 101.2, + "other_ms": -79.5, + "label": "shadow" + }, + { + "target": 256, + "ttft_ms": 656.4, + "wall_ms": 754.5, + "prompt_tokens": 264, + "reused": 0, + "fresh": 264, + "prefill_ms": 635, + "prefill_tps": 415.8, + "decode_ms": 117, + "park_tokens": 271, + "park_ms": 100.5, + "other_ms": -79.1, + "label": "shadow" + }, + { + "target": 512, + "ttft_ms": 776.9, + "wall_ms": 845.2, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 755, + "prefill_tps": 603.9, + "decode_ms": 87, + "park_tokens": 271, + "park_ms": 100.7, + "other_ms": -78.8, + "label": "shadow" + }, + { + "target": 512, + "ttft_ms": 755.8, + "wall_ms": 847.1, + "prompt_tokens": 456, + "reused": 0, + "fresh": 456, + "prefill_ms": 734, + "prefill_tps": 621.4, + "decode_ms": 110, + "park_tokens": 463, + "park_ms": 102.8, + "kv_hit_pct": 93.27, + "kv_mib_from_ram": 5.6, + "other_ms": -81.0, + "label": "shadow" + }, + { + "target": 1024, + "ttft_ms": 1169.0, + "wall_ms": 1237.8, + "prompt_tokens": 842, + "reused": 0, + "fresh": 842, + "prefill_ms": 1146, + "prefill_tps": 734.6, + "decode_ms": 87, + "park_tokens": 463, + "park_ms": 101.2, + "kv_hit_pct": 94.07, + "kv_mib_from_ram": 5.6, + "other_ms": -78.2, + "label": "shadow" + }, + { + "target": 1024, + "ttft_ms": 1165.1, + "wall_ms": 1242.0, + "prompt_tokens": 842, + "reused": 0, + "fresh": 842, + "prefill_ms": 1143, + "prefill_tps": 736.9, + "decode_ms": 95, + "park_tokens": 849, + "park_ms": 102.1, + "kv_hit_pct": 93.28, + "kv_mib_from_ram": 10.3, + "other_ms": -80.0, + "label": "shadow" + }, + { + "target": 2048, + "ttft_ms": 1701.7, + "wall_ms": 1771.1, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1678, + "prefill_tps": 958.6, + "decode_ms": 88, + "park_tokens": 849, + "park_ms": 104.3, + "other_ms": -80.6, + "label": "shadow" + }, + { + "target": 2048, + "ttft_ms": 1723.2, + "wall_ms": 1815.8, + "prompt_tokens": 1608, + "reused": 0, + "fresh": 1608, + "prefill_ms": 1699, + "prefill_tps": 946.4, + "decode_ms": 111, + "park_tokens": 1615, + "park_ms": 110.6, + "kv_hit_pct": 93.32, + "kv_mib_from_ram": 19.5, + "other_ms": -86.4, + "label": "shadow" + } +] \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/strata-intercept-cause.sh b/dirigo-results/0.2.13-port/intercept/strata-intercept-cause.sh new file mode 100755 index 00000000000..c785da58ba9 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/strata-intercept-cause.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# strata-intercept-cause.sh - which engine path owns the ~220 ms that remains with the cache off? +# +# The ladder probe put the fixed cost at 222 ms (cache off) / 434 ms (cache on) inside the engine's own prefill +# accounting, with the PLE read path already excluded by the row-cache sweep. This fixes the prompt at ~75 tokens +# and toggles one engine subsystem per card window, six samples each: +# +# base deployed config +# nospec --spec 0 (no MTP/suffix drafting: the verify geometry at request start) +# nocache --prompt-cache 0 (no conversation cache slots at all) +# base2 deployed config again (drift check) +# +# UNDO: rm this file and ~/strata-intercept-cause +set -uo pipefail +S=/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731 +TREE=$S/strata-0.2.13 +BASE=$TREE/strata-coder-iq1_m-sycl-tuned.json +OUT=$HOME/strata-intercept-cause +PROBE=$HOME/bin/strata-intercept-probe.py +mkdir -p "$OUT" + +variant() { # name extra-args... + local name=$1; shift + python3 - "$BASE" "$OUT/conf-$name.json" "$@" <<'PY' +import json, sys +src, dst, extra = sys.argv[1], sys.argv[2], sys.argv[3:] +d = json.load(open(src)); d["args"] = list(d.get("args", [])) + extra +json.dump(d, open(dst, "w"), indent=1); print(dst, "extra:", extra) +PY +} + +arm() { # tag + local tag=$1 conf="$OUT/conf-$1.json" + echo "[$(date +%H:%M:%S)] ===== cause-$tag" + ~/bin/strata-card-window.sh "$TREE" 8097 --conf "$conf" -- bash -c " + EL=\$(python3 -c \"import json;print(json.load(open('$conf'))['log'])\") + curl -s -m 5 http://127.0.0.1:\$PORT/v1/status | python3 -c 'import json,sys;d=json.load(sys.stdin);print(\" engine:\",d[\"engine\"])' + /usr/bin/python3 $PROBE --port \$PORT --log \"\$EL\" --out $OUT/probe-$tag.json --label $tag \ + --sizes 75 --samples 6 + " 2>&1 | grep -E '=====|engine:|^\s+\{|wrote|restore' +} + +variant base +variant base2 +variant nospec --spec 0 +variant nocache --prompt-cache 0 + +for tag in base nospec nocache base2; do + arm "$tag" +done +echo "[$(date +%H:%M:%S)] cause sweep done" diff --git a/dirigo-results/0.2.13-port/intercept/strata-intercept-probe.py b/dirigo-results/0.2.13-port/intercept/strata-intercept-probe.py new file mode 100755 index 00000000000..e3eaad963c8 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/strata-intercept-probe.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Where a request's fixed ~330 ms goes: prompt read, conversation-cache park, or neither. + +Sends unique prompts of a ladder of sizes and, for each one, reads the engine's own accounting out of its log: + + prompt N tokens = R reused + F read in X ms (Y tok/s), G generated in Z ms ... + conversation cache: parked N tokens in X ms; ... + +so three numbers can be separated per request: TTFT as the client sees it, the engine's prompt read, and the park +copy. The difference between TTFT and those two is everything else the request pays (queue, server, first-token +setup) — the term nobody has named. + + ~/bin/strata-intercept-probe.py --port 8097 --log --out /tmp/ip.json [--label a] + +Unique prompts only: an identical prefix hits the conversation cache and reports 14 000 tok/s "prefill", which +measures the cache instead of the prompt path. +""" +import argparse +import json +import os +import random +import re +import time +import urllib.request + +FILLER = ("The engine streams expert weights over PCIe while the model is resident in RAM. ") +SIZES = [4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048] +RE_PROMPT = re.compile(r"prompt (\d+) tokens = (\d+) reused \+ (\d+) read in (\d+) ms \(([0-9.]+) tok/s\), " + r"(\d+) generated in (\d+) ms") +RE_PARK = re.compile(r"conversation cache: parked (\d+) tokens in ([0-9.]+) ms") +RE_KV = re.compile(r"KV streaming: ([0-9.]+)% of (\d+) block reads hit VRAM, ([0-9.]+) MiB read from RAM") + + +def prompt_of(tokens: int, salt: str) -> str: + body = (FILLER * (tokens // 15 + 2))[: max(8, tokens * 4)] + return salt + " " + body + + +def tail_since(path: str, offset: int) -> str: + with open(path, "rb") as f: + f.seek(offset) + return f.read().decode("utf-8", "replace") + + +def one(port: int, tokens: int, salt: str, log: str, wait_s: float = 60.0) -> dict: + off = os.path.getsize(log) + body = json.dumps({"model": "m", "max_tokens": 8, "temperature": 0.0, "stream": True, + "messages": [{"role": "user", "content": prompt_of(tokens, salt)}]}).encode() + req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/chat/completions", data=body, + headers={"Content-Type": "application/json"}) + t0 = time.time() + first = None + with urllib.request.urlopen(req, timeout=180) as r: + for raw in r: + if first is not None: + continue + line = raw.decode("utf-8", "replace").strip() + if not line.startswith("data:"): + continue + try: + d = json.loads(line[5:].strip()) + except Exception: + continue + delta = (d.get("choices") or [{}])[0].get("delta") or {} + # the first CONTENT token: the stream opens before the prompt is read, so the first byte is not TTFT + if delta.get("content") or delta.get("reasoning_content"): + first = time.time() - t0 + wall = time.time() - t0 + # the engine writes its accounting at the end of the request; poll for the prompt line + seg = "" + deadline = time.time() + wait_s + while time.time() < deadline: + seg = tail_since(log, off) + if RE_PROMPT.search(seg): + break + time.sleep(0.2) + m = None + for m in RE_PROMPT.finditer(seg): + pass + p = None + for p in RE_PARK.finditer(seg): + pass + kv = None + for kv in RE_KV.finditer(seg): + pass + row = dict(target=tokens, ttft_ms=None if first is None else round(first * 1000, 1), + wall_ms=round(wall * 1000, 1)) + if m: + row.update(prompt_tokens=int(m.group(1)), reused=int(m.group(2)), fresh=int(m.group(3)), + prefill_ms=int(m.group(4)), prefill_tps=float(m.group(5)), + decode_ms=int(m.group(7))) + if p: + row.update(park_tokens=int(p.group(1)), park_ms=float(p.group(2))) + if kv: + row.update(kv_hit_pct=float(kv.group(1)), kv_mib_from_ram=float(kv.group(3))) + # what TTFT pays beyond the engine's own prompt read (server, queue, first-token setup). + # The park copy is written AFTER the answer in both modes, so it is not subtracted here. + if row.get("ttft_ms") is not None and "prefill_ms" in row: + row["server_ms"] = round(row["ttft_ms"] - row["prefill_ms"], 1) + return row + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--port", type=int, default=8097) + ap.add_argument("--log", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--label", default="arm") + ap.add_argument("--samples", type=int, default=2) + ap.add_argument("--sizes", default="", help="comma-separated ladder override, e.g. 75") + args = ap.parse_args() + rows = [] + # warm the server so the first request's cost is not the load + one(args.port, 32, f"warm-{random.random():.9f}", args.log) + sizes = [int(s) for s in args.sizes.split(",") if s.strip()] or SIZES + for size in sizes: + for i in range(args.samples): + salt = f"probe {args.label} {size} {i} {random.random():.9f}" + r = one(args.port, size, salt, args.log) + r["label"] = args.label + rows.append(r) + print(f" {json.dumps(r)}", flush=True) + json.dump(rows, open(args.out, "w"), indent=1) + print(f"wrote {args.out}: {len(rows)} samples") + + +if __name__ == "__main__": + main() diff --git a/dirigo-results/0.2.13-port/intercept/strata-intercept-sweep.sh b/dirigo-results/0.2.13-port/intercept/strata-intercept-sweep.sh new file mode 100755 index 00000000000..1b8437086c0 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/strata-intercept-sweep.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# strata-intercept-sweep.sh - price a request's fixed cost, with and without the fork's cache routing. +# +# Two card windows on the 0.2.13 tree, same engine, same model: +# on the deployed config (conv_route: on) -> single-shot traffic runs with the prompt cache OFF +# shadow conv_route: shadow (classify and log, never switch) -> every request parks, like stock +# +# Each window runs ~/bin/strata-intercept-probe.py over a 4..2048-token ladder of unique prompts. The probe reads +# the engine's own "prompt N tokens = ... read in X ms" and "conversation cache: parked ... in Y ms" lines, so +# TTFT splits into prompt read + park + everything else. +# +# UNDO: rm this file and ~/strata-intercept-sweep +set -uo pipefail +S=/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731 +TREE=$S/strata-0.2.13 +BASE=$TREE/strata-coder-iq1_m-sycl-tuned.json +OUT=$HOME/strata-intercept-sweep +PROBE=$HOME/bin/strata-intercept-probe.py +mkdir -p "$OUT" + +variant() { # name key value + python3 - "$BASE" "$OUT/conf-$1.json" "$2" "$3" <<'PY' +import json, sys +src, dst, key, val = sys.argv[1:5] +d = json.load(open(src)) +try: val = json.loads(val) +except Exception: pass +d[key] = val +json.dump(d, open(dst, "w"), indent=1) +print(dst, key, "=", d[key]) +PY +} + +arm() { # tag + local tag=$1 conf="$OUT/conf-$1.json" + echo "[$(date +%H:%M:%S)] ===== intercept-$tag" + ~/bin/strata-card-window.sh "$TREE" 8097 --conf "$conf" -- bash -c " + EL=\$(python3 -c \"import json;print(json.load(open('$conf'))['log'])\") + curl -s -m 5 http://127.0.0.1:\$PORT/v1/status | python3 -c 'import json,sys;d=json.load(sys.stdin);print(\" engine:\",d[\"engine\"])' + /usr/bin/python3 $PROBE --port \$PORT --log \"\$EL\" --out $OUT/probe-$tag.json --label $tag + " 2>&1 | grep -E '=====|engine:|^\s+\{|wrote|restore' +} + +variant on conv_route on +variant shadow conv_route shadow + +for tag in on shadow; do + arm "$tag" +done +echo "[$(date +%H:%M:%S)] intercept sweep done" diff --git a/dirigo-results/0.2.13-port/intercept/summary.json b/dirigo-results/0.2.13-port/intercept/summary.json new file mode 100644 index 00000000000..9d5eac26a6a --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/summary.json @@ -0,0 +1,97 @@ +{ + "on": { + "ttft_ms": { + "intercept_ms": 242.8, + "ms_per_token": 0.772 + }, + "prefill_ms": { + "intercept_ms": 222.4, + "ms_per_token": 0.766 + }, + "server_ms": [ + 21.4, + 21.8, + 21.9, + 22.3, + 22.8, + 22.8, + 24.5, + 24.8, + 27.2, + 32.0 + ], + "park_ms": [ + 98.1, + 101.8, + 101.9 + ] + }, + "shadow": { + "ttft_ms": { + "intercept_ms": 455.0, + "ms_per_token": 0.79 + }, + "prefill_ms": { + "intercept_ms": 434.1, + "ms_per_token": 0.788 + }, + "server_ms": [ + 20.7, + 21.4, + 21.7, + 21.4, + 21.9, + 21.8, + 23.0, + 22.1, + 23.7, + 24.2 + ], + "park_ms": [ + 101.7, + 104.6, + 100.6, + 102.1, + 101.3, + 101.8, + 102.3, + 101.7, + 100.3, + 99.9, + 101.0, + 102.6, + 101.2, + 100.5, + 100.7, + 102.8, + 101.2, + 102.1, + 104.3, + 110.6 + ] + }, + "base": { + "n": 6, + "prefill_median_ms": 381.5, + "prefill_min_max": [ + 327, + 537 + ] + }, + "nocache": { + "n": 6, + "prefill_median_ms": 324.0, + "prefill_min_max": [ + 323, + 339 + ] + }, + "base2": { + "n": 6, + "prefill_median_ms": 382.0, + "prefill_min_max": [ + 325, + 537 + ] + } +} \ No newline at end of file diff --git a/docs/PERF-NEXT.md b/docs/PERF-NEXT.md index 14028f52985..71e2235f970 100644 --- a/docs/PERF-NEXT.md +++ b/docs/PERF-NEXT.md @@ -36,6 +36,7 @@ important as the open one. | `--ple-row-cache` 4 M / 16 M rows, `--ple-inflight` 1024 | 4 M rows and inflight 1024 are **flat** (inside the 0.2–3.6 % drift band); 16 M rows (1.44 GB) costs **−8 % on fresh 4.2k, −8…−12 % on 8.8k, −17 % on a 4.4k follow-up, −10 % on 30k**. The 94 MB default and inflight 256 stand — [the sweep](../dirigo-results/0.2.13-port/ple-sweep/) | | `spec_window` policy | recovered only half the retrieval loss (+1.3…4.8 % vs +3.6…7.5 %) — left off | | DFlash2 / DSpark drafters | **CUDA/HIP only**; the SYCL delta in 0.2.12 is 3 lines in `prefill.cpp` | +| `--spec 0` to remove drafting entirely | the engine refuses it on a native IQ pack: *"it needs `--native` SHARD1, `--spec T (T >= 2)` and `--prefill` CHUNK"* — so drafting cannot be excluded this way | ## 3. Where the remaining time is @@ -48,13 +49,14 @@ fixed term. | # | idea | evidence it could pay | cost | first test | | --- | --- | --- | --- | --- | -| 1 | ship `conv-route-batch-after` 1–2 | measured **−21 % median TTFT** (0.371 → 0.293 s), summed −29 %, steady state at request 2 instead of 5; the default is still 4 | a flag + a probe | a chat-after-single-shot shape to price the flip-back | -| 2 | attribute the ~330 ms fixed per-request cost | nobody has split it on SYCL: slot init, expert warm-up for the first layers, KV allocation. The PLE read path is now excluded by measurement (§5) | an hour, no code | `bench_prefill` at 8/16/32/64/128/256 tokens | -| 3 | residency-biased router on SYCL | `route_prior.cu` is CUDA-only; here **12,288 of 24,576** experts are resident, so half of every token's experts cross PCIe. Upstream measured +15 % / +37 % on a 12 GB NVIDIA card | engine port, days | decode lanes, 5 interleaved pairs | -| 4 | prefill chunk at 4k/8k/12k with intervals | 16k/32k were tested, but 8,192 was never shown to be the knee | ~20 min | paired, three prompt sizes, per-prompt medians | -| 5 | CPU pool workers and affinity on the 7700X | 15 workers is a 16-thread default; p-core pinning was **+27 %** decode on the 13700K | 15 min | workers 8/12/15/16, `--pool-affinity` on/off | -| 6 | dispatch overhead on SYCL | the Vulkan analysis found ~88 % of per-dispatch cost is fixed; nobody has counted dispatches per token on SYCL | a day | `unitrace` kernel count per token, then fuse the five most frequent small ops | -| 7 | DFlash2 block drafter on SYCL | upstream's headline decode lever; 7 drafts per pass | weeks | only after 1–3 | +| 1 | ship `conv-route-batch-after` 1–2 | measured **−21 % median TTFT** (0.371 → 0.293 s); §6 prices what it saves at **+212 ms per request** while the cache is on, and at `batch_after 4` the first three fresh requests of a run still pay it | a flag + a probe | a chat-after-single-shot shape to price the flip-back | +| 2 | engine-side per-request timer | §6 leaves **~165 ms** inside prefill that no configuration lever explains (PLE reads excluded, cache slots ~58 ms of it); naming it needs a phase timer in the request path | code, a day | timestamp slot setup / first-layer expert reach / KV alloc per request | +| 3 | a one-shot mode that also drops the cache slots | §6: `--prompt-cache 0` is **58 ms cheaper per request** than the router's cache-off path at 75 tokens, and far more stable (324 ms against 382, spread 323–339 against 327–537). The setting is restart-class today | engine work to make it per-request | the same ladder with a 0-slot variant | +| 4 | residency-biased router on SYCL | `route_prior.cu` is CUDA-only; here **12,288 of 24,576** experts are resident, so half of every token's experts cross PCIe. Upstream measured +15 % / +37 % on a 12 GB NVIDIA card | engine port, days | decode lanes, 5 interleaved pairs | +| 5 | prefill chunk at 4k/8k/12k with intervals | 16k/32k were tested, but 8,192 was never shown to be the knee | ~20 min | paired, three prompt sizes, per-prompt medians | +| 6 | CPU pool workers and affinity on the 7700X | 15 workers is a 16-thread default; p-core pinning was **+27 %** decode on the 13700K | 15 min | workers 8/12/15/16, `--pool-affinity` on/off | +| 7 | dispatch overhead on SYCL | the Vulkan analysis found ~88 % of per-dispatch cost is fixed; nobody has counted dispatches per token on SYCL | a day | `unitrace` kernel count per token, then fuse the five most frequent small ops | +| 8 | DFlash2 block drafter on SYCL | upstream's headline decode lever; 7 drafts per pass | weeks | only after 1–3 | ## 5. The PLE sweep, run 2026-11-14 — closed @@ -81,7 +83,36 @@ reuse the KV cache instead of re-reading rows. Artifacts: [`0.2.13-port/ple-sweep/`](../dirigo-results/0.2.13-port/ple-sweep/) — the six configs, the per-leg JSON, the log with each window's restore line, `summary.json`, and the script. -## 6. Method rules these numbers were produced under +## 6. The fixed per-request cost, measured 2026-11-14 + +A ladder of unique prompts (74 … 1,608 real tokens), two arms — the deployed `conv_route: on` and `shadow` (cache +on, like stock) — client TTFT from the first content delta, and the engine's own `prompt … read in X ms` and +`conversation cache: parked … in Y ms` lines for the split: + +| arm | TTFT fit | prefill fit (engine) | server (TTFT − prefill) | park | +| --- | --- | --- | --- | --- | +| `conv_route: on` (cache off for one-shot) | **243 ms + 0.77 ms/token** | **222 ms + 0.77 ms/token** | 21–32 ms | ~100 ms on the first 3 requests only, then none | +| `shadow` (cache on, like stock) | **455 ms + 0.79 ms/token** | **434 ms + 0.79 ms/token** | 21–24 ms | ~101 ms on every request | + +1. **Nothing hides outside the engine's own accounting.** TTFT is prefill plus ~21 ms of server, in both modes; the + unaccounted term that earlier notes called "450–500 ms" is not overhead, it is prefill. +2. **The cache-on state costs +212 ms per request** (intercept 434 against 222) — the prize the router takes. At + `batch_after 4` a run's first three fresh requests still pay it: 484 / 485 / 402 ms at 74 / 74 / 75 tokens against + 290 ms once the cache is off. +3. **Parking is ~101 ms and happens after the answer**, in both modes; it is not part of TTFT, and with the cache off + it stops entirely. +4. **The cache slots cost something even when unused**: `--prompt-cache 0` measured 324 ms median at 75 tokens + (323–339) against 382 ms on the router's cache-off path (327–537), the same prompts and the same build, with a + repeated default arm reproducing 382 ms exactly. +5. **The draft layer could not be switched off to test its share**: `--spec 0` is refused on a native IQ pack, which + needs `--spec T (T >= 2)`. +6. **What is left is ~165 ms** inside prefill after the cache slots' ~58 ms, unexplained by any configuration lever + tried so far (PLE row reads are excluded by §5). Naming it needs a phase timer inside the request path. + +Artifacts: [`0.2.13-port/intercept/`](../dirigo-results/0.2.13-port/intercept/) — both ladders, the three +cause-sweep arms, `summary.json`, the probe and both drivers, and the logs with every restore line. + +## 7. Method rules these numbers were produced under - Paired or interleaved arms; median of repeats; the box's run-to-run spread stated next to the effect. - One prompt per configuration is not a measurement: every lane here has several requests, and a single-prompt