From ba4e80dd3e82df58f8c0bf4b3a8ae68707eb3640 Mon Sep 17 00:00:00 2001 From: rwkeyes <86691029+rwkeyes@users.noreply.github.com> Date: Sat, 14 Nov 2026 19:37:35 -0500 Subject: [PATCH] perf: the cross-weight check ran after all - the profile was the wrong shape The Q2_0 arm could not start, and the reason was not the pack: the config pointed --expert-profile at data/expert-profile-mine.bin, which is 48x256, the profile of the PRUNED coder model (GSQ-RCO Coder keeps 256 of 512 experts). The Q2_0 pack declares 48x512 and the tree already ships data/expert-profile.bin at 48x512. Pointed at that, Q2_0 served with no calibration at all. tokens Q2_0 read Q2_0 refill slots per slot IQ1_M read IQ1_M refill 128 1075 ms 75.2 ms 313 0.240 252 ms 71.7 ms 458 1556 ms 81.6 ms 365 0.224 432 ms 82.6 ms 1610 2055 ms 160.9 ms 726 0.222 1245 ms 188.2 ms So the mechanism reproduces on different weights - same phase, same position between read and prompt done, same scaling with lent slots, a different rate (0.22-0.26 against 0.319 ms/slot) as different expert payloads should give. The magnitudes are NOT comparable and are not used: Q2_0 is the unpruned 512-expert model with 19,208 of 24,576 slots resident and 6,761 experts mirrored in 8.70 GiB of pinned host memory, and its dense shard sits on the cold disk. Calibrating Q2_0 from its own traces is blocked three ways, all logged: serve + --dump-routing writes a 0-byte file (SYCL device path does not write it), serve + STRATA_USAGE_DUMP writes nothing, and the one-shot collector dies on "--spec needs the device residency table" - and where it did prefill, 37 tokens took 80.3 s at 0.5 tok/s. Corrects the previous claim that the cross-weight check was blocked. --- .../intercept/trace/q2_0/conf-q2_0.json | 61 ++++++ .../intercept/trace/q2_0/ladder-q2_0.json | 182 ++++++++++++++++++ .../trace/q2_0/strata-calibrate-prompts.py | 70 +++++++ .../trace/q2_0/strata-calibrate-q2_0.sh | 39 ++++ docs/PERF-NEXT.md | 49 +++-- 5 files changed, 387 insertions(+), 14 deletions(-) create mode 100644 dirigo-results/0.2.13-port/intercept/trace/q2_0/conf-q2_0.json create mode 100644 dirigo-results/0.2.13-port/intercept/trace/q2_0/ladder-q2_0.json create mode 100755 dirigo-results/0.2.13-port/intercept/trace/q2_0/strata-calibrate-prompts.py create mode 100755 dirigo-results/0.2.13-port/intercept/trace/q2_0/strata-calibrate-q2_0.sh diff --git a/dirigo-results/0.2.13-port/intercept/trace/q2_0/conf-q2_0.json b/dirigo-results/0.2.13-port/intercept/trace/q2_0/conf-q2_0.json new file mode 100644 index 00000000000..75ce827ffc1 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/trace/q2_0/conf-q2_0.json @@ -0,0 +1,61 @@ +{ + "exe": "/home/bob/bin/strata-sycl-host.sh", + "args": [ + "--pack", + "/home/bob/strata-nvme/q2_0", + "--native", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/public/strata-gguf/Q2_0/Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00001-of-00002.gguf", + "--expert-profile", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/data/expert-profile.bin", + "--expert-cache", + "auto", + "--stream-experts", + "--prefill", + "auto", + "--prompt-cache", + "6", + "--prompt-cache-every", + "4096", + "--batch", + "0", + "--spec", + "6", + "--spec-min-p", + "0.5", + "--mtp", + "/home/bob/XeStrata-data/mtp/rt", + "--mtp-draft-vocab", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/data/draft_vocab_en.bin", + "--mtp-window", + "8192", + "--max-context", + "131072", + "--kv", + "int8", + "--kv-resident", + "32768", + "--vram-reserve-mib", + "700", + "--conversation-cache-mib", + "2048" + ], + "cwd": "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13", + "tokenizer": "/home/bob/XeStrata-data/packs/coder-iq1_m/tokenizer", + "model_name": "qwen3.8-flash-next-coder-iq1_m", + "log": "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/strata-q2_0-cal.log", + "port": 8095, + "backend": "sycl", + "open_browser": false, + "thinking_policy": "on", + "conv_route": "on", + "conv_route_batch_after": 4, + "short_answer": { + "spec": 0, + "max_tokens": 16 + }, + "spec_window": 0, + "spec_window_needle_min": 6000, + "code_tune": { + "mode": "off" + } +} \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/trace/q2_0/ladder-q2_0.json b/dirigo-results/0.2.13-port/intercept/trace/q2_0/ladder-q2_0.json new file mode 100644 index 00000000000..77a794208c8 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/trace/q2_0/ladder-q2_0.json @@ -0,0 +1,182 @@ +[ + { + "target": 75, + "ttft_ms": 1423.6, + "wall_ms": 1517.6, + "prompt_tokens": 129, + "reused": 0, + "fresh": 129, + "prefill_ms": 1399, + "prefill_tps": 92.2, + "decode_ms": 116, + "park_tokens": 97, + "park_ms": 117.1, + "kv_hit_pct": 93.28, + "kv_mib_from_ram": 1.2, + "server_ms": 24.6, + "label": "q2_0" + }, + { + "target": 75, + "ttft_ms": 1379.6, + "wall_ms": 1474.0, + "prompt_tokens": 129, + "reused": 0, + "fresh": 129, + "prefill_ms": 1354, + "prefill_tps": 95.2, + "decode_ms": 117, + "park_tokens": 137, + "park_ms": 103.5, + "kv_hit_pct": 93.42, + "kv_mib_from_ram": 1.7, + "server_ms": 25.6, + "label": "q2_0" + }, + { + "target": 75, + "ttft_ms": 1280.5, + "wall_ms": 1380.3, + "prompt_tokens": 129, + "reused": 0, + "fresh": 129, + "prefill_ms": 1255, + "prefill_tps": 102.8, + "decode_ms": 122, + "park_tokens": 137, + "park_ms": 104.0, + "server_ms": 25.5, + "label": "q2_0" + }, + { + "target": 75, + "ttft_ms": 1176.0, + "wall_ms": 1270.4, + "prompt_tokens": 129, + "reused": 0, + "fresh": 129, + "prefill_ms": 1151, + "prefill_tps": 112.1, + "decode_ms": 117, + "kv_hit_pct": 92.01, + "kv_mib_from_ram": 1.7, + "server_ms": 25.0, + "label": "q2_0" + }, + { + "target": 512, + "ttft_ms": 1728.3, + "wall_ms": 1823.3, + "prompt_tokens": 459, + "reused": 0, + "fresh": 459, + "prefill_ms": 1701, + "prefill_tps": 269.8, + "decode_ms": 118, + "server_ms": 27.3, + "label": "q2_0" + }, + { + "target": 512, + "ttft_ms": 1677.2, + "wall_ms": 1770.8, + "prompt_tokens": 459, + "reused": 0, + "fresh": 459, + "prefill_ms": 1651, + "prefill_tps": 278.1, + "decode_ms": 117, + "kv_hit_pct": 91.6, + "kv_mib_from_ram": 5.7, + "server_ms": 26.2, + "label": "q2_0" + }, + { + "target": 512, + "ttft_ms": 1627.3, + "wall_ms": 1721.2, + "prompt_tokens": 459, + "reused": 0, + "fresh": 459, + "prefill_ms": 1601, + "prefill_tps": 286.7, + "decode_ms": 117, + "kv_hit_pct": 91.6, + "kv_mib_from_ram": 5.7, + "server_ms": 26.3, + "label": "q2_0" + }, + { + "target": 512, + "ttft_ms": 1660.7, + "wall_ms": 1754.8, + "prompt_tokens": 459, + "reused": 0, + "fresh": 459, + "prefill_ms": 1633, + "prefill_tps": 281.1, + "decode_ms": 117, + "kv_hit_pct": 91.6, + "kv_mib_from_ram": 5.7, + "server_ms": 27.7, + "label": "q2_0" + }, + { + "target": 2048, + "ttft_ms": 2318.1, + "wall_ms": 2413.2, + "prompt_tokens": 1611, + "reused": 0, + "fresh": 1611, + "prefill_ms": 2286, + "prefill_tps": 704.6, + "decode_ms": 121, + "kv_hit_pct": 91.6, + "kv_mib_from_ram": 5.7, + "server_ms": 32.1, + "label": "q2_0" + }, + { + "target": 2048, + "ttft_ms": 2252.3, + "wall_ms": 2347.0, + "prompt_tokens": 1611, + "reused": 0, + "fresh": 1611, + "prefill_ms": 2220, + "prefill_tps": 725.7, + "decode_ms": 120, + "kv_hit_pct": 91.65, + "kv_mib_from_ram": 19.6, + "server_ms": 32.3, + "label": "q2_0" + }, + { + "target": 2048, + "ttft_ms": 2245.0, + "wall_ms": 2340.2, + "prompt_tokens": 1611, + "reused": 0, + "fresh": 1611, + "prefill_ms": 2214, + "prefill_tps": 727.8, + "decode_ms": 121, + "kv_hit_pct": 91.65, + "kv_mib_from_ram": 19.6, + "server_ms": 31.0, + "label": "q2_0" + }, + { + "target": 2048, + "ttft_ms": 2248.8, + "wall_ms": 2343.7, + "prompt_tokens": 1611, + "reused": 0, + "fresh": 1611, + "prefill_ms": 2217, + "prefill_tps": 726.6, + "decode_ms": 121, + "server_ms": 31.8, + "label": "q2_0" + } +] \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/trace/q2_0/strata-calibrate-prompts.py b/dirigo-results/0.2.13-port/intercept/trace/q2_0/strata-calibrate-prompts.py new file mode 100755 index 00000000000..7a29d6d24a2 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/trace/q2_0/strata-calibrate-prompts.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Send a small varied corpus to a live server so the engine dumps the routing it needs for a profile. + +tools/collect_profile_traces.py starts the engine once PER prompt (20-60 s each, and ~185 s of profile fill for the Q2_0 +pack). This sends the same kinds of text to ONE running server with --dump-routing, which writes the same record format +that tools/make_profile_mix.py reads — so a profile costs one startup, not forty-six. + + ~/bin/strata-calibrate-prompts.py --url http://127.0.0.1:8097 [--max-new 120] +""" +import argparse +import json +import time +import urllib.request + +CORPUS = [ + ("code", "Write a Python function that merges two sorted lists without using sorted(). Include a docstring and " + "three assertions that exercise the empty, equal-length and unequal-length cases."), + ("code", "Refactor this into idiomatic Go: a loop that reads lines from a file, parses each as JSON, and appends " + "the ones with an 'id' field to a slice. Explain the error handling you chose."), + ("code", "Explain what this shell pipeline does and where it fails on filenames with newlines: " + "find . -name '*.log' -print0 | xargs -0 grep -l ERROR | sort | uniq -c"), + ("chat", "I am planning a three-week trip to Japan in April with two teenagers. Suggest an itinerary that balances " + "cities and nature, and say what I should book early."), + ("chat", "My laptop fan runs constantly since an update last week. Walk me through what to check, in order, without " + "telling me to reinstall the operating system."), + ("math", "A tank fills in 6 hours with pipe A and 9 hours with pipe B. Both run for two hours, then A is closed. " + "How long does B take to finish? Show each step."), + ("math", "Prove that the sum of the first n odd numbers is n squared, then use it to compute 1+3+...+999."), + ("summary", "Summarise in five bullet points: a release note describing a new cache that stores parked conversations " + "in RAM, a flag that switches it off for one-shot traffic, measurements of 5.3 s to 0.29 s on repeats, " + "and a caveat that a wrong switch costs one full prefill."), + ("summary", "Extract the dates, owners and deadlines from: 'Marta will finish the migration by 3 March. The audit " + "starts 10 February and is owned by Priya. Retest on 28 February, before the freeze.'"), + ("writing", "Rewrite this sentence three ways, one formal, one plain, one for a child: 'Subsequent to the completion " + "of the review, the team will initiate the deployment process.'"), + ("science", "Why does a metal spoon feel colder than a wooden one at the same room temperature? Answer with the " + "mechanism, not the analogy."), + ("tool", "Call the weather tool for Seattle tomorrow, then, if it will rain, call the calendar tool to move my 9 am " + "run to the same time on Friday."), +] + + +def ask(url, prompt, max_new): + body = json.dumps({"model": "m", "max_tokens": max_new, "temperature": 0.0, + "messages": [{"role": "user", "content": prompt}]}).encode() + req = urllib.request.Request(url + "/v1/chat/completions", data=body, + headers={"Content-Type": "application/json"}) + t0 = time.time() + with urllib.request.urlopen(req, timeout=600) as r: + d = json.loads(r.read().decode()) + msg = (d.get("choices") or [{}])[0].get("message") or {} + text = msg.get("content") or msg.get("reasoning_content") or "" + return time.time() - t0, len(text) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--url", default="http://127.0.0.1:8097") + ap.add_argument("--max-new", type=int, default=120) + args = ap.parse_args() + total = 0 + for i, (kind, prompt) in enumerate(CORPUS): + wall, chars = ask(args.url, prompt, args.max_new) + total += chars + print(f" {i+1:>2}/{len(CORPUS)} {kind:<8} {wall:5.1f} s {chars:>4} chars", flush=True) + print(f"corpus done: {len(CORPUS)} prompts, {total} characters of output routed") + + +if __name__ == "__main__": + main() diff --git a/dirigo-results/0.2.13-port/intercept/trace/q2_0/strata-calibrate-q2_0.sh b/dirigo-results/0.2.13-port/intercept/trace/q2_0/strata-calibrate-q2_0.sh new file mode 100755 index 00000000000..36fe929675f --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/trace/q2_0/strata-calibrate-q2_0.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# strata-calibrate-q2_0.sh - build an expert-cache profile for the Q2_0 pack from ONE serve session. +# +# The published path (tools/collect_profile_traces.py) starts the engine once per prompt; for Q2_0 each start pays ~185 s +# of profile fill, so 45 prompts would be hours. Instead: one startup with --dump-routing, ~12 varied prompts, then +# tools/make_profile_mix.py over that trace - same record format, one startup. +# +# usage: strata-calibrate-q2_0.sh [max-new] +# UNDO: rm this file and ~/strata-trace-run/routing-q2_0.txt +set -uo pipefail +MAXNEW=${1:-120} +S=/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731 +TREE=$S/strata-0.2.13 +OUT=$HOME/strata-trace-run +CONF=$OUT/conf-q2_0-route.json +TRACE=$OUT/routing-q2_0.txt +mkdir -p "$OUT" +rm -f "$TRACE" + +python3 - "$OUT/conf-q2_0.json" "$CONF" <<'PY' +import json, sys +src, dst = sys.argv[1:3] +d = json.load(open(src)); a = list(d["args"]) +# --dump-routing is the CUDA/HIP and SYCL-host mechanism; on the SYCL device path it writes nothing (measured: 13 +# prompts, 0-byte file). The device-side counter is STRATA_USAGE_DUMP, which works with STRATA_VERIFY_NO_HOST. +if "--dump-routing" in a: + del a[a.index("--dump-routing"):a.index("--dump-routing") + 2] +d["args"] = a +d["log"] = "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/strata-q2_0-usage.log" +json.dump(d, open(dst, "w"), indent=1) +print("STRATA_USAGE_DUMP ->", sys.argv[0] and "set for the engine") +PY + +echo "[$(date +%H:%M:%S)] ===== calibrating q2_0 (max-new $MAXNEW)" +STRATA_USAGE_DUMP=$TRACE ~/bin/strata-card-window.sh "$TREE" 8097 --conf "$CONF" -- bash -c " + curl -s -m 5 http://127.0.0.1:\$PORT/v1/status | python3 -c 'import json,sys;d=json.load(sys.stdin);print(\" engine:\",d[\"engine\"])' + /usr/bin/python3 $HOME/bin/strata-calibrate-prompts.py --url http://127.0.0.1:\$PORT --max-new $MAXNEW +" 2>&1 | grep -aE '=====|engine:|/[0-9]+ |corpus done|restore' +echo "[$(date +%H:%M:%S)] routing trace: $(ls -la "$TRACE" 2>/dev/null | awk '{print $5" bytes"}')" diff --git a/docs/PERF-NEXT.md b/docs/PERF-NEXT.md index f1b8dddcaf1..c5577467462 100644 --- a/docs/PERF-NEXT.md +++ b/docs/PERF-NEXT.md @@ -212,22 +212,43 @@ first token would take 72–188 ms off TTFT — the largest remaining cheap win, rather than a research project. Artifacts: [`0.2.13-port/intercept/trace/`](../dirigo-results/0.2.13-port/intercept/trace/) — both ladders, the engine -logs with every trace line, the parser and the window driver. - -### Cross-weight check: blocked, three engine constraints - -The same trace was attempted on a second weight set (the Q2_0 pack, 34 GB, copied to NVMe from the cold disk for this) -and could not start: - -| attempt | engine's answer | +logs with every trace line, the parser and the window driver, and [`trace/q2_0/`](../dirigo-results/0.2.13-port/intercept/trace/q2_0/) +for the second weight set: its ladder, its config, its engine log, and both calibration attempts. + +### Cross-weight check: it ran, and the fix was the profile + +The first three attempts failed, and the last failure named the real problem: the config pointed `--expert-profile` at +`data/expert-profile-mine.bin`, which is **48×256** — the profile of the *pruned* coder model (GSQ-RCO Coder keeps 256 of +512 experts). The Q2_0 pack declares 48×512, and the tree already ships `data/expert-profile.bin` at 48×512. Pointed at +that, Q2_0 served without any calibration: + +| real tokens | Q2_0 read | Q2_0 refill | lent slots | refill per slot | IQ1_M read | IQ1_M refill | +| --- | --- | --- | --- | --- | --- | --- | +| 128 | 1,075 ms | 75.2 ms | 313 | 0.240 | 252 ms | 71.7 ms | +| 458 | 1,556 ms | 81.6 ms | 365 | 0.224 | 432 ms | 82.6 ms | +| 1,610 | 2,055 ms | 160.9 ms | 726 | 0.222 | 1,245 ms | 188.2 ms | + +1. **The mechanism reproduces on different weights.** The refill is there, sits between `read` and `prompt done`, and + scales with lent slots: 0.22–0.26 ms/slot on Q2_0 against 0.319 on IQ1_M — a different rate, as different expert + payloads should give, but the same phase and the same ordering. +2. **The absolute numbers are not comparable, and are not used above.** Q2_0 is the **unpruned 512-expert** model: its + `--expert-cache auto` found 19,208 of 24,576 slots (24.73 GiB) and mirrored the other 6,761 experts in 8.70 GiB of + pinned host memory, so ~27 % of experts cross PCIe; IQ1_M is the pruned 256-expert variant. Q2_0's dense shard also + sits on the cold disk (there was no room for 37.6 GB on this NVMe), which inflates its read further. +3. **So §9's four findings stand on two weight sets for the mechanism, and on one for the magnitudes.** The ladder in + §9 remains the IQ1_M one; nothing above claims Q2_0 is slower or faster than it. + +**Calibrating Q2_0 by its own traces is blocked, three ways** (all logs shipped): + +| attempt | result | | --- | --- | -| Q2_0 pack + `--ple-gguf` at the shared PLE shard | `native dense: duplicate split shard number` | -| pack only, embedding from the GGUF, no `--native` | `is a native (IQ) pack: it needs --native SHARD1, --spec T (T >= 2) and --prefill CHUNK` | -| fixed `--expert-cache 12288`, no profile | `REFUSED: 11158 experts are neither in VRAM nor mirrored` | +| serve + `--dump-routing` | 0-byte file after 13 prompts — the SYCL device path does not write it | +| serve + `STRATA_USAGE_DUMP` | no file after 12 prompts | +| `tools/collect_profile_traces.py --mode usage` | `--spec needs the device residency table (--expert-profile, --expert-cache and the token graph)`; where it did prefill, 37 tokens took **80.3 s** at 0.5 tok/s (5,364 experts streamed, 77.4 s of it on the host) | -The shipped `expert-profile-mine.bin` is 48×256 and the Q2_0 pack declares 48×512, so the second weight set needs its -own calibration run before it can serve. Until then every number in this inventory stands on one weight set (IQ1_M), -which is stated in §10 and is the honest bound on all of it. +A Q2_0-specific profile therefore needs the serve-mode usage dump to flush (an engine fix) or the one-shot collector to +work on a native pack — not another parameter. Until then Q2_0 runs on the shipped full profile, which is enough for a +mechanism check but not for a ranking study. ## 10. Method rules these numbers were produced under