diff --git a/dirigo-results/0.2.13-port/intercept/trace/q2_0/conf-q2_0.json b/dirigo-results/0.2.13-port/intercept/trace/q2_0/conf-q2_0.json new file mode 100644 index 00000000000..75ce827ffc1 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/trace/q2_0/conf-q2_0.json @@ -0,0 +1,61 @@ +{ + "exe": "/home/bob/bin/strata-sycl-host.sh", + "args": [ + "--pack", + "/home/bob/strata-nvme/q2_0", + "--native", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/public/strata-gguf/Q2_0/Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00001-of-00002.gguf", + "--expert-profile", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/data/expert-profile.bin", + "--expert-cache", + "auto", + "--stream-experts", + "--prefill", + "auto", + "--prompt-cache", + "6", + "--prompt-cache-every", + "4096", + "--batch", + "0", + "--spec", + "6", + "--spec-min-p", + "0.5", + "--mtp", + "/home/bob/XeStrata-data/mtp/rt", + "--mtp-draft-vocab", + "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/data/draft_vocab_en.bin", + "--mtp-window", + "8192", + "--max-context", + "131072", + "--kv", + "int8", + "--kv-resident", + "32768", + "--vram-reserve-mib", + "700", + "--conversation-cache-mib", + "2048" + ], + "cwd": "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13", + "tokenizer": "/home/bob/XeStrata-data/packs/coder-iq1_m/tokenizer", + "model_name": "qwen3.8-flash-next-coder-iq1_m", + "log": "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/strata-q2_0-cal.log", + "port": 8095, + "backend": "sycl", + "open_browser": false, + "thinking_policy": "on", + "conv_route": "on", + "conv_route_batch_after": 4, + "short_answer": { + "spec": 0, + "max_tokens": 16 + }, + "spec_window": 0, + "spec_window_needle_min": 6000, + "code_tune": { + "mode": "off" + } +} \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/trace/q2_0/ladder-q2_0.json b/dirigo-results/0.2.13-port/intercept/trace/q2_0/ladder-q2_0.json new file mode 100644 index 00000000000..77a794208c8 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/trace/q2_0/ladder-q2_0.json @@ -0,0 +1,182 @@ +[ + { + "target": 75, + "ttft_ms": 1423.6, + "wall_ms": 1517.6, + "prompt_tokens": 129, + "reused": 0, + "fresh": 129, + "prefill_ms": 1399, + "prefill_tps": 92.2, + "decode_ms": 116, + "park_tokens": 97, + "park_ms": 117.1, + "kv_hit_pct": 93.28, + "kv_mib_from_ram": 1.2, + "server_ms": 24.6, + "label": "q2_0" + }, + { + "target": 75, + "ttft_ms": 1379.6, + "wall_ms": 1474.0, + "prompt_tokens": 129, + "reused": 0, + "fresh": 129, + "prefill_ms": 1354, + "prefill_tps": 95.2, + "decode_ms": 117, + "park_tokens": 137, + "park_ms": 103.5, + "kv_hit_pct": 93.42, + "kv_mib_from_ram": 1.7, + "server_ms": 25.6, + "label": "q2_0" + }, + { + "target": 75, + "ttft_ms": 1280.5, + "wall_ms": 1380.3, + "prompt_tokens": 129, + "reused": 0, + "fresh": 129, + "prefill_ms": 1255, + "prefill_tps": 102.8, + "decode_ms": 122, + "park_tokens": 137, + "park_ms": 104.0, + "server_ms": 25.5, + "label": "q2_0" + }, + { + "target": 75, + "ttft_ms": 1176.0, + "wall_ms": 1270.4, + "prompt_tokens": 129, + "reused": 0, + "fresh": 129, + "prefill_ms": 1151, + "prefill_tps": 112.1, + "decode_ms": 117, + "kv_hit_pct": 92.01, + "kv_mib_from_ram": 1.7, + "server_ms": 25.0, + "label": "q2_0" + }, + { + "target": 512, + "ttft_ms": 1728.3, + "wall_ms": 1823.3, + "prompt_tokens": 459, + "reused": 0, + "fresh": 459, + "prefill_ms": 1701, + "prefill_tps": 269.8, + "decode_ms": 118, + "server_ms": 27.3, + "label": "q2_0" + }, + { + "target": 512, + "ttft_ms": 1677.2, + "wall_ms": 1770.8, + "prompt_tokens": 459, + "reused": 0, + "fresh": 459, + "prefill_ms": 1651, + "prefill_tps": 278.1, + "decode_ms": 117, + "kv_hit_pct": 91.6, + "kv_mib_from_ram": 5.7, + "server_ms": 26.2, + "label": "q2_0" + }, + { + "target": 512, + "ttft_ms": 1627.3, + "wall_ms": 1721.2, + "prompt_tokens": 459, + "reused": 0, + "fresh": 459, + "prefill_ms": 1601, + "prefill_tps": 286.7, + "decode_ms": 117, + "kv_hit_pct": 91.6, + "kv_mib_from_ram": 5.7, + "server_ms": 26.3, + "label": "q2_0" + }, + { + "target": 512, + "ttft_ms": 1660.7, + "wall_ms": 1754.8, + "prompt_tokens": 459, + "reused": 0, + "fresh": 459, + "prefill_ms": 1633, + "prefill_tps": 281.1, + "decode_ms": 117, + "kv_hit_pct": 91.6, + "kv_mib_from_ram": 5.7, + "server_ms": 27.7, + "label": "q2_0" + }, + { + "target": 2048, + "ttft_ms": 2318.1, + "wall_ms": 2413.2, + "prompt_tokens": 1611, + "reused": 0, + "fresh": 1611, + "prefill_ms": 2286, + "prefill_tps": 704.6, + "decode_ms": 121, + "kv_hit_pct": 91.6, + "kv_mib_from_ram": 5.7, + "server_ms": 32.1, + "label": "q2_0" + }, + { + "target": 2048, + "ttft_ms": 2252.3, + "wall_ms": 2347.0, + "prompt_tokens": 1611, + "reused": 0, + "fresh": 1611, + "prefill_ms": 2220, + "prefill_tps": 725.7, + "decode_ms": 120, + "kv_hit_pct": 91.65, + "kv_mib_from_ram": 19.6, + "server_ms": 32.3, + "label": "q2_0" + }, + { + "target": 2048, + "ttft_ms": 2245.0, + "wall_ms": 2340.2, + "prompt_tokens": 1611, + "reused": 0, + "fresh": 1611, + "prefill_ms": 2214, + "prefill_tps": 727.8, + "decode_ms": 121, + "kv_hit_pct": 91.65, + "kv_mib_from_ram": 19.6, + "server_ms": 31.0, + "label": "q2_0" + }, + { + "target": 2048, + "ttft_ms": 2248.8, + "wall_ms": 2343.7, + "prompt_tokens": 1611, + "reused": 0, + "fresh": 1611, + "prefill_ms": 2217, + "prefill_tps": 726.6, + "decode_ms": 121, + "server_ms": 31.8, + "label": "q2_0" + } +] \ No newline at end of file diff --git a/dirigo-results/0.2.13-port/intercept/trace/q2_0/strata-calibrate-prompts.py b/dirigo-results/0.2.13-port/intercept/trace/q2_0/strata-calibrate-prompts.py new file mode 100755 index 00000000000..7a29d6d24a2 --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/trace/q2_0/strata-calibrate-prompts.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Send a small varied corpus to a live server so the engine dumps the routing it needs for a profile. + +tools/collect_profile_traces.py starts the engine once PER prompt (20-60 s each, and ~185 s of profile fill for the Q2_0 +pack). This sends the same kinds of text to ONE running server with --dump-routing, which writes the same record format +that tools/make_profile_mix.py reads — so a profile costs one startup, not forty-six. + + ~/bin/strata-calibrate-prompts.py --url http://127.0.0.1:8097 [--max-new 120] +""" +import argparse +import json +import time +import urllib.request + +CORPUS = [ + ("code", "Write a Python function that merges two sorted lists without using sorted(). Include a docstring and " + "three assertions that exercise the empty, equal-length and unequal-length cases."), + ("code", "Refactor this into idiomatic Go: a loop that reads lines from a file, parses each as JSON, and appends " + "the ones with an 'id' field to a slice. Explain the error handling you chose."), + ("code", "Explain what this shell pipeline does and where it fails on filenames with newlines: " + "find . -name '*.log' -print0 | xargs -0 grep -l ERROR | sort | uniq -c"), + ("chat", "I am planning a three-week trip to Japan in April with two teenagers. Suggest an itinerary that balances " + "cities and nature, and say what I should book early."), + ("chat", "My laptop fan runs constantly since an update last week. Walk me through what to check, in order, without " + "telling me to reinstall the operating system."), + ("math", "A tank fills in 6 hours with pipe A and 9 hours with pipe B. Both run for two hours, then A is closed. " + "How long does B take to finish? Show each step."), + ("math", "Prove that the sum of the first n odd numbers is n squared, then use it to compute 1+3+...+999."), + ("summary", "Summarise in five bullet points: a release note describing a new cache that stores parked conversations " + "in RAM, a flag that switches it off for one-shot traffic, measurements of 5.3 s to 0.29 s on repeats, " + "and a caveat that a wrong switch costs one full prefill."), + ("summary", "Extract the dates, owners and deadlines from: 'Marta will finish the migration by 3 March. The audit " + "starts 10 February and is owned by Priya. Retest on 28 February, before the freeze.'"), + ("writing", "Rewrite this sentence three ways, one formal, one plain, one for a child: 'Subsequent to the completion " + "of the review, the team will initiate the deployment process.'"), + ("science", "Why does a metal spoon feel colder than a wooden one at the same room temperature? Answer with the " + "mechanism, not the analogy."), + ("tool", "Call the weather tool for Seattle tomorrow, then, if it will rain, call the calendar tool to move my 9 am " + "run to the same time on Friday."), +] + + +def ask(url, prompt, max_new): + body = json.dumps({"model": "m", "max_tokens": max_new, "temperature": 0.0, + "messages": [{"role": "user", "content": prompt}]}).encode() + req = urllib.request.Request(url + "/v1/chat/completions", data=body, + headers={"Content-Type": "application/json"}) + t0 = time.time() + with urllib.request.urlopen(req, timeout=600) as r: + d = json.loads(r.read().decode()) + msg = (d.get("choices") or [{}])[0].get("message") or {} + text = msg.get("content") or msg.get("reasoning_content") or "" + return time.time() - t0, len(text) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--url", default="http://127.0.0.1:8097") + ap.add_argument("--max-new", type=int, default=120) + args = ap.parse_args() + total = 0 + for i, (kind, prompt) in enumerate(CORPUS): + wall, chars = ask(args.url, prompt, args.max_new) + total += chars + print(f" {i+1:>2}/{len(CORPUS)} {kind:<8} {wall:5.1f} s {chars:>4} chars", flush=True) + print(f"corpus done: {len(CORPUS)} prompts, {total} characters of output routed") + + +if __name__ == "__main__": + main() diff --git a/dirigo-results/0.2.13-port/intercept/trace/q2_0/strata-calibrate-q2_0.sh b/dirigo-results/0.2.13-port/intercept/trace/q2_0/strata-calibrate-q2_0.sh new file mode 100755 index 00000000000..36fe929675f --- /dev/null +++ b/dirigo-results/0.2.13-port/intercept/trace/q2_0/strata-calibrate-q2_0.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# strata-calibrate-q2_0.sh - build an expert-cache profile for the Q2_0 pack from ONE serve session. +# +# The published path (tools/collect_profile_traces.py) starts the engine once per prompt; for Q2_0 each start pays ~185 s +# of profile fill, so 45 prompts would be hours. Instead: one startup with --dump-routing, ~12 varied prompts, then +# tools/make_profile_mix.py over that trace - same record format, one startup. +# +# usage: strata-calibrate-q2_0.sh [max-new] +# UNDO: rm this file and ~/strata-trace-run/routing-q2_0.txt +set -uo pipefail +MAXNEW=${1:-120} +S=/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731 +TREE=$S/strata-0.2.13 +OUT=$HOME/strata-trace-run +CONF=$OUT/conf-q2_0-route.json +TRACE=$OUT/routing-q2_0.txt +mkdir -p "$OUT" +rm -f "$TRACE" + +python3 - "$OUT/conf-q2_0.json" "$CONF" <<'PY' +import json, sys +src, dst = sys.argv[1:3] +d = json.load(open(src)); a = list(d["args"]) +# --dump-routing is the CUDA/HIP and SYCL-host mechanism; on the SYCL device path it writes nothing (measured: 13 +# prompts, 0-byte file). The device-side counter is STRATA_USAGE_DUMP, which works with STRATA_VERIFY_NO_HOST. +if "--dump-routing" in a: + del a[a.index("--dump-routing"):a.index("--dump-routing") + 2] +d["args"] = a +d["log"] = "/media/bob/3d651e2c-e9a4-4758-ba77-863725fe3731/strata-0.2.13/strata-q2_0-usage.log" +json.dump(d, open(dst, "w"), indent=1) +print("STRATA_USAGE_DUMP ->", sys.argv[0] and "set for the engine") +PY + +echo "[$(date +%H:%M:%S)] ===== calibrating q2_0 (max-new $MAXNEW)" +STRATA_USAGE_DUMP=$TRACE ~/bin/strata-card-window.sh "$TREE" 8097 --conf "$CONF" -- bash -c " + curl -s -m 5 http://127.0.0.1:\$PORT/v1/status | python3 -c 'import json,sys;d=json.load(sys.stdin);print(\" engine:\",d[\"engine\"])' + /usr/bin/python3 $HOME/bin/strata-calibrate-prompts.py --url http://127.0.0.1:\$PORT --max-new $MAXNEW +" 2>&1 | grep -aE '=====|engine:|/[0-9]+ |corpus done|restore' +echo "[$(date +%H:%M:%S)] routing trace: $(ls -la "$TRACE" 2>/dev/null | awk '{print $5" bytes"}')" diff --git a/docs/PERF-NEXT.md b/docs/PERF-NEXT.md index f1b8dddcaf1..c5577467462 100644 --- a/docs/PERF-NEXT.md +++ b/docs/PERF-NEXT.md @@ -212,22 +212,43 @@ first token would take 72–188 ms off TTFT — the largest remaining cheap win, rather than a research project. Artifacts: [`0.2.13-port/intercept/trace/`](../dirigo-results/0.2.13-port/intercept/trace/) — both ladders, the engine -logs with every trace line, the parser and the window driver. - -### Cross-weight check: blocked, three engine constraints - -The same trace was attempted on a second weight set (the Q2_0 pack, 34 GB, copied to NVMe from the cold disk for this) -and could not start: - -| attempt | engine's answer | +logs with every trace line, the parser and the window driver, and [`trace/q2_0/`](../dirigo-results/0.2.13-port/intercept/trace/q2_0/) +for the second weight set: its ladder, its config, its engine log, and both calibration attempts. + +### Cross-weight check: it ran, and the fix was the profile + +The first three attempts failed, and the last failure named the real problem: the config pointed `--expert-profile` at +`data/expert-profile-mine.bin`, which is **48×256** — the profile of the *pruned* coder model (GSQ-RCO Coder keeps 256 of +512 experts). The Q2_0 pack declares 48×512, and the tree already ships `data/expert-profile.bin` at 48×512. Pointed at +that, Q2_0 served without any calibration: + +| real tokens | Q2_0 read | Q2_0 refill | lent slots | refill per slot | IQ1_M read | IQ1_M refill | +| --- | --- | --- | --- | --- | --- | --- | +| 128 | 1,075 ms | 75.2 ms | 313 | 0.240 | 252 ms | 71.7 ms | +| 458 | 1,556 ms | 81.6 ms | 365 | 0.224 | 432 ms | 82.6 ms | +| 1,610 | 2,055 ms | 160.9 ms | 726 | 0.222 | 1,245 ms | 188.2 ms | + +1. **The mechanism reproduces on different weights.** The refill is there, sits between `read` and `prompt done`, and + scales with lent slots: 0.22–0.26 ms/slot on Q2_0 against 0.319 on IQ1_M — a different rate, as different expert + payloads should give, but the same phase and the same ordering. +2. **The absolute numbers are not comparable, and are not used above.** Q2_0 is the **unpruned 512-expert** model: its + `--expert-cache auto` found 19,208 of 24,576 slots (24.73 GiB) and mirrored the other 6,761 experts in 8.70 GiB of + pinned host memory, so ~27 % of experts cross PCIe; IQ1_M is the pruned 256-expert variant. Q2_0's dense shard also + sits on the cold disk (there was no room for 37.6 GB on this NVMe), which inflates its read further. +3. **So §9's four findings stand on two weight sets for the mechanism, and on one for the magnitudes.** The ladder in + §9 remains the IQ1_M one; nothing above claims Q2_0 is slower or faster than it. + +**Calibrating Q2_0 by its own traces is blocked, three ways** (all logs shipped): + +| attempt | result | | --- | --- | -| Q2_0 pack + `--ple-gguf` at the shared PLE shard | `native dense: duplicate split shard number` | -| pack only, embedding from the GGUF, no `--native` | `is a native (IQ) pack: it needs --native SHARD1, --spec T (T >= 2) and --prefill CHUNK` | -| fixed `--expert-cache 12288`, no profile | `REFUSED: 11158 experts are neither in VRAM nor mirrored` | +| serve + `--dump-routing` | 0-byte file after 13 prompts — the SYCL device path does not write it | +| serve + `STRATA_USAGE_DUMP` | no file after 12 prompts | +| `tools/collect_profile_traces.py --mode usage` | `--spec needs the device residency table (--expert-profile, --expert-cache and the token graph)`; where it did prefill, 37 tokens took **80.3 s** at 0.5 tok/s (5,364 experts streamed, 77.4 s of it on the host) | -The shipped `expert-profile-mine.bin` is 48×256 and the Q2_0 pack declares 48×512, so the second weight set needs its -own calibration run before it can serve. Until then every number in this inventory stands on one weight set (IQ1_M), -which is stated in §10 and is the honest bound on all of it. +A Q2_0-specific profile therefore needs the serve-mode usage dump to flush (an engine fix) or the one-shot collector to +work on a native pack — not another parameter. Until then Q2_0 runs on the shipped full profile, which is enough for a +mechanism check but not for a ranking study. ## 10. Method rules these numbers were produced under