From b523c900bb110b22012af8e2f8891b3363edd499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:55:54 +0200 Subject: [PATCH 01/31] Flash Next: opt-in continuous batching on native Metal Pack session windows into one target layer loop behind the shared lane core. Isolate recurrent and attention caches, normalize kept states with a fenced GPU copy, preserve draft and snapshot semantics, and keep the single-reply and TP drivers as defaults. Include CPU metadata contracts, checkpoint exactness/lifecycle gates and served concurrency comparisons. --- README.md | 4 +- build.zig | 8 + docs/recipes/qwen3.8-flash-next.md | 43 +++ tools/zig/flashnext_batch_prompts.py | 36 ++ tools/zig/flashnext_batch_serve.py | 159 +++++++++ zig/flashnext_host.zig | 1 + zig/src/families/flashnext/backend.zig | 324 ++++++++++++++++++ zig/src/families/flashnext/batch_forward.zig | 122 +++++++ zig/src/families/flashnext/batch_meta.zig | 59 ++++ .../families/flashnext/batch_meta_test.zig | 64 ++++ zig/src/families/flashnext/replay.zig | 11 +- zig/src/families/flashnext/session.zig | 139 ++++++++ zig/src/families/flashnext/snapshot.zig | 11 +- zig/src/native/flashnext_batch_host.zig | 90 +++++ zig/src/native/metal.zig | 11 + zig/src/tensorfold.zig | 4 + zig/tests/flashnext_batch.zig | 168 +++++++++ 17 files changed, 1242 insertions(+), 12 deletions(-) create mode 100644 tools/zig/flashnext_batch_prompts.py create mode 100644 tools/zig/flashnext_batch_serve.py create mode 100644 zig/src/families/flashnext/backend.zig create mode 100644 zig/src/families/flashnext/batch_forward.zig create mode 100644 zig/src/families/flashnext/batch_meta.zig create mode 100644 zig/src/families/flashnext/batch_meta_test.zig create mode 100644 zig/src/families/flashnext/session.zig create mode 100644 zig/src/native/flashnext_batch_host.zig create mode 100644 zig/tests/flashnext_batch.zig diff --git a/README.md b/README.md index 91d4cc1fb7..d6ae433b1d 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,9 @@ Every accepted draft must equal the token the same native engine would produce w A resumed request must equal fresh execution, and each concurrent stream must equal its solo run. The comparison fixes the checkpoint, backend, settings and runtime. Different quantizations and different backends can produce different outputs. -Flash Next and Qwen3.8-27B run one active reply per engine; Nemotron, GLM and the 2B model use shared lane rounds. +Flash Next on Metal uses shared target rounds with an explicit `--parallel N` above one on a single Mac. +Its default `auto`, `--parallel 1` and two-Mac speed-up mode serve one active reply; MTP heads remain serial. +On Metal, Qwen3.8-27B runs one active reply per engine; Nemotron, GLM and the 2B model use shared lane rounds. ## Models on disk diff --git a/build.zig b/build.zig index 9515dc3618..2b2ff99d90 100644 --- a/build.zig +++ b/build.zig @@ -240,6 +240,14 @@ fn nativeServer(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.b }) }); b.installArtifact(reuse); b.step("tf-flashnext-reuse", "Flash Next prompt reuse against fresh prompt passes: replies at every depth and kept states' bytes").dependOn(&b.addInstallArtifact(reuse, .{}).step); + const batch = b.addExecutable(.{ .name = "tf-flashnext-batch", .root_module = b.createModule(.{ + .root_source_file = b.path("zig/tests/flashnext_batch.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + .imports = &.{ .{ .name = "metal", .module = metal }, .{ .name = "tensorfold", .module = engine } }, + }) }); + b.step("tf-flashnext-batch", "Flash Next shared rounds against fresh legacy replies with prompt admission and cancellation").dependOn(&b.addInstallArtifact(batch, .{}).step); return exe; } diff --git a/docs/recipes/qwen3.8-flash-next.md b/docs/recipes/qwen3.8-flash-next.md index 0dafc7fdbf..613ae04d5e 100644 --- a/docs/recipes/qwen3.8-flash-next.md +++ b/docs/recipes/qwen3.8-flash-next.md @@ -57,6 +57,49 @@ path, resolved matmul route and GPU identity; changing arithmetic requires a fre Load-time shared-forward checks compare each stream with its own call. A failed check limits forwards to one stream, while successful checks allow the lane engine to combine requests. +## Native Metal shared sessions + +The native 6-bit MTP engine enables shared target rounds with an explicit `--parallel N` above one: + +```bash +tensorfold-native serve TensorFold/Qwen3.8-Flash-Next-MLX-6bit-MTP \ + --parallel 4 --context 16384 --temperature 0 +``` + +Each request holds its own recurrent state, attention cache, n-gram history and MTP cache. The scheduler packs +up to 16 target rows, including draft rows, into one layer loop. Projections and experts process those rows +together; recurrence, convolution and attention read each request's own state. MTP heads draft serially. +New requests join between rounds, and cancelling a request releases its slot for another request. + +`auto` and `--parallel 1` retain the existing single-reply driver. Shared sessions currently run on one Mac; +`--parallel N` above one with `--speed-up` is refused. Use `--parallel 1` for the two-Mac link. + +Additional slots allocate caches for the selected context. Startup reserves their metadata and the prompt-cache +budget, then checks the physical footprint before accepting requests. A fixed slot count that does not fit is +refused; lower the count, context or prompt-cache budget. `--prompt-cache-over-cap` overrides only the prompt-cache +budget. Greedy decoding remains required, so clients must send temperature 0. + +The checkpoint gate compares shared replies with fresh legacy replies, with drafting off and on, then checks +prompt admission, cancellation, slot reuse and a restored prefix while another request remains active: + +```bash +zig build tf-flashnext-batch +python tools/zig/flashnext_batch_prompts.py MODEL_DIR /tmp/flashnext-prompts +FZ_N=128 zig-out/bin/tf-flashnext-batch MODEL_DIR /tmp/flashnext-prompts/*.json +``` + +Run with one model process on the GPU. For a served comparison with identical requests and settings: + +```bash +zig build native +python tools/zig/flashnext_batch_serve.py zig-out/native/bin/tensorfold-native \ + MODEL_DIR /tmp/flashnext-prompts /tmp/flashnext-served --parallel 1,4 +``` + +The served gate starts one server at a time on loopback port 52417, compares response bodies with the serial +reference and records wall time, generated tokens and peak running and waiting requests. End-to-end time includes +the prompts. The checkpoint gate separately establishes exact token equality. + ## CUDA On CUDA Flash Next serves NVFP4 and EXL3 checkpoints, and the MLX 4-bit checkpoint as the portable option: the same diff --git a/tools/zig/flashnext_batch_prompts.py b/tools/zig/flashnext_batch_prompts.py new file mode 100644 index 0000000000..f353012474 --- /dev/null +++ b/tools/zig/flashnext_batch_prompts.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Write public, deterministic token fixtures for tf-flashnext-batch using a local checkpoint tokenizer.""" +import argparse +import json +from pathlib import Path + +from transformers import AutoTokenizer + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("model", type=Path) + parser.add_argument("output", type=Path) + args = parser.parse_args() + tokenizer = AutoTokenizer.from_pretrained(args.model, local_files_only=True) + args.output.mkdir(parents=True, exist_ok=True) + cases = [ + ("short", "List the first twenty prime numbers, with no introduction.", False), + ("thinking", "Explain why a square root of two cannot be a rational number.", True), + ("sparse", "\n".join(f"Item {i}: value {i % 97}." for i in range(240)) + + "\nSummarize the list and explain how the values repeat.", False), + ("sparse-thinking", "\n".join(f"Row {i}: the box contains {i % 31} blue counters." for i in range(720)) + + "\nDescribe an algorithm to calculate the total number of blue counters.", True), + ] + for name, text, thinking in cases: + tokens = tokenizer.apply_chat_template( + [{"role": "user", "content": text}], tokenize=True, + add_generation_prompt=True, enable_thinking=thinking, return_dict=False, + ) + path = args.output / f"{name}.json" + path.write_text(json.dumps({"prompt": tokens, "text": text, "thinking": thinking}) + "\n") + print(f"{path.name}: {len(tokens)} prompt tokens, thinking={thinking}") + + +if __name__ == "__main__": + main() diff --git a/tools/zig/flashnext_batch_serve.py b/tools/zig/flashnext_batch_serve.py new file mode 100644 index 0000000000..fffaec092c --- /dev/null +++ b/tools/zig/flashnext_batch_serve.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Compare serial and shared native HTTP replies on identical public fixtures, with one server at a time.""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import json +from pathlib import Path +import subprocess +import threading +import time +import urllib.request + + +BASE = "http://127.0.0.1:52417" + + +def get(path): + with urllib.request.urlopen(BASE + path, timeout=3) as response: + return response.read().decode() + + +def request(case, tokens, gate): + payload = { + "model": "batch-gate", "messages": [{"role": "user", "content": case["text"]}], + "max_tokens": tokens, "temperature": 0, "stream": False, + "chat_template_kwargs": {"enable_thinking": case["thinking"]}, + } + gate.wait() + start = time.monotonic() + req = urllib.request.Request(BASE + "/v1/chat/completions", json.dumps(payload).encode(), + {"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=300) as response: + result = json.load(response) + choice = result["choices"][0] + usage = result["usage"] + token_sha = result["tensorfold"].get("token_sha") + if not isinstance(token_sha, str) or len(token_sha) != 12 or any(c not in "0123456789abcdef" for c in token_sha): + raise RuntimeError("missing native token fingerprint") + prompt = usage["prompt_tokens"] + cached = usage["prompt_tokens_details"]["cached_tokens"] + tokens = usage["completion_tokens"] + if type(prompt) is not int or prompt < 1 or type(cached) is not int or cached != 0 or type(tokens) is not int or tokens < 1: + raise RuntimeError("invalid token counts or unexpected prefix reuse") + return { + "reply": {"message": choice["message"], "finish_reason": choice["finish_reason"]}, + "tokens": tokens, "prompt_tokens": prompt, "cached_tokens": cached, "token_sha": token_sha, + "seconds": time.monotonic() - start, + } + + +def run(args, slots, cases): + log_path = args.output / f"serve-{slots}.log" + with log_path.open("w") as log: + process = subprocess.Popen([ + str(args.binary), "serve", str(args.model), "--name", "batch-gate", "--host", "127.0.0.1", + "--port", "52417", "--context", "16384", "--parallel", str(slots), + "--temperature", "0", "--prompt-cache-gib", "0", "--no-update-check", + ], stdout=log, stderr=subprocess.STDOUT) + (args.output / "server.pid").write_text(str(process.pid) + "\n") + try: + deadline = time.monotonic() + 240 + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError(f"native server exited {process.returncode}; inspect {log_path.name}") + try: + health = json.loads(get("/health")) + if health.get("status") == "ok" and not health.get("warming", False): + break + except (OSError, ValueError): + pass + time.sleep(0.2) + else: + raise TimeoutError("native server startup") + batches = [] + for iteration in range(args.rounds): + stop = threading.Event() + peaks = [0, 0] + + def monitor(): + while not stop.is_set(): + try: + for line in get("/v1/metrics").splitlines(): + for index, key in enumerate(("tensorfold:requests_running", "tensorfold:requests_waiting")): + if line.startswith(key + " "): + peaks[index] = max(peaks[index], int(float(line.split()[1]))) + except (OSError, ValueError): + pass + stop.wait(0.05) + + watcher = threading.Thread(target=monitor, daemon=True) + watcher.start() + start = time.monotonic() + try: + gate = threading.Barrier(len(cases)) + with ThreadPoolExecutor(max_workers=len(cases)) as pool: + futures = [pool.submit(request, case, args.tokens, gate) for case in cases] + results = [future.result() for future in futures] + seconds = time.monotonic() - start + finally: + stop.set() + watcher.join(timeout=4) + generated = sum(result["tokens"] for result in results) + batch = {"iteration": iteration, "seconds": seconds, "tokens": generated, + "tokens_per_second": generated / seconds, "peak_running": peaks[0], + "peak_waiting": peaks[1], "results": results} + batches.append(batch) + print(json.dumps({"parallel": slots, **{key: batch[key] for key in + ("iteration", "seconds", "tokens", "tokens_per_second", "peak_running", "peak_waiting")}}), flush=True) + return batches + finally: + process.terminate() + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + (args.output / "server.pid").unlink(missing_ok=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary", type=Path) + parser.add_argument("model", type=Path) + parser.add_argument("fixtures", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--tokens", type=int, default=128) + parser.add_argument("--rounds", type=int, default=3) + parser.add_argument("--parallel", default="1,4") + args = parser.parse_args() + if args.tokens < 1 or args.rounds < 1: + parser.error("tokens and rounds must be positive") + slots = [int(value) for value in args.parallel.split(",")] + if not slots or slots[0] != 1 or any(value < 1 or value > 16 for value in slots): + parser.error("parallel must start with 1 and stay within 1..16") + args.output.mkdir(parents=True, exist_ok=True) + cases = [json.loads((args.fixtures / f"{name}.json").read_text()) for name in + ("short", "thinking", "sparse", "sparse-thinking")] + receipt = {"reply_token_limit": args.tokens, "rounds": args.rounds, "runs": {}} + reference = None + for count in slots: + batches = run(args, count, cases) + receipt["runs"][str(count)] = batches + for batch in batches: + replies = [{key: result[key] for key in ("reply", "tokens", "prompt_tokens", "token_sha")} + for result in batch["results"]] + if reference is None: + reference = replies + if replies != reference: + receipt["exact"] = False + (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + raise RuntimeError("HTTP replies differ from serial reference") + if count > 1 and not any(batch["peak_running"] > 1 for batch in batches): + raise RuntimeError("no simultaneous HTTP sessions observed") + (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + receipt["exact"] = True + (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/zig/flashnext_host.zig b/zig/flashnext_host.zig index 06782189f6..2091e630e6 100644 --- a/zig/flashnext_host.zig +++ b/zig/flashnext_host.zig @@ -2,4 +2,5 @@ test { _ = @import("src/families/flashnext/host_test.zig"); _ = @import("src/families/flashnext/pack.zig"); + _ = @import("src/families/flashnext/batch_meta_test.zig"); } diff --git a/zig/src/families/flashnext/backend.zig b/zig/src/families/flashnext/backend.zig new file mode 100644 index 0000000000..6cfc6d5265 --- /dev/null +++ b/zig/src/families/flashnext/backend.zig @@ -0,0 +1,324 @@ +//! Flash Next's isolated sessions behind the shared lane scheduler; target rounds batch, MTP drafts are serial. +const std = @import("std"); +const lanes = @import("lanes"); +const fz = @import("replay.zig"); +const Engine = @import("engine.zig").Engine; +const Session = @import("session.zig").Session; +const meta = @import("batch_meta.zig"); +const forward = @import("batch_forward.zig"); +const snap = @import("snapshot.zig"); +const be = lanes.backend; +const Stream = lanes.Stream; +const Buf = fz.Buf; +const mtl = @import("metal"); +const segments = @import("../../core/segments.zig"); + +pub const Backend = struct { + gpa: std.mem.Allocator, + e: *Engine, + primary: *fz.Model, + sessions: []Session, + by: std.AutoHashMapUnmanaged(*Stream, usize) = .empty, + hidden: Buf, + window_count: u64 = 0, + shared_count: u64 = 0, + + pub fn init(gpa: std.mem.Allocator, e: *Engine, n: u32, capacity: usize) !Backend { + if (n < 2 or n > fz.MAXR or e.r.tp != null) return error.BadBatchSlots; + const sessions = try gpa.alloc(Session, n); + errdefer gpa.free(sessions); + var made: usize = 0; + errdefer for (sessions[0..made]) |*s| s.deinit(gpa); + for (sessions, 0..) |*s, i| { + s.* = try Session.init(gpa, e, capacity, i == 0); + made += 1; + } + const hidden = try e.r.device.buffer(fz.MAXR * fz.WIDE * 2, fz.opts); + return .{ .gpa = gpa, .e = e, .primary = e.m, .sessions = sessions, .hidden = .{ .b = hidden } }; + } + + pub fn deinit(b: *Backend) void { + b.e.m = b.primary; + b.by.deinit(b.gpa); + for (b.sessions) |*s| s.deinit(b.gpa); + b.gpa.free(b.sessions); + b.hidden.b.deinit(); + } + + pub fn backend(b: *Backend) be.Backend { + return .{ .ptr = b, .vtable = &.{ .prefill = prefill, .first = first, .queue = queue, .read = read, .verify = verify, .keep = keep, .draft = draft, .release = release } }; + } + + pub fn facts(b: *Backend) lanes.Model { + return .{ .exact_width = fz.MAXR, .first_copy_rows = 4, .mtp = true, .speculate = true, .speculate_early = false, .drafts = 8, .streams_exact = true, .hidden_rows = true, .batch_rows = fz.MAXR, .max_streams = @intCast(b.sessions.len), .draft_streams = false }; + } + + fn self(ptr: *anyopaque) *Backend { + return @ptrCast(@alignCast(ptr)); + } + fn session(b: *Backend, s: *Stream) !*Session { + return &b.sessions[b.by.get(s) orelse return error.UnknownStream]; + } + fn settle(b: *Backend) !void { + var needed = false; + for (b.sessions) |*s| if (s.used) { + if (s.round.pending) try s.keep(s.round.rows); + needed = needed or s.m.state != 0 or s.m.state_row != 0; + }; + if (!needed) return; + const cb = b.e.r.queue.commandBuffer(); + const blit = cb.blit(); + for (b.sessions) |*s| if (s.used) s.normalizeGpu(blit); + blit.end(); + cb.commit(); + cb.wait(); + if (cb.failure()) |message| { + std.log.err("session state copy failed: {s}", .{message}); + return error.GpuFailed; + } + for (b.sessions) |*s| if (s.used) { + s.m.state = 0; + s.m.state_row = 0; + }; + } + + const Binding = struct { + model: *fz.Model, + host: @FieldType(Engine, "host"), + cins: [2]Buf, + select: ?fz.Select, + shape: @TypeOf(@as(fz.Select, undefined).q_shape), + passed: @FieldType(Engine, "passed"), + fn restore(old: Binding, b: *Backend) void { + b.e.m = old.model; + b.e.host = old.host; + b.e.cins = old.cins; + b.e.r.sel = old.select; + b.e.r.shapes.put(b.e.r.arena, "Kc_shape", old.shape) catch unreachable; + b.e.passed = old.passed; + } + }; + + fn bind(b: *Backend, s: *Session) !Binding { + const e = b.e; + const old: Binding = .{ .model = e.m, .host = e.host, .cins = e.cins, .select = e.r.sel, .shape = e.r.shapes.get("Kc_shape").?, .passed = e.passed }; + try e.r.shapes.put(e.r.arena, "Kc_shape", s.shape); + e.m = &s.m; + e.host = .{ .rows = s.m.t.rows, .mdims = s.m.t.mdims, .pos8 = s.m.t.pos8, .nk8 = s.m.t.nk8, .kvmeta = s.m.t.kvmeta, .slots = s.m.mtp.slots }; + e.cins[0] = s.m.ple.cin; + e.r.sel = s.select[fz.CATCH - 1]; + e.passed = null; + e.hostMode(); + return old; + } + + fn prefill(ptr: *anyopaque, stream: *Stream) anyerror!void { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const b = self(ptr); + if (stream.sampling) |p| if (p.temperature > 0) return error.GreedyOnly; + const prompt = stream.prompt(); + if (prompt.len == 0) return error.EmptyPrompt; + try b.settle(); + const i = for (b.sessions, 0..) |s, j| { + if (!s.used) break j; + } else return error.NoFreeSlot; + const s = &b.sessions[i]; + if (prompt.len + stream.max_new + fz.MAXR + 1 > s.m.cap) return error.ContextFull; + try b.by.put(b.gpa, stream, i); + s.used = true; + s.held_n = 0; + s.round = .{}; + s.row0 = 0; + outputs(b, s, 0); + const old = try b.bind(s); + defer old.restore(b); + s.m.reset(); + stream.cached = 0; + if (stream.reuse.saved) |saved| { + const state: *snap.State = @ptrCast(@alignCast(saved)); + if (state.at < prompt.len and state.at < s.m.cap) { + try snap.restore(b.e, state); + stream.cached = @intCast(state.at); + } else stream.reuse_failed = true; + } + var at: usize = s.m.pos; + while (at < prompt.len) { + if (stream.isCancelled()) return error.Cancelled; + var end = prompt.len; + for (stream.chunks) |mark| if (mark > at) { + end = @min(end, mark); + break; + }; + for (stream.reuse.marks) |mark| if (mark > at) { + end = @min(end, mark); + break; + }; + const call = if (b.e.segments) segments.next(end - at, b.e.pr.step, @import("engine.zig").SEG_MIN) else segments.Call{ .rows = @min(end - at, b.e.pr.step), .parts = 1 }; + end = at + call.rows; + const ps = [2]*fz.Prompt{ b.e.pr, b.e.pr2 }; + s.first = try fz.Prompt.chunkN(ps[0..call.parts], &s.m, b.gpa, prompt[at..end]); + for (ps[0..call.parts], 0..) |p, k| { + const start = at + segments.start(call.rows, call.parts, k); + const rows = segments.rows(call.rows, call.parts, k); + const count = if (start + rows < prompt.len) rows else rows - 1; + try p.mtpKeys(&s.m, start, prompt[start + 1 .. start + 1 + count], p.last); + if (start + rows == prompt.len) Session.copy(p.last.at((rows - 1) * fz.WIDE * 2), s.last, fz.WIDE * 2); + } + s.normalize(); + at = end; + if (std.mem.indexOfScalar(u32, stream.reuse.marks, @intCast(at)) != null) + if (stream.reuse.hook) |hook| hook.at(hook.ptr, stream, @intCast(at)); + } + s.m.mtp.pos = prompt.len - 1; + s.m.mtp.drafted = 0; + } + + fn first(ptr: *anyopaque, stream: *Stream, position: u64) anyerror!u64 { + const s = try self(ptr).session(stream); + if (position != s.m.pos) return error.PositionMismatch; + return s.first; + } + fn queue(_: *anyopaque, _: *Stream, _: be.Feed, _: u64) anyerror!u64 { + return error.Unsupported; + } + fn read(_: *anyopaque, handle: u64) anyerror!u32 { + return @intCast(handle); + } + + fn outputs(b: *Backend, s: *Session, row: usize) void { + var li: usize = 0; + for (&s.m.layers) |*l| if (l.linear) { + l.cs[1] = .{ .b = b.e.o_cs, .off = (li * fz.MAXR + row) * fz.CS_ROW }; + l.so[1] = .{ .b = b.e.o_so, .off = (li * fz.MAXR + row) * fz.SO_ROW }; + li += 1; + }; + } + + fn verify(ptr: *anyopaque, windows: []const be.Window, out: []be.Verified) anyerror!void { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const b = self(ptr); + try b.settle(); + if (windows.len == 0 or windows.len > fz.MAXR or out.len != windows.len) return error.BadBatch; + var desc: [fz.MAXR]meta.Window = undefined; + var segs: [fz.MAXR]forward.Seg = undefined; + for (windows, out, 0..) |w, o, i| { + const s = try b.session(w.stream); + if (w.parents != null or w.early or w.held > s.held_n) return error.WindowOutOfStep; + if (w.rows() > fz.MAXR or w.rows() > s.m.cap - s.m.pos or + w.positions.len != w.rows() or o.sampled.len != w.rows() or o.drafts.len != w.rows() - 1) + return error.WindowOutOfStep; + for (w.positions, 0..) |p, j| if (p != s.m.pos + j + 1) return error.PositionMismatch; + desc[i] = .{ .slot = @intCast(b.by.get(w.stream).?), .pos = @intCast(s.m.pos), .rows = @intCast(w.rows()) }; + } + const plan = try meta.Plan.init(desc[0..windows.len], fz.CAP); + const t = &b.primary.t; + for (windows, out, 0..) |w, o, i| { + const s = try b.session(w.stream); + const row = plan.starts[i]; + s.row0 = row; + try s.round.begin(s.m.pos, w.rows()); + s.tokens[0] = w.pending; + @memcpy(s.tokens[1..][0..w.held], s.held[0..w.held]); + @memcpy(s.tokens[1 + w.held ..][0..w.tokens.len], w.tokens); + s.m.windowMeta(s.round.rows); + s.m.pleIds(s.tokens[0..s.round.rows]); + @memcpy(t.ids8.b.slice(u32, fz.MAXR)[row..][0..s.round.rows], s.tokens[0..s.round.rows]); + @memcpy(t.ple_ids.b.slice(u32, fz.MAXR * 16)[row * 16 ..][0 .. s.round.rows * 16], s.m.t.ple_ids.b.slice(u32, s.round.rows * 16)); + outputs(b, s, row); + segs[i] = .{ .s = s, .row0 = row, .rows = s.round.rows }; + @memcpy(o.drafts, s.tokens[1..s.round.rows]); + } + t.rows.b.slice(i32, 1)[0] = @intCast(plan.total); + t.mdims.b.slice(i32, 2)[0] = @intCast(plan.total); + @memcpy(t.pos8.b.slice(u32, fz.MAXR)[0..plan.total], plan.positions[0..plan.total]); + const cb = b.e.r.queue.commandBuffer(); + b.e.r.enc = cb.compute(.serial); + try forward.encode(b.e, segs[0..windows.len], plan.total, t.ids8); + try b.primary.finish(cb); + Session.copy(b.primary.last, b.hidden, plan.total * fz.WIDE * 2); + for (out, 0..) |o, i| @memcpy(o.sampled, t.picks.b.slice(u32, plan.total)[plan.starts[i]..][0..o.sampled.len]); + b.window_count += 1; + if (windows.len > 1) b.shared_count += 1; + if (std.c.getenv("FZ_BATCH_LOG") != null) + std.log.info("flashnext batch: streams {d}, rows {d}, shared rounds {d}", .{ windows.len, plan.total, b.shared_count }); + } + + fn keep(ptr: *anyopaque, windows: []const be.Window, paths: []const []const u32) anyerror!void { + const b = self(ptr); + for (windows, paths) |w, path| { + for (path, 0..) |r, j| if (r != j) return error.TreeWindowsUnsupported; + try (try b.session(w.stream)).keep(path.len); + } + } + + fn draft(ptr: *anyopaque, requests: []const be.DraftRequest) anyerror!void { + const b = self(ptr); + for (requests) |r| { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + if (r.early or r.lanes != null or r.ranks or r.depth >= fz.MAXR) return error.DraftOutOfStep; + const s = try b.session(r.stream); + var one: [1]u32 = undefined; + const follow = if (r.first) |f| blk: { + one[0] = switch (f) { + .value => |v| v, + .handle => |h| @intCast(h), + }; + break :blk one[0..]; + } else r.follow; + const hidden = if (r.rows) |path| blk: { + try s.round.draft(r.start, path, follow.len); + break :blk b.hidden.at(s.row0 * fz.WIDE * 2); + } else s.last; + const old = try b.bind(s); + defer old.restore(b); + s.held_n = r.depth; + if (follow.len == 0) return error.DraftOutOfStep; + const token = try s.m.mtpAbsorb(follow, hidden); + if (r.depth > 0) s.held[0] = token; + if (r.depth > 1) for (1..r.depth) |i| { + s.held[i] = try s.m.mtpChain(s.held[i - 1]); + }; + } + } + + fn release(ptr: *anyopaque, stream: *Stream) void { + const b = self(ptr); + const kv = b.by.fetchRemove(stream) orelse return; + b.sessions[kv.value].used = false; + b.sessions[kv.value].round = .{}; + b.sessions[kv.value].held_n = 0; + } + + pub fn snapBytes(ptr: *anyopaque, at: u32) u64 { + return self(ptr).e.snap_pool.size(snap.bytes(at)); + } + pub fn snapCharged(_: *anyopaque, saved: *anyopaque) u64 { + const state: *snap.State = @ptrCast(@alignCast(saved)); + return state.cap; + } + pub fn snapSpare(ptr: *anyopaque) u64 { + return self(ptr).e.snap_pool.spare(); + } + pub fn snapTrim(ptr: *anyopaque, room: u64) void { + self(ptr).e.snap_pool.trim(room); + } + pub fn snapReuses(ptr: *anyopaque, at: u32) bool { + return self(ptr).e.snap_pool.fits(snap.bytes(at)); + } + pub fn snapSave(ptr: *anyopaque, owner: ?*anyopaque, at: u32) anyerror!*anyopaque { + const b = self(ptr); + const s = try b.session(@ptrCast(@alignCast(owner orelse return error.NoStream))); + const old = try b.bind(s); + defer old.restore(b); + return try snap.save(b.e, b.gpa, at); + } + pub fn snapRestore(_: *anyopaque, _: ?*anyopaque, _: *anyopaque) anyerror!void { + return error.BackendRestores; + } + pub fn snapDrop(ptr: *anyopaque, saved: *anyopaque) void { + snap.drop(self(ptr).gpa, @ptrCast(@alignCast(saved))); + } +}; diff --git a/zig/src/families/flashnext/batch_forward.zig b/zig/src/families/flashnext/batch_forward.zig new file mode 100644 index 0000000000..1a4e0fbca8 --- /dev/null +++ b/zig/src/families/flashnext/batch_forward.zig @@ -0,0 +1,122 @@ +//! One target forward over packed session windows, with segmented recurrence, convolution and attention. +const std = @import("std"); +const mtl = @import("metal"); +const fz = @import("replay.zig"); +const Session = @import("session.zig").Session; +const Engine = @import("engine.zig").Engine; +const Buf = fz.Buf; +pub const Seg = struct { s: *Session, row0: usize, rows: usize }; + +pub fn encode(e: *Engine, segs: []const Seg, rows: usize, ids: Buf) !void { + const r = e.r; + const m = e.m; + const t = &m.t; + if (r.tp != null or r.gpu_round) return error.BatchModeUnsupported; + r.rows = rows; + var cur: usize = 0; + try r.call("qa_embed_rows@embed", &.{ ids, m.embed[0], m.embed[1], m.embed[2] }, &.{t.h[0]}); + for (0..fz.LAYERS) |i| { + const l = &m.layers[i]; + if (i > 0) try m.grouped(t.h[cur], t.h[1 - cur]) else try r.call("q4_hc_norm_none#[10240]", &.{t.h[cur]}, &.{ t.h[1 - cur], t.ssp }); + cur = 1 - cur; + if (i == 1) { + const p = &m.ple; + var tabs: [2 + 3 * fz.GROUPS]Buf = undefined; + tabs[0] = t.ple_ids; + tabs[1] = p.starts; + for (p.tables, 0..) |b, j| tabs[2 + j] = b; + try r.call("qa_ple_lookup@ple", &tabs, &.{t.emb}); + try m.lane(t.emb, fz.D, p.kv, "lane_qmm_bytes_grouped@ple.kv", t.kvp); + for (segs) |g| { + const v = view(t.*, g.row0); + r.rows = g.rows; + const pp = &g.s.m.ple; + try r.call("q4_ple_gate@ple", &.{ v.kvp, v.h[cur], pp.ks, pp.qs, pp.cs, t.eps }, &.{ v.gated, pp.cin.at(fz.PLE_TAIL * fz.WIDE * 2) }); + try r.call("q4_ple_conv@ple", &.{ pp.cin, pp.conv, v.gated, v.h[cur] }, &.{v.hout}); + } + r.rows = rows; + try r.call("q4_hc_norm_none#[10240]", &.{t.hout}, &.{ t.h[1 - cur], t.ssp }); + cur = 1 - cur; + } + try m.hcProject(t.h[cur], l.ahc, "qa_hc_down@ahc", "qa_hc_up@ahc", t.inj_a); + try m.lane(t.mixed, fz.D, l.proj, if (l.linear) "lane_qmm_bytes_grouped@gdn.in" else "lane_qmm_bytes_grouped@att.proj", t.p); + r.touch(l.out.wq); + r.touch(l.out.sbt); + if (l.linear) { + for (segs) |g| { + const v = view(t.*, g.row0); + const sl = &g.s.m.layers[i]; + r.rows = g.rows; + const ins = [_]Buf{ v.p, sl.cs[0], sl.so[0], l.conv, l.alog, l.dt, l.norm, t.eps, g.s.m.t.rows }; + const outs = [_]Buf{ v.gout, sl.cs[1], sl.so[1] }; + if (r.gdn_pipe) |pipe| fz.gdn_step.step(r, pipe, &ins, &outs, 0, 48) else try r.callAs("q4_gdn@gdn", if (r.gdn_step and g.rows > 1) 8 else g.rows, &ins, &outs); + } + } else { + try r.call("q4_attn_prep@att", &.{ t.p, t.pos8, l.qn, l.kn, l.iqn, t.eps, t.log2base }, &.{ t.q, t.kout, t.iq }); + for (segs) |g| try attention(e, g, i); + r.rows = rows; + try r.call("q4_attn_merge_gate#[24, 16, 256]", &.{ t.po, t.pm, t.p }, &.{t.aout}); + } + r.rows = rows; + try m.lane(if (l.linear) t.gout else t.aout, 6144, l.out, if (l.linear) "lane_qmm_bytes_grouped@gdn.out" else "lane_qmm_bytes_grouped@att.o", t.branch); + try r.call("q4_hc_norm_plain#[10240]", &.{ t.h[cur], t.inj_a, t.branch }, &.{ t.h[1 - cur], t.ssp }); + cur = 1 - cur; + r.touch(l.router); + try m.hcProject(t.h[cur], l.mhc, "qa_hc_down@mhc", "qa_hc_up@mhc", t.inj_m); + try r.router("q4_router_float@moe", t.mixed, l.router, t.rows, t.lg, rows); + try r.experts("qa_expert_gateup@moe.gate", "qa_expert_down_y@moe.down", t.mixed, t.lg, l.ex, t.act, t.pick, t.wts, t.rows, t.ydown); + } + try m.grouped(t.h[cur], t.h[1 - cur]); + cur = 1 - cur; + m.last = t.h[cur]; + try m.hcProject(t.h[cur], m.mix, "qa_hc_down@mix", "qa_hc_up@mix", t.inj_a); + try m.lane(t.mixed, fz.D, m.head, "lane_qmm_bytes_grouped@head", t.logits); + r.enc.setPipeline(r.argmax_pipe); + for ([_]Buf{ t.logits, t.picks, t.vocab }, 0..) |b, j| r.enc.setBuffer(b.b, b.off, j); + r.enc.dispatchThreads(mtl.Size.of(1024 * rows, 1, 1), mtl.Size.of(1024, 1, 1)); + if (!r.serial) r.enc.barrier(); +} + +fn attention(e: *Engine, g: Seg, i: usize) !void { + const r = e.r; + var t = view(e.m.t, g.row0); + t.p = e.m.t.p.at(g.row0 * 13952 * 2); + const sl = &g.s.m.layers[i]; + const st = &g.s.m.t; + r.rows = g.rows; + r.enc.setPipeline(r.kv_pipe); + for ([_]Buf{ t.kout, t.p, sl.keys, sl.vals, sl.raw, st.kvmeta }, 0..) |b, j| + r.enc.setBuffer(b.b, b.off, j); + r.enc.dispatchThreads(mtl.Size.of(512 * g.rows, 1, 1), mtl.Size.of(256, 1, 1)); + if (!r.serial) r.enc.barrier(); + const old_shape = r.shapes.get("Kc_shape").?; + const old_ids = r.shapes.get("IDS_shape").?; + try r.shapes.put(r.arena, "Kc_shape", g.s.shape); + defer r.shapes.put(r.arena, "Kc_shape", old_shape) catch unreachable; + defer r.shapes.put(r.arena, "IDS_shape", old_ids) catch unreachable; + if (g.s.select[i / 4]) |*sel| { + if (sel.meta(g.s.m.pos, g.rows)) { + try sel.encode(r, sl, t.iq, t.eps, t.log2base, g.s.m.pos, g.rows); + try r.shapes.put(r.arena, "IDS_shape", sel.ids_shape); + try r.call("q4_attn_parts#[24, 256]", &.{ t.q, sl.keys, sl.vals, sel.keys, sel.counts, sel.sparse, t.scale }, &.{ t.po, t.pm }); + return; + } + } + try r.call("q4_attn_parts#[24, 256]", &.{ t.q, sl.keys, sl.vals, t.ids81, st.nk8, t.zero8, t.scale }, &.{ t.po, t.pm }); +} + +fn view(t: fz.Tmp, row: usize) fz.Tmp { + var v = t; + v.h = .{ t.h[0].at(row * fz.WIDE * 2), t.h[1].at(row * fz.WIDE * 2) }; + v.kvp = t.kvp.at(row * 12800 * 2); + v.gated = t.gated.at(row * fz.WIDE * 2); + v.hout = t.hout.at(row * fz.WIDE * 2); + v.p = t.p.at(row * 16480 * 2); + v.gout = t.gout.at(row * 6144 * 2); + v.q = t.q.at(row * 24 * 256 * 2); + v.kout = t.kout.at(row * 2 * 256 * 2); + v.iq = t.iq.at(row * 4 * 128 * 2); + v.po = t.po.at(row * 24 * 16 * 256 * 4); + v.pm = t.pm.at(row * 24 * 16 * 2 * 4); + return v; +} diff --git a/zig/src/families/flashnext/batch_meta.zig b/zig/src/families/flashnext/batch_meta.zig new file mode 100644 index 0000000000..78feb2c428 --- /dev/null +++ b/zig/src/families/flashnext/batch_meta.zig @@ -0,0 +1,59 @@ +//! CPU validation and immutable row positions for a shared Flash Next forward. +const std = @import("std"); +pub const max_rows = 16; +pub const Window = struct { slot: u32, pos: u32, rows: u32 }; + +/// The last forward remains a draft source after keep; only its unsettled state may be committed once. +pub const Round = struct { + start: u64 = 0, + rows: usize = 0, + pending: bool = false, + + pub fn begin(r: *Round, start: u64, rows: usize) !void { + if (r.pending or rows == 0 or rows > max_rows) return error.WindowOutOfStep; + r.* = .{ .start = start, .rows = rows, .pending = true }; + } + pub fn keep(r: *Round, n: usize) !void { + if (!r.pending or n == 0 or n > r.rows) return error.KeepOutOfStep; + r.pending = false; + } + pub fn draft(r: Round, start: u64, path: []const u32, follows: usize) !void { + if (start != r.start or path.len == 0 or path.len > r.rows or path.len != follows) + return error.DraftOutOfStep; + for (path, 0..) |row, i| if (row != i) return error.TreeDraftsUnsupported; + } +}; + +pub const Plan = struct { + total: usize = 0, + starts: [max_rows]usize = @splat(0), + positions: [max_rows]u32 = @splat(0), + keys: [max_rows]u32 = @splat(0), + + pub fn init(windows: []const Window, capacity: u32) !Plan { + if (windows.len == 0) return error.EmptyBatch; + if (windows.len > max_rows) return error.TooManyRows; + var p: Plan = .{}; + for (windows, 0..) |w, i| { + if (w.rows == 0) return error.EmptyWindow; + if (w.rows > max_rows - p.total) return error.TooManyRows; + if (w.pos > capacity or w.rows > capacity - w.pos) return error.ContextFull; + for (windows[0..i]) |old| if (old.slot == w.slot) return error.DuplicateSlot; + p.starts[i] = p.total; + for (0..w.rows) |r| { + p.positions[p.total + r] = w.pos + @as(u32, @intCast(r)); + p.keys[p.total + r] = p.positions[p.total + r] + 1; + } + p.total += w.rows; + } + return p; + } +}; + +pub fn fit(want: u32, fixed: bool, room: u64, per_slot: u64) !u32 { + if (per_slot == 0) return error.BadSlotBudget; + const extra: u64 = @min(max_rows - 1, room / per_slot); + const n: u32 = @intCast(@min(@as(u64, @min(@max(want, 1), max_rows)), 1 + extra)); + if (fixed and n < want) return error.OverMemoryLimit; + return n; +} diff --git a/zig/src/families/flashnext/batch_meta_test.zig b/zig/src/families/flashnext/batch_meta_test.zig new file mode 100644 index 0000000000..70c03f1715 --- /dev/null +++ b/zig/src/families/flashnext/batch_meta_test.zig @@ -0,0 +1,64 @@ +const std = @import("std"); +const meta = @import("batch_meta.zig"); + +test "packed windows keep unequal session positions and contiguous scratch rows" { + const p = try meta.Plan.init(&.{ + .{ .slot = 3, .pos = 19, .rows = 3 }, + .{ .slot = 0, .pos = 2047, .rows = 2 }, + .{ .slot = 5, .pos = 9000, .rows = 1 }, + }, 10000); + try std.testing.expectEqual(@as(usize, 6), p.total); + try std.testing.expectEqualSlices(u32, &.{ 19, 20, 21, 2047, 2048, 9000 }, p.positions[0..p.total]); + try std.testing.expectEqualSlices(u32, &.{ 20, 21, 22, 2048, 2049, 9001 }, p.keys[0..p.total]); + try std.testing.expectEqualSlices(usize, &.{ 0, 3, 5 }, p.starts[0..3]); +} + +test "invalid batches fail before any session state is changed" { + try std.testing.expectError(error.EmptyBatch, meta.Plan.init(&.{}, 32)); + try std.testing.expectError(error.EmptyWindow, meta.Plan.init(&.{.{ .slot = 0, .pos = 0, .rows = 0 }}, 32)); + try std.testing.expectError(error.DuplicateSlot, meta.Plan.init(&.{ + .{ .slot = 1, .pos = 0, .rows = 1 }, .{ .slot = 1, .pos = 8, .rows = 1 }, + }, 32)); + try std.testing.expectError(error.TooManyRows, meta.Plan.init(&.{ + .{ .slot = 0, .pos = 0, .rows = 9 }, .{ .slot = 1, .pos = 0, .rows = 8 }, + }, 32)); + try std.testing.expectError(error.ContextFull, meta.Plan.init(&.{.{ + .slot = 0, + .pos = std.math.maxInt(u32), + .rows = 1, + }}, 32)); + const p = try meta.Plan.init(&.{.{ .slot = 0, .pos = 31, .rows = 1 }}, 32); + try std.testing.expectEqual(@as(u32, 32), p.keys[0]); +} + +test "slot admission reserves all caches before admitting a fixed request count" { + try std.testing.expectEqual(@as(u32, 3), try meta.fit(8, false, 20, 8)); + try std.testing.expectError(error.OverMemoryLimit, meta.fit(4, true, 20, 8)); + try std.testing.expectEqual(@as(u32, 1), try meta.fit(8, false, 0, 8)); + try std.testing.expectError(error.BadSlotBudget, meta.fit(2, false, 20, 0)); + try std.testing.expectEqual(@as(u32, 16), try meta.fit(16, true, std.math.maxInt(u64), 1)); + try std.testing.expectError(error.OverMemoryLimit, meta.fit(17, true, std.math.maxInt(u64), 1)); +} + +test "shared draft before keep and lone draft after rejection use the same forward provenance" { + var shared: meta.Round = .{}; + var lone: meta.Round = .{}; + try shared.begin(2047, 4); + try lone.begin(9000, 3); + try shared.draft(2047, &.{ 0, 1 }, 2); + try shared.keep(2); + try lone.keep(1); + try lone.draft(9000, &.{0}, 1); + try std.testing.expectError(error.KeepOutOfStep, lone.keep(1)); + try std.testing.expectError(error.DraftOutOfStep, lone.draft(9001, &.{0}, 1)); + try std.testing.expectError(error.TreeDraftsUnsupported, shared.draft(2047, &.{ 0, 2 }, 2)); + try std.testing.expectError(error.DraftOutOfStep, shared.draft(2047, &.{ 0, 1 }, 1)); + try std.testing.expectError(error.DraftOutOfStep, shared.draft(2047, &.{ 0, 1, 2, 3, 4 }, 5)); + try shared.begin(2049, 1); + try shared.keep(1); + try shared.draft(2049, &.{0}, 1); + lone = .{}; + try std.testing.expectError(error.DraftOutOfStep, lone.draft(9000, &.{0}, 1)); + try lone.begin(0, 1); + try std.testing.expectError(error.WindowOutOfStep, lone.begin(1, 1)); +} diff --git a/zig/src/families/flashnext/replay.zig b/zig/src/families/flashnext/replay.zig index 1bdd2f2857..f25f53c64c 100644 --- a/zig/src/families/flashnext/replay.zig +++ b/zig/src/families/flashnext/replay.zig @@ -1854,6 +1854,7 @@ pub const Model = struct { ple: Ple, t: Tmp, pos: usize = 0, + cap: usize = CAP, state: usize = 0, // the DeltaNet buffer pair holding the state state_row: usize = 0, // its row gpu_seconds: f64 = 0, @@ -2190,7 +2191,7 @@ pub const Model = struct { nk8[i] = if (i < rows) @intCast(h.pos + i + 1) else 0; } const kvm = sl.kvmeta.b.slice(u32, 3); - kvm[0], kvm[1], kvm[2] = .{ @intCast(h.pos), CAP, @intCast(rows) }; + kvm[0], kvm[1], kvm[2] = .{ @intCast(h.pos), @intCast(m.cap), @intCast(rows) }; sl.n_add.b.slice(u32, 1)[0] = @intCast(rows * WIDE); } @@ -2897,7 +2898,7 @@ pub const Prompt = struct { b.nk.b.slice(i32, PMAX)[i] = @intCast(pos + i + 1); } const kvm = b.kvmeta.b.slice(u32, 3); - kvm[0], kvm[1], kvm[2] = .{ @intCast(pos), CAP, @intCast(rows) }; + kvm[0], kvm[1], kvm[2] = .{ @intCast(pos), @intCast(m.cap), @intCast(rows) }; b.rows.b.slice(i32, 1)[0] = @intCast(rows); const pp = &m.ple; const seq = try gpa.alloc(i64, 2 + rows); @@ -3012,7 +3013,7 @@ pub const Prompt = struct { try sl.encode(r, L, b.iq, t.eps, t.log2base, s.pos, rows); } p.bind(p.sattn, &.{ b.q, L.keys, L.vals, sl.keys, sl.counts, sl.sparse, b.p, t.scale }); - const cap: i32 = CAP; + const cap: i32 = @intCast(m.cap); r.enc.setBytes(std.mem.asBytes(&cap), 8); r.enc.setBuffer(b.aout.b, b.aout.off, 9); r.enc.dispatchThreads(mtl.Size.of(rows * 128, 2, 1), mtl.Size.of(128, 1, 1)); @@ -3028,7 +3029,7 @@ pub const Prompt = struct { try r.callRows("q4_attn_merge_gate#[24, 16, 256]", rows, &.{ b.po, b.pm, b.p }, &.{b.aout}, null); } else if (p.skip & 4 == 0) { p.bind(p.attn256, &.{ b.q, L.keys, L.vals, b.p }); - const ap = [4]i32{ @intCast(rows), @intCast(s.pos + rows), @intCast(s.pos), CAP }; + const ap = [4]i32{ @intCast(rows), @intCast(s.pos + rows), @intCast(s.pos), @intCast(m.cap) }; r.enc.setBytes(std.mem.asBytes(&ap), 4); r.enc.setBuffer(t.scale.b, t.scale.off, 5); r.enc.setBuffer(b.aout.b, b.aout.off, 6); @@ -3357,7 +3358,7 @@ pub const Prompt = struct { @memcpy(b.mids.b.slice(u32, n), nexts); for (0..n) |i| b.pos.b.slice(i32, PMAX)[i] = @intCast(start + i); const kvm = b.kvmeta.b.slice(u32, 3); - kvm[0], kvm[1], kvm[2] = .{ @intCast(start), CAP, @intCast(n) }; + kvm[0], kvm[1], kvm[2] = .{ @intCast(start), @intCast(m.cap), @intCast(n) }; b.n_add.b.slice(u32, 1)[0] = @intCast(n * WIDE); const cb = r.queue.commandBuffer(); r.enc = cb.compute(if (r.serial) .serial else .concurrent); diff --git a/zig/src/families/flashnext/session.zig b/zig/src/families/flashnext/session.zig new file mode 100644 index 0000000000..2945b07091 --- /dev/null +++ b/zig/src/families/flashnext/session.zig @@ -0,0 +1,139 @@ +//! One admitted stream's caches and metadata beside a shared Flash Next runner and scratch plane. +const std = @import("std"); +const mtl = @import("metal"); +const fz = @import("replay.zig"); +const Engine = @import("engine.zig").Engine; +const Buf = fz.Buf; + +pub const Session = struct { + m: fz.Model, + owned: std.ArrayList(mtl.Buffer) = .empty, + select: [fz.CATCH]?fz.Select = @splat(null), + shape: mtl.Buffer = undefined, + last: Buf = undefined, + held: [fz.MAXR]u32 = @splat(0), + held_n: usize = 0, + first: u32 = 0, + used: bool = false, + tokens: [fz.MAXR]u32 = @splat(0), + row0: usize = 0, + round: @import("batch_meta.zig").Round = .{}, + + pub fn init(gpa: std.mem.Allocator, e: *Engine, capacity: usize, original: bool) !Session { + var s: Session = .{ .m = e.m.* }; + errdefer s.deinit(gpa); + s.m.cap = if (original) fz.CAP else capacity; + s.m.pos = 0; + s.m.state = 0; + s.m.state_row = 0; + s.m.recs = null; + s.m.marks = null; + if (!original) { + for (&s.m.layers) |*l| if (l.linear) { + l.cs[0] = try s.buffer(gpa, e, fz.CS_ROW); + l.so[0] = try s.buffer(gpa, e, fz.SO_ROW); + } else { + try s.attention(gpa, e, l); + }; + try s.attention(gpa, e, &s.m.mtp); + s.m.ple.cin = try s.buffer(gpa, e, (fz.PLE_TAIL + fz.MAXR) * fz.WIDE * 2); + } + s.m.t.rows = try s.words(gpa, e, &.{1}); + s.m.t.mdims = try s.words(gpa, e, &.{ 1, 16, 0, 0, 0, 0, 0, 0 }); + s.m.t.pos8 = try s.buffer(gpa, e, fz.MAXR * 4); + s.m.t.nk8 = try s.buffer(gpa, e, fz.MAXR * 4); + s.m.t.kvmeta = try s.words(gpa, e, &.{ 0, @intCast(s.m.cap), 1 }); + s.m.t.ple_ids = try s.buffer(gpa, e, fz.MAXR * 16 * 4); + s.shape = (try s.words(gpa, e, &.{ 1, 2, @intCast(s.m.cap), 256 })).b; + s.last = try s.buffer(gpa, e, fz.WIDE * 2); + for (&s.select) |*sel| { + sel.* = try s.selector(gpa, e); + } + return s; + } + + fn buffer(s: *Session, gpa: std.mem.Allocator, e: *Engine, size: usize) !Buf { + const b = try e.r.device.buffer(@max(size, 64), fz.opts); + errdefer b.deinit(); + try s.owned.append(gpa, b); + @memset(b.contents()[0..@max(size, 64)], 0); + return .{ .b = b }; + } + + fn words(s: *Session, gpa: std.mem.Allocator, e: *Engine, values: []const i32) !Buf { + const b = try s.buffer(gpa, e, values.len * 4); + @memcpy(b.b.slice(i32, values.len), values); + return b; + } + + fn attention(s: *Session, gpa: std.mem.Allocator, e: *Engine, l: anytype) !void { + l.keys = try s.buffer(gpa, e, 2 * s.m.cap * 256 * 2); + l.vals = try s.buffer(gpa, e, 2 * s.m.cap * 256 * 2); + l.raw = try s.buffer(gpa, e, s.m.cap * 128 * 2); + l.pooled = try s.buffer(gpa, e, (s.m.cap + 3) / 4 * 128 * 2); + l.pooled_n = 0; + } + + fn selector(s: *Session, gpa: std.mem.Allocator, e: *Engine) !?fz.Select { + var sel = e.r.sel orelse return null; + sel.start = try s.buffer(gpa, e, 16); + sel.starts = try s.buffer(gpa, e, fz.CATCH * 256); + sel.sc = try s.buffer(gpa, e, fz.MAXR * ((s.m.cap + 3) / 4) * 4); + sel.keys = try s.buffer(gpa, e, fz.MAXR * fz.KW * 4); + sel.complete = try s.buffer(gpa, e, fz.MAXR * 4); + sel.ends = try s.buffer(gpa, e, fz.MAXR * 4); + sel.counts = try s.buffer(gpa, e, fz.MAXR * 4); + sel.sparse = try s.buffer(gpa, e, fz.MAXR * 4); + sel.pooled_shape = (try s.buffer(gpa, e, 16)).b; + sel.q_shape = (try s.buffer(gpa, e, 16)).b; + sel.sc_shape = (try s.buffer(gpa, e, 16)).b; + sel.ids_shape = (try s.buffer(gpa, e, 16)).b; + return sel; + } + + pub fn deinit(s: *Session, gpa: std.mem.Allocator) void { + for (s.owned.items) |b| b.deinit(); + s.owned.deinit(gpa); + } + + pub fn normalize(s: *Session) void { + const m = &s.m; + if (m.state == 0 and m.state_row == 0) return; + for (&m.layers) |*l| if (l.linear) { + copy(l.cs[m.state].at(m.state_row * fz.CS_ROW), l.cs[0], fz.CS_ROW); + copy(l.so[m.state].at(m.state_row * fz.SO_ROW), l.so[0], fz.SO_ROW); + }; + m.state = 0; + m.state_row = 0; + } + + pub fn keep(s: *Session, n: usize) !void { + try s.round.keep(n); + s.m.keepRows(s.tokens[0..s.round.rows], n); + } + + pub fn normalizeGpu(s: *Session, blit: mtl.BlitEncoder) void { + const m = &s.m; + if (m.state == 0 and m.state_row == 0) return; + for (&m.layers) |*l| if (l.linear) { + const cs = l.cs[m.state].at(m.state_row * fz.CS_ROW); + const so = l.so[m.state].at(m.state_row * fz.SO_ROW); + blit.copy(cs.b, cs.off, l.cs[0].b, l.cs[0].off, fz.CS_ROW); + blit.copy(so.b, so.off, l.so[0].b, l.so[0].off, fz.SO_ROW); + }; + } + + pub fn copy(src: Buf, dst: Buf, n: usize) void { + std.mem.copyForwards(u8, dst.b.contents()[dst.off..][0..n], src.b.contents()[src.off..][0..n]); + } + + pub fn bytes(capacity: usize) u64 { + return 36 * (fz.CS_ROW + fz.SO_ROW) + 13 * (capacity * 2304 + (capacity + 3) / 4 * 256) + + overhead(capacity); + } + + /// Metadata and selectors are owned even when the original engine supplies this slot's caches. + pub fn overhead(capacity: usize) u64 { + return fz.CATCH * (fz.MAXR * ((capacity + 3) / 4) * 4 + fz.MAXR * fz.KW * 4 + 4096) + (2 << 20); + } +}; diff --git a/zig/src/families/flashnext/snapshot.zig b/zig/src/families/flashnext/snapshot.zig index 4394096aaa..b352585f79 100644 --- a/zig/src/families/flashnext/snapshot.zig +++ b/zig/src/families/flashnext/snapshot.zig @@ -5,7 +5,6 @@ const fz = @import("replay.zig"); const Engine = @import("engine.zig").Engine; const LAYERS = fz.LAYERS; -const CAP = fz.CAP; const CS_ROW = fz.CS_ROW; const SO_ROW = fz.SO_ROW; const KEY_ROW = 256 * 2; // a key or value row of one head (bf16) @@ -149,17 +148,17 @@ fn parts(m: *fz.Model, at: usize, passed: ?Engine.Passed, out: *[MAX_PARTS]Part) li += 1; }; for (&m.layers) |*L| if (!L.linear) { - n += rows(L.keys, L.vals, L.raw, at, out[n..]); + n += rows(L.keys, L.vals, L.raw, m.cap, at, out[n..]); }; - n += rows(m.mtp.keys, m.mtp.vals, m.mtp.raw, at, out[n..]); + n += rows(m.mtp.keys, m.mtp.vals, m.mtp.raw, m.cap, at, out[n..]); out[n] = .{ .live = if (passed) |ps| ps.tail else .{ .b = m.ple.cin.b, .off = m.ple.cin.off }, .len = TAIL }; return n + 1; } -fn rows(keys: fz.Buf, vals: fz.Buf, raw: fz.Buf, at: usize, out: []Part) usize { +fn rows(keys: fz.Buf, vals: fz.Buf, raw: fz.Buf, cap: usize, at: usize, out: []Part) usize { for (0..2) |hd| { - out[hd] = .{ .live = .{ .b = keys.b, .off = keys.off + hd * CAP * KEY_ROW }, .len = at * KEY_ROW }; - out[2 + hd] = .{ .live = .{ .b = vals.b, .off = vals.off + hd * CAP * KEY_ROW }, .len = at * KEY_ROW }; + out[hd] = .{ .live = .{ .b = keys.b, .off = keys.off + hd * cap * KEY_ROW }, .len = at * KEY_ROW }; + out[2 + hd] = .{ .live = .{ .b = vals.b, .off = vals.off + hd * cap * KEY_ROW }, .len = at * KEY_ROW }; } out[4] = .{ .live = raw, .len = at * RAW_ROW }; return 5; diff --git a/zig/src/native/flashnext_batch_host.zig b/zig/src/native/flashnext_batch_host.zig new file mode 100644 index 0000000000..4956950c60 --- /dev/null +++ b/zig/src/native/flashnext_batch_host.zig @@ -0,0 +1,90 @@ +//! Opt-in single-Mac Flash Next serving through the lane core, sharing each target forward. +const std = @import("std"); +const mtl = @import("metal"); +const api = @import("engine_api"); +const tf = @import("tensorfold"); +const lanes = tf.lanes; +const Backend = tf.flashnext_backend.Backend; +const Session = tf.flashnext_session.Session; +const fx = tf.flashnext_engine; +const fit = @import("cache_fit.zig"); + +const Host = struct { + gpa: std.mem.Allocator, + eng: *fx.Engine, + back: Backend, + cfg: lanes.Config, + clock: lanes.backend.WallClock, + core: lanes.Engine, + host: api.LaneHost, + cache: ?api.prompt_cache.Store = null, + warm: mtl.keepalive.Target, +}; + +pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const u8, window: i64, streams: u32, cache_gib: ?f64, over: bool, a: std.mem.Allocator, why: *[]const u8) !api.Opened { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const capacity: usize = @intCast(@min(tf.flashnext_replay.CAP, window + fx.MARGIN)); + const eng = try fx.Engine.load(gpa, io, dir, dump); + errdefer eng.deinit(); + if (eng.r.tp != null) return error.BatchSpeedUpUnsupported; + try eng.warm(); + eng.hostMode(); + const ready = fit.footprint() orelse return error.NoMemoryReading; + const total = fit.ram() orelse return error.NoMemoryReading; + const reserved = if (!over) (if (cache_gib) |g| (if (g > 0) std.math.lossyCast(u64, g * fit.GiB) else 0) else 0) else 0; + const fixed = Session.overhead(tf.flashnext_replay.CAP) + tf.flashnext_replay.MAXR * tf.flashnext_replay.WIDE * 2; + const available = total / 100 * fit.SHARE_PERCENT -| ready -| fit.MARGIN -| reserved; + if (available < fixed) return error.OverMemoryLimit; + const room = available - fixed; + const slots = try tf.flashnext_batch_meta.fit(streams, true, room, Session.bytes(capacity)); + const h = try gpa.create(Host); + errdefer gpa.destroy(h); + h.gpa = gpa; + h.eng = eng; + h.back = try Backend.init(gpa, eng, slots, capacity); + errdefer h.back.deinit(); + const allocated = fit.footprint() orelse return error.NoMemoryReading; + if (allocated +| fit.MARGIN +| reserved > total / 100 * fit.SHARE_PERCENT) + return error.OverMemoryLimit; + h.cfg = try lanes.Config.init(gpa, h.back.facts(), tf.flashnext_replay.MAXR, tf.flashnext_replay.MAXR - 1); + errdefer h.cfg.deinit(gpa); + h.clock = .{ .io = io }; + h.core = lanes.Engine.init(gpa, &h.cfg, h.back.backend(), h.clock.clock()); + errdefer h.core.deinit(); + const plan = try fit.budget(cache_gib, over, 0, a, why, ""); + eng.snap_pool.max = plan.budget_bytes; + h.host = api.LaneHost.init(gpa, io, &h.core, .{ .name = "flashnext-zig", .lanes = slots, .greedy_only = true, .context_window = @intCast(window), .prefill_step = @intCast(eng.pr.step), .prompt_cache_plan = plan }); + h.cache = null; + if (plan.budget_bytes > 0) { + h.cache = api.prompt_cache.Store.init(gpa, .{ .ptr = &h.back, .vtable = &.{ .bytes = Backend.snapBytes, .save = Backend.snapSave, .restore = Backend.snapRestore, .drop = Backend.snapDrop, .charged = Backend.snapCharged, .spare = Backend.snapSpare, .trim = Backend.snapTrim, .reuses = Backend.snapReuses } }, .{ .lookahead = 1, .planned = true }, plan.budget_bytes); + h.host.cache = &h.cache.?; + h.host.info_.prompt_cache = true; + } + errdefer if (h.cache) |*store| store.deinit(); + h.host.explain = .{ .text = words }; + h.warm = .{ .queue = eng.r.queue }; + h.host.keepalive_target = .{ .ctx = &h.warm, .tick = mtl.keepalive.Target.tick }; + try h.host.start(); + std.log.info("Flash Next: {d} session slots, shared target forwards, serial MTP drafts", .{slots}); + return .{ .engine = h.host.engine(), .close = close, .ctx = h }; +} + +fn words(_: ?*anyopaque, err: anyerror) ?[]const u8 { + return switch (err) { + error.GreedyOnly => "the native Flash Next engine decodes greedily only: send temperature 0", + error.ContextFull => "the prompt and max_tokens do not fit this server's --context", + else => null, + }; +} + +fn close(ctx: *anyopaque) void { + const h: *Host = @ptrCast(@alignCast(ctx)); + h.host.stop(); + if (h.cache) |*store| store.deinit(); + h.core.deinit(); + h.cfg.deinit(h.gpa); + h.back.deinit(); + h.eng.deinit(); + h.gpa.destroy(h); +} diff --git a/zig/src/native/metal.zig b/zig/src/native/metal.zig index 5d579230c7..a419977be5 100644 --- a/zig/src/native/metal.zig +++ b/zig/src/native/metal.zig @@ -300,6 +300,17 @@ fn openFlashNext(a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: problem.* = try std.fmt.allocPrint(a, "--context {d} exceeds this model's {d}-token window", .{ window, native }); return null; } + if (o.lanes_fixed and o.lanes > 1) { + if (o.speed_up != null or std.c.getenv("TF_FLASHNEXT_TP") != null) { + problem.* = "Flash Next shared sessions are single-Mac only; use --parallel 1 with --speed-up"; + return null; + } + var why: []const u8 = ""; + return @import("flashnext_batch_host.zig").open(gpa, io, o.dir, dump, if (window > 0) @min(window, tf.flashnext_replay.CAP - tf.flashnext_engine.MARGIN) else tf.flashnext_replay.CAP - tf.flashnext_engine.MARGIN, o.lanes, o.prompt_cache_gib, o.prompt_cache_over_cap, a, &why) catch |err| { + problem.* = if (err == error.CacheOverCap) why else try std.fmt.allocPrint(a, "the native Flash Next batching engine cannot open ({s})", .{@errorName(err)}); + return null; + }; + } const pool = mtl.objc.Pool.push(); defer pool.pop(); var why: []const u8 = ""; diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 8bff5a4ddb..bdb31c39d7 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -17,6 +17,10 @@ pub const nemotron = @import("families/nemotron/nemotron.zig"); pub const flashnext_replay = @import("families/flashnext/replay.zig"); pub const flashnext_engine = @import("families/flashnext/engine.zig"); pub const flashnext_snapshot = @import("families/flashnext/snapshot.zig"); +pub const flashnext_session = @import("families/flashnext/session.zig"); +pub const flashnext_batch_forward = @import("families/flashnext/batch_forward.zig"); +pub const flashnext_backend = @import("families/flashnext/backend.zig"); +pub const flashnext_batch_meta = @import("families/flashnext/batch_meta.zig"); pub const glm = @import("families/glm/glm.zig"); pub const qwen35 = @import("families/qwen3_5/qwen3_5.zig"); pub const flashnext_pack = @import("families/flashnext/pack.zig"); diff --git a/zig/tests/flashnext_batch.zig b/zig/tests/flashnext_batch.zig new file mode 100644 index 0000000000..8f04569f10 --- /dev/null +++ b/zig/tests/flashnext_batch.zig @@ -0,0 +1,168 @@ +//! Checkpoint gate: shared target rounds against fresh legacy greedy replies, including admission and cancellation. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const lanes = tf.lanes; +const fx = tf.flashnext_engine; +const Backend = tf.flashnext_backend.Backend; + +const Mark = struct { + back: *Backend, + saved: ?*anyopaque = null, + err: ?anyerror = null, + fn at(ptr: *anyopaque, stream: *lanes.Stream, pos: u32) void { + const m: *Mark = @ptrCast(@alignCast(ptr)); + m.saved = Backend.snapSave(m.back, stream, pos) catch |err| { + m.err = err; + return; + }; + } +}; + +const Reply = struct { + a: std.mem.Allocator, + tokens: std.ArrayList(u32) = .empty, + fn prefilled(_: *anyopaque) void {} + fn emitted(ptr: *anyopaque, tokens: []const u32) bool { + const r: *Reply = @ptrCast(@alignCast(ptr)); + r.tokens.appendSlice(r.a, tokens) catch @panic("reply allocation failed"); + return false; + } + fn cancelled(_: *anyopaque) bool { + return false; + } + fn marked(_: *anyopaque, _: usize) void {} + fn out(r: *Reply) fx.Out { + return .{ .ctx = r, .prefilled = prefilled, .tokens = emitted, .cancelled = cancelled, .marked = marked }; + } +}; + +fn prompt(a: std.mem.Allocator, path: []const u8) ![]u32 { + const f = try mtl.MappedFile.open(try std.fmt.allocPrintSentinel(a, "{s}", .{path}, 0)); + defer f.deinit(); + const json = try std.json.parseFromSliceLeaky(std.json.Value, a, f.bytes[0..f.size], .{}); + const items = json.object.get("prompt").?.array.items; + const ids = try a.alloc(u32, items.len); + for (items, ids) |item, *id| id.* = @intCast(item.integer); + return ids; +} + +pub fn main(init: std.process.Init) !void { + const a = init.arena.allocator(); + const args = try init.minimal.args.toSlice(a); + if (args.len < 4) return error.ExpectedModelAndTwoPromptFiles; + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const n: u32 = if (std.c.getenv("FZ_N")) |v| try std.fmt.parseInt(u32, std.mem.span(v), 10) else 32; + if (n < 8) return error.NeedEightReplyTokens; + const count = args.len - 2; + if (count > 8) return error.TooManyPrompts; + const prompts = try a.alloc([]u32, count); + const refs = try a.alloc(Reply, count); + const eng = try fx.Engine.load(init.gpa, init.io, args[1], null); + defer eng.deinit(); + try eng.warm(); + eng.copy = false; + var capacity: usize = 1024; + var baseline_seconds: f64 = 0; + var baseline_mtp_seconds: f64 = 0; + for (args[2..], prompts, refs) |path, *ids, *ref| { + ids.* = try prompt(a, path); + capacity = @max(capacity, ids.len + n + fx.MARGIN); + ref.* = .{ .a = a }; + const start = mtl.clock.seconds(); + _ = try eng.generateFrom(ids.*, 0, &.{}, n, &.{}, 0, ref.out()); + baseline_seconds += mtl.clock.seconds() - start; + if (ref.tokens.items.len != n) return error.ShortBaseline; + var drafted: Reply = .{ .a = a }; + const mtp_start = mtl.clock.seconds(); + _ = try eng.generateFrom(ids.*, 0, &.{}, n, &.{}, null, drafted.out()); + baseline_mtp_seconds += mtl.clock.seconds() - mtp_start; + if (!std.mem.eql(u32, drafted.tokens.items, ref.tokens.items)) return error.LegacyMtpTokenMismatch; + } + eng.hostMode(); + var longest: usize = 0; + for (prompts, 0..) |ids, i| if (ids.len > prompts[longest].len) { + longest = i; + }; + var back = try Backend.init(init.gpa, eng, @intCast(@max(4, count)), capacity); + defer back.deinit(); + for ([_]bool{ false, true }) |drafts| { + var cfg = try lanes.Config.init(init.gpa, back.facts(), tf.flashnext_replay.MAXR, 15); + defer cfg.deinit(init.gpa); + var clock: lanes.backend.WallClock = .{ .io = init.io }; + var core = lanes.Engine.init(init.gpa, &cfg, back.backend(), clock.clock()); + defer core.deinit(); + const streams = try a.alloc(lanes.Stream, count); + for (streams, prompts, 0..) |*s, ids, i| s.* = try lanes.Stream.init(init.gpa, .{ + .id = try std.fmt.allocPrint(a, "stream-{d}", .{i}), + .prompt = ids, + .max_new = n, + .drafts = drafts, + }); + defer for (streams) |*s| s.deinit(init.gpa); + const start = mtl.clock.seconds(); + const before = back.shared_count; + for (streams) |*s| try core.addStream(s); + while (core.activeCount() > 0) try core.step(); + const seconds = mtl.clock.seconds() - start; + for (streams, refs) |s, ref| { + if (!std.mem.eql(u32, s.emitted(), ref.tokens.items)) { + std.debug.print("FAIL {s} drafts {any}: got {any}, expected {any}\n", .{ s.id, drafts, s.emitted(), ref.tokens.items }); + return error.TokenMismatch; + } + } + if (back.shared_count == before) return error.NoSharedForward; + const baseline = if (drafts) baseline_mtp_seconds else baseline_seconds; + std.debug.print("PASS drafts {any}: {d} streams, {d} exact tokens, {d} shared forwards, {d:.3}s, legacy serial {d:.3}s\n", .{ drafts, count, count * n, back.shared_count - before, seconds, baseline }); + // Admit a third prompt after a shared round, cancel its peer, and reuse the released slot. + var first = try lanes.Stream.init(init.gpa, .{ .id = "survivor", .prompt = prompts[0], .max_new = n, .drafts = drafts }); + defer first.deinit(init.gpa); + var cancelled = try lanes.Stream.init(init.gpa, .{ .id = "cancelled", .prompt = prompts[1], .max_new = n, .drafts = drafts }); + defer cancelled.deinit(init.gpa); + var admitted = try lanes.Stream.init(init.gpa, .{ .id = "admitted", .prompt = prompts[1], .max_new = n, .drafts = drafts }); + defer admitted.deinit(init.gpa); + try core.addStream(&first); + try core.addStream(&cancelled); + try core.step(); + core.discard(&cancelled); + try core.addStream(&admitted); + while (core.activeCount() > 0) try core.step(); + if (!std.mem.eql(u32, first.emitted(), refs[0].tokens.items) or + !std.mem.eql(u32, admitted.emitted(), refs[1].tokens.items)) return error.AdmissionTokenMismatch; + std.debug.print("PASS drafts {any}: cancellation, slot reuse and prompt admission\n", .{drafts}); + if (prompts[longest].len <= eng.pr.step) return error.NeedMultiChunkFixture; + const marks = [_]u32{@intCast(eng.pr.step / 2)}; + var mark: Mark = .{ .back = &back }; + var primer = try lanes.Stream.init(init.gpa, .{ + .id = "cache-primer", + .prompt = prompts[longest], + .max_new = 1, + .drafts = drafts, + .reuse = .{ .marks = &marks, .hook = .{ .ptr = &mark, .at = Mark.at } }, + }); + defer primer.deinit(init.gpa); + try core.addStream(&primer); + if (mark.err) |err| return err; + const saved = mark.saved orelse return error.NoSnapshot; + defer Backend.snapDrop(&back, saved); + var peer = try lanes.Stream.init(init.gpa, .{ .id = "cache-peer", .prompt = prompts[0], .max_new = n, .drafts = drafts }); + defer peer.deinit(init.gpa); + var resumed = try lanes.Stream.init(init.gpa, .{ + .id = "cache-resumed", + .prompt = prompts[longest], + .max_new = n, + .drafts = drafts, + .reuse = .{ .saved = saved, .at = marks[0] }, + }); + defer resumed.deinit(init.gpa); + try core.addStream(&peer); + try core.step(); + try core.addStream(&resumed); + if (resumed.cached != marks[0]) return error.PrefixWasNotRestored; + while (core.activeCount() > 0) try core.step(); + if (!std.mem.eql(u32, peer.emitted(), refs[0].tokens.items) or + !std.mem.eql(u32, resumed.emitted(), refs[longest].tokens.items)) return error.CacheTokenMismatch; + std.debug.print("PASS drafts {any}: multi-chunk mark and restored prefix in a dynamic-capacity slot\n", .{drafts}); + } +} From fd272efdb2ecdfcc00940ff93b2ee6807268274b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:11:38 +0200 Subject: [PATCH 02/31] Preserve alternating benchmark receipts on the current upstream base --- tools/zig/flashnext_batch_serve.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tools/zig/flashnext_batch_serve.py b/tools/zig/flashnext_batch_serve.py index fffaec092c..672a211a4b 100644 --- a/tools/zig/flashnext_batch_serve.py +++ b/tools/zig/flashnext_batch_serve.py @@ -47,8 +47,8 @@ def request(case, tokens, gate): } -def run(args, slots, cases): - log_path = args.output / f"serve-{slots}.log" +def run(args, slots, cases, block): + log_path = args.output / f"serve-{slots}-block-{block}.log" with log_path.open("w") as log: process = subprocess.Popen([ str(args.binary), "serve", str(args.model), "--name", "batch-gate", "--host", "127.0.0.1", @@ -132,13 +132,16 @@ def main(): if not slots or slots[0] != 1 or any(value < 1 or value > 16 for value in slots): parser.error("parallel must start with 1 and stay within 1..16") args.output.mkdir(parents=True, exist_ok=True) + receipt = {"reply_token_limit": args.tokens, "rounds": args.rounds, "exact": False, "runs": {}} + (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") cases = [json.loads((args.fixtures / f"{name}.json").read_text()) for name in ("short", "thinking", "sparse", "sparse-thinking")] - receipt = {"reply_token_limit": args.tokens, "rounds": args.rounds, "runs": {}} reference = None - for count in slots: - batches = run(args, count, cases) - receipt["runs"][str(count)] = batches + for block, count in enumerate(slots): + batches = run(args, count, cases, block) + for batch in batches: + batch["block"] = block + receipt["runs"].setdefault(str(count), []).extend(batches) for batch in batches: replies = [{key: result[key] for key in ("reply", "tokens", "prompt_tokens", "token_sha")} for result in batch["results"]] From 0c8d4d56468b79dc594a6d39a659e6f3a03b4ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:24:11 +0200 Subject: [PATCH 03/31] Apply Zig 0.17 formatting required by current CI --- zig/src/core/tokenizer/tokenizer.zig | 18 +++++++++++++----- zig/src/native/cuda.zig | 5 ++--- zig/tests/moe_bench.zig | 1 - 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/zig/src/core/tokenizer/tokenizer.zig b/zig/src/core/tokenizer/tokenizer.zig index 6d276e2857..bcb2f5ee8f 100644 --- a/zig/src/core/tokenizer/tokenizer.zig +++ b/zig/src/core/tokenizer/tokenizer.zig @@ -395,8 +395,11 @@ test "without a decoder, decode joins known ids' tokens with spaces" { test "raw ByteLevel pieces preserve an incomplete UTF-8 character" { const a = std.testing.allocator; var t = Tokenizer{ - .allocator = a, .vocab = .init(a), .id_to_token = .{ .allocator = a }, - .model = .empty(.bpe), .decoder = .byte_level, + .allocator = a, + .vocab = .init(a), + .id_to_token = .{ .allocator = a }, + .model = .empty(.bpe), + .decoder = .byte_level, }; defer t.deinit(); try t.id_to_token.put(0, "\u{c3}"); @@ -421,8 +424,11 @@ test "raw token bytes refuse replacements after the decoder joins token boundari .byte_level, .{ .replace = .{ .pattern = .{ .literal = "ab" }, .content = "x" } }, }; var t = Tokenizer{ - .allocator = a, .vocab = .init(a), .id_to_token = .{ .allocator = a }, - .model = .empty(.bpe), .decoder = .{ .sequence = &decoders }, + .allocator = a, + .vocab = .init(a), + .id_to_token = .{ .allocator = a }, + .model = .empty(.bpe), + .decoder = .{ .sequence = &decoders }, }; defer t.deinit(); try t.id_to_token.put(0, "a"); @@ -449,7 +455,9 @@ test "raw token bytes conservatively refuse context-sensitive decoder kinds" { .{ .sequence = &after_fuse }, }; var t = Tokenizer{ - .allocator = a, .vocab = .init(a), .id_to_token = .{ .allocator = a }, + .allocator = a, + .vocab = .init(a), + .id_to_token = .{ .allocator = a }, .model = .empty(.bpe), }; defer t.deinit(); diff --git a/zig/src/native/cuda.zig b/zig/src/native/cuda.zig index 4f7025162e..a1b9b93ebc 100644 --- a/zig/src/native/cuda.zig +++ b/zig/src/native/cuda.zig @@ -376,9 +376,8 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O h.* = .{ .gpa = gpa, .gpu = g, .family = loaded.ctx, .release = loaded.deinit, .inner = loaded.backend, .vtable = undefined, .cfg = undefined, .clock = undefined, .core = undefined, .host = undefined, .lone = loaded.lone }; const cache = budget.cacheBytes(o.prompt_cache_gib, room, loaded.stream_bytes * (streams -| free)); h.startup = try std.fmt.allocPrint(gpa, "CUDA sm_{d} device {d} ({s}{s}): model {d:.2} GiB; {d} stream{s} at once, {d:.2} GiB each at a {d}-token window, of {d:.1} GiB left after a {d:.1} GiB reserve; prompts in {d}-row chunks{s}{s}; {s}", .{ - capability, device, name, if (after.unified) ", memory shared with the host" else "", toGib(model), streams, if (streams == 1) "" else "s", toGib(loaded.stream_bytes), window, toGib(room), toGib(after.reserve), F.prompt_rows, if (segments > 1) try std.fmt.allocPrint(a, ", {d} staggered segments a call", .{segments}) else "", - if (cache > 0) try std.fmt.allocPrint(a, ", a {d:.1} GiB prompt cache", .{toGib(cache)}) else ", no prompt cache", - if (kernels) |dir| try std.fmt.allocPrint(a, "glue kernels captured at {s}", .{dir}) else "own glue kernels", + capability, device, name, if (after.unified) ", memory shared with the host" else "", toGib(model), streams, if (streams == 1) "" else "s", toGib(loaded.stream_bytes), window, toGib(room), toGib(after.reserve), F.prompt_rows, if (segments > 1) try std.fmt.allocPrint(a, ", {d} staggered segments a call", .{segments}) else "", + if (cache > 0) try std.fmt.allocPrint(a, ", a {d:.1} GiB prompt cache", .{toGib(cache)}) else ", no prompt cache", if (kernels) |dir| try std.fmt.allocPrint(a, "glue kernels captured at {s}", .{dir}) else "own glue kernels", }); errdefer gpa.free(h.startup); if (cache > 0) { diff --git a/zig/tests/moe_bench.zig b/zig/tests/moe_bench.zig index de021c7e21..6e381ff8d5 100644 --- a/zig/tests/moe_bench.zig +++ b/zig/tests/moe_bench.zig @@ -209,4 +209,3 @@ pub fn main(init: std.process.Init) !void { std.debug.print("{s:<44} {s:<26} {d:8.3} ms {d:6.1} TFLOPS differ from first {d}\n", .{ k.path, k.entry, ms[trials / 2], flops / (ms[trials / 2] * 1e-3) / 1e12, differ }); } } - From 6dbf17059a846f2c9c40a92c79b5558d41f2960f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:24:11 +0200 Subject: [PATCH 04/31] Flash Next: chain batch MTP drafts in one Metal command buffer --- tools/zig/flashnext_mtp_chain.py | 71 ++++++++++++++++++++++++++ zig/src/families/flashnext/backend.zig | 25 ++++++--- zig/src/families/flashnext/session.zig | 56 +++++++++++++++++++- zig/tests/flashnext_batch.zig | 71 +++++++++++++++++++++++++- 4 files changed, 214 insertions(+), 9 deletions(-) create mode 100644 tools/zig/flashnext_mtp_chain.py diff --git a/tools/zig/flashnext_mtp_chain.py b/tools/zig/flashnext_mtp_chain.py new file mode 100644 index 0000000000..8513130d16 --- /dev/null +++ b/tools/zig/flashnext_mtp_chain.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Compare serial and GPU-chained MTP drafts with alternating HTTP blocks and exact token receipts.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import statistics + +import flashnext_batch_serve as gate + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary", type=Path) + parser.add_argument("model", type=Path) + parser.add_argument("fixtures", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--tokens", type=int, default=128) + parser.add_argument("--rounds", type=int, default=4) + args = parser.parse_args() + if args.tokens < 1 or args.rounds < 3: + parser.error("tokens must be positive; rounds must include one warmup and at least two measured rounds") + args.output.mkdir(parents=True, exist_ok=True) + cases = [json.loads((args.fixtures / f"{name}.json").read_text()) for name in + ("short", "thinking", "sparse", "sparse-thinking")] + receipt = {"exact": False, "binary_sha256": hashlib.sha256(args.binary.read_bytes()).hexdigest(), + "tokens": args.tokens, "rounds": args.rounds, "warmup_rounds": 1, "blocks": []} + + def save(): + (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + + save() + reference = None + previous = os.environ.get("FZ_BATCH_MTP_CHAIN") + try: + # The one-slot legacy engine is a control; four-slot blocks reverse order to limit drift. + for block, (slots, chain) in enumerate(((1, 0), (1, 1), (4, 0), (4, 1), (4, 1), (4, 0))): + os.environ["FZ_BATCH_MTP_CHAIN"] = str(chain) + print(f"BLOCK {block}: parallel={slots} chain={chain}", flush=True) + batches = gate.run(args, slots, cases, block) + receipt["blocks"].append({"parallel": slots, "chain": chain, "batches": batches}) + save() + for batch in batches: + replies = [{key: result[key] for key in ("reply", "tokens", "prompt_tokens", "token_sha")} + for result in batch["results"]] + if reference is None: + reference = replies + if replies != reference: + raise RuntimeError(f"token or reply mismatch in block {block}, round {batch['iteration']}") + receipt["exact"] = True + medians = {} + for slots in (1, 4): + seconds = {chain: statistics.median( + batch["seconds"] for block in receipt["blocks"] + if block["parallel"] == slots and block["chain"] == chain + for batch in block["batches"][1:]) for chain in (0, 1)} + medians[str(slots)] = {"serial_seconds": seconds[0], "chain_seconds": seconds[1], + "throughput_gain_percent": (seconds[0] / seconds[1] - 1) * 100} + receipt["warm_medians"] = medians + save() + print(json.dumps(medians, indent=2), flush=True) + finally: + if previous is None: + os.environ.pop("FZ_BATCH_MTP_CHAIN", None) + else: + os.environ["FZ_BATCH_MTP_CHAIN"] = previous + + +if __name__ == "__main__": + main() diff --git a/zig/src/families/flashnext/backend.zig b/zig/src/families/flashnext/backend.zig index 6cfc6d5265..e4c02f2591 100644 --- a/zig/src/families/flashnext/backend.zig +++ b/zig/src/families/flashnext/backend.zig @@ -22,6 +22,10 @@ pub const Backend = struct { hidden: Buf, window_count: u64 = 0, shared_count: u64 = 0, + chain_drafts: bool = false, + draft_calls: u64 = 0, + draft_waits: u64 = 0, + draft_seconds: f64 = 0, pub fn init(gpa: std.mem.Allocator, e: *Engine, n: u32, capacity: usize) !Backend { if (n < 2 or n > fz.MAXR or e.r.tp != null) return error.BadBatchSlots; @@ -34,7 +38,7 @@ pub const Backend = struct { made += 1; } const hidden = try e.r.device.buffer(fz.MAXR * fz.WIDE * 2, fz.opts); - return .{ .gpa = gpa, .e = e, .primary = e.m, .sessions = sessions, .hidden = .{ .b = hidden } }; + return .{ .gpa = gpa, .e = e, .primary = e.m, .sessions = sessions, .hidden = .{ .b = hidden }, .chain_drafts = if (std.c.getenv("FZ_BATCH_MTP_CHAIN")) |v| v[0] == '1' else false }; } pub fn deinit(b: *Backend) void { @@ -276,11 +280,20 @@ pub const Backend = struct { defer old.restore(b); s.held_n = r.depth; if (follow.len == 0) return error.DraftOutOfStep; - const token = try s.m.mtpAbsorb(follow, hidden); - if (r.depth > 0) s.held[0] = token; - if (r.depth > 1) for (1..r.depth) |i| { - s.held[i] = try s.m.mtpChain(s.held[i - 1]); - }; + const start = mtl.clock.seconds(); + if (b.chain_drafts and r.depth > 0 and follow.len <= fz.MAXR) { + try s.draftChain(follow, hidden, r.depth); + b.draft_waits += 1; + } else { + const token = try s.m.mtpAbsorb(follow, hidden); + if (r.depth > 0) s.held[0] = token; + if (r.depth > 1) for (1..r.depth) |i| { + s.held[i] = try s.m.mtpChain(s.held[i - 1]); + }; + b.draft_waits += (follow.len + fz.MAXR - 1) / fz.MAXR + (r.depth -| 1); + } + b.draft_calls += 1; + b.draft_seconds += mtl.clock.seconds() - start; } } diff --git a/zig/src/families/flashnext/session.zig b/zig/src/families/flashnext/session.zig index 2945b07091..612b37e434 100644 --- a/zig/src/families/flashnext/session.zig +++ b/zig/src/families/flashnext/session.zig @@ -9,6 +9,8 @@ pub const Session = struct { m: fz.Model, owned: std.ArrayList(mtl.Buffer) = .empty, select: [fz.CATCH]?fz.Select = @splat(null), + draft_select: [fz.MAXR]?fz.Select = @splat(null), + draft_ids: Buf = undefined, shape: mtl.Buffer = undefined, last: Buf = undefined, held: [fz.MAXR]u32 = @splat(0), @@ -49,6 +51,19 @@ pub const Session = struct { for (&s.select) |*sel| { sel.* = try s.selector(gpa, e); } + s.draft_ids = try s.buffer(gpa, e, fz.MAXR * 4); + for (&s.draft_select) |*sel| sel.* = try s.selectorMeta(gpa, e, s.select[fz.CATCH - 1]); + // Sessions draft serially, but every encoded step needs immutable host metadata. + for (s.m.mtp.slots[0..fz.MAXR]) |*sl| sl.* = .{ + .rows = try s.words(gpa, e, &.{1}), + .md = try s.words(gpa, e, &.{ 1, 16, 0, 0, 0, 0, 0, 0 }), + .md4 = try s.words(gpa, e, &.{ 4, 16, 0, 0, 0, 0, 0, 0 }), + .ids8 = try s.buffer(gpa, e, fz.MAXR * 4), + .pos8 = try s.buffer(gpa, e, fz.MAXR * 4), + .nk8 = try s.buffer(gpa, e, fz.MAXR * 4), + .kvmeta = try s.buffer(gpa, e, 3 * 4), + .n_add = try s.buffer(gpa, e, 4), + }; return s; } @@ -76,10 +91,16 @@ pub const Session = struct { fn selector(s: *Session, gpa: std.mem.Allocator, e: *Engine) !?fz.Select { var sel = e.r.sel orelse return null; - sel.start = try s.buffer(gpa, e, 16); sel.starts = try s.buffer(gpa, e, fz.CATCH * 256); sel.sc = try s.buffer(gpa, e, fz.MAXR * ((s.m.cap + 3) / 4) * 4); sel.keys = try s.buffer(gpa, e, fz.MAXR * fz.KW * 4); + return s.selectorMeta(gpa, e, sel); + } + + // Large score/key scratch is shared by sequential GPU steps; CPU-written inputs are not. + fn selectorMeta(s: *Session, gpa: std.mem.Allocator, e: *Engine, source: ?fz.Select) !?fz.Select { + var sel = source orelse return null; + sel.start = try s.buffer(gpa, e, 16); sel.complete = try s.buffer(gpa, e, fz.MAXR * 4); sel.ends = try s.buffer(gpa, e, fz.MAXR * 4); sel.counts = try s.buffer(gpa, e, fz.MAXR * 4); @@ -91,6 +112,36 @@ pub const Session = struct { return sel; } + /// Absorb a kept window and generate its drafts with one GPU submission and one readback. + pub fn draftChain(s: *Session, follow: []const u32, hidden: Buf, depth: usize) !void { + if (follow.len == 0 or follow.len > fz.MAXR or depth == 0 or depth >= fz.MAXR) return error.DraftOutOfStep; + const m = &s.m; + const h = &m.mtp; + const r = m.r; + const old_select = r.sel; + defer r.sel = old_select; + h.pos -= h.drafted; + h.drafted = 0; + h.pooled_n = @min(h.pooled_n, h.pos / 4); + @memcpy(h.slots[0].ids8.b.slice(u32, follow.len), follow); + const cb = r.queue.commandBuffer(); + r.enc = cb.compute(if (r.serial) .serial else .concurrent); + var ended = false; + errdefer if (!ended) r.enc.end(); + for (0..depth) |i| { + r.sel = s.draft_select[i]; + const rows = if (i == 0) follow.len else 1; + const input = if (i == 0) h.slots[0].ids8 else s.draft_ids.at((i - 1) * 4); + const streams = if (i == 0) hidden else h.h[1]; + try m.mtpEncode(i, rows, input, streams, s.draft_ids.at(i * 4)); + h.pos += rows; + } + ended = true; // finish ends the encoder even if submission fails. + try m.finish(cb); + h.drafted = depth - 1; + @memcpy(s.held[0..depth], s.draft_ids.b.slice(u32, depth)); + } + pub fn deinit(s: *Session, gpa: std.mem.Allocator) void { for (s.owned.items) |b| b.deinit(); s.owned.deinit(gpa); @@ -134,6 +185,7 @@ pub const Session = struct { /// Metadata and selectors are owned even when the original engine supplies this slot's caches. pub fn overhead(capacity: usize) u64 { - return fz.CATCH * (fz.MAXR * ((capacity + 3) / 4) * 4 + fz.MAXR * fz.KW * 4 + 4096) + (2 << 20); + return fz.CATCH * (fz.MAXR * ((capacity + 3) / 4) * 4 + fz.MAXR * fz.KW * 4 + 4096) + + fz.MAXR * 17 * 64 + 64 + (2 << 20); } }; diff --git a/zig/tests/flashnext_batch.zig b/zig/tests/flashnext_batch.zig index 8f04569f10..dd3702c2c5 100644 --- a/zig/tests/flashnext_batch.zig +++ b/zig/tests/flashnext_batch.zig @@ -47,6 +47,65 @@ fn prompt(a: std.mem.Allocator, path: []const u8) ![]u32 { return ids; } +fn draftDepths(a: std.mem.Allocator, back: *Backend, prompts: []const []u32) !void { + const fz = tf.flashnext_replay; + const hidden = try a.alloc(u8, fz.WIDE * 2); + defer a.free(hidden); + for (prompts, 0..) |ids, p| { + var stream = try lanes.Stream.init(a, .{ .id = "draft-depth", .prompt = ids, .max_new = 32 }); + defer stream.deinit(a); + const be = back.backend(); + try be.prefill(&stream); + defer be.release(&stream); + const first = try be.first(&stream, ids.len); + const s = &back.sessions[back.by.get(&stream).?]; + const base_pos = s.m.mtp.pos; + const base_pool = s.m.mtp.pooled_n; + var path: [fz.MAXR]u32 = undefined; + var follow: [fz.MAXR]u32 = undefined; + for (0..fz.MAXR) |i| { + path[i] = @intCast(i); + follow[i] = @intCast(first); + @memcpy(back.hidden.b.contents()[i * fz.WIDE * 2 ..][0 .. fz.WIDE * 2], s.last.b.contents()[s.last.off..][0 .. fz.WIDE * 2]); + } + // Controlled row inputs exercise the head independently of the scheduler's depth choice. + s.round = .{ .start = ids.len, .rows = fz.MAXR }; + for ([_]usize{ 1, 4, 16 }) |rows| for ([_]bool{ false, true }) |rewind| for ([_]u32{ 0, 1, 2, 4, 8, 15 }) |depth| { + const request = [_]lanes.backend.DraftRequest{.{ .stream = &stream, .follow = if (rows == 1) &.{} else follow[0..rows], .first = if (rows == 1) .{ .handle = first } else null, .rows = if (rows == 1) null else path[0..rows], .start = ids.len, .position = ids.len + rows, .depth = depth }}; + s.m.mtp.pos = base_pos; + s.m.mtp.drafted = 0; + s.m.mtp.pooled_n = base_pool; + back.chain_drafts = false; + if (rewind) { + const seed = [_]lanes.backend.DraftRequest{.{ .stream = &stream, .follow = &.{}, .first = .{ .handle = first }, .rows = null, .start = ids.len, .position = ids.len + 1, .depth = 4 }}; + try be.draft(&seed); + if (s.m.mtp.drafted != 3) return error.MissingRewindCoverage; + } + const start_pos = s.m.mtp.pos; + const start_drafted = s.m.mtp.drafted; + const start_pool = s.m.mtp.pooled_n; + back.chain_drafts = false; + try be.draft(&request); + const expected = s.held; + const pos = s.m.mtp.pos; + const drafted = s.m.mtp.drafted; + const pooled = s.m.mtp.pooled_n; + @memcpy(hidden, s.m.mtp.last.b.contents()[s.m.mtp.last.off..][0..hidden.len]); + s.m.mtp.pos = start_pos; + s.m.mtp.drafted = start_drafted; + s.m.mtp.pooled_n = start_pool; + back.chain_drafts = true; + const waits = back.draft_waits; + try be.draft(&request); + if (back.draft_waits - waits != 1) return error.ChainWaitCount; + if (!std.mem.eql(u32, expected[0..depth], s.held[0..depth]) or + pos != s.m.mtp.pos or drafted != s.m.mtp.drafted or pooled != s.m.mtp.pooled_n or + !std.mem.eql(u8, hidden, s.m.mtp.last.b.contents()[s.m.mtp.last.off..][0..hidden.len])) return error.DraftChainMismatch; + }; + std.debug.print("PASS draft chain: prompt {d}, {d} tokens, absorb 1/4/16, rewind 0/3, depths 0/1/2/4/8/15, exact drafts, hidden bytes and cache positions\n", .{ p, ids.len }); + } +} + pub fn main(init: std.process.Init) !void { const a = init.arena.allocator(); const args = try init.minimal.args.toSlice(a); @@ -87,7 +146,13 @@ pub fn main(init: std.process.Init) !void { }; var back = try Backend.init(init.gpa, eng, @intCast(@max(4, count)), capacity); defer back.deinit(); - for ([_]bool{ false, true }) |drafts| { + try draftDepths(init.gpa, &back, prompts); + const boundary = 4 * (tf.flashnext_replay.TOP + 1) - 2; + if (prompts[longest].len < boundary) return error.NeedSparseBoundaryFixture; + try draftDepths(init.gpa, &back, &.{prompts[longest][0..boundary]}); + for ([_]struct { drafts: bool, chain: bool }{ .{ .drafts = false, .chain = false }, .{ .drafts = true, .chain = false }, .{ .drafts = true, .chain = true } }) |mode| { + const drafts = mode.drafts; + back.chain_drafts = mode.chain; var cfg = try lanes.Config.init(init.gpa, back.facts(), tf.flashnext_replay.MAXR, 15); defer cfg.deinit(init.gpa); var clock: lanes.backend.WallClock = .{ .io = init.io }; @@ -103,6 +168,9 @@ pub fn main(init: std.process.Init) !void { defer for (streams) |*s| s.deinit(init.gpa); const start = mtl.clock.seconds(); const before = back.shared_count; + const draft_before = back.draft_calls; + const waits_before = back.draft_waits; + const draft_seconds_before = back.draft_seconds; for (streams) |*s| try core.addStream(s); while (core.activeCount() > 0) try core.step(); const seconds = mtl.clock.seconds() - start; @@ -115,6 +183,7 @@ pub fn main(init: std.process.Init) !void { if (back.shared_count == before) return error.NoSharedForward; const baseline = if (drafts) baseline_mtp_seconds else baseline_seconds; std.debug.print("PASS drafts {any}: {d} streams, {d} exact tokens, {d} shared forwards, {d:.3}s, legacy serial {d:.3}s\n", .{ drafts, count, count * n, back.shared_count - before, seconds, baseline }); + std.debug.print("MEASURE chain {any}: draft calls {d}, waits {d}, draft {d:.6}s, total {d:.6}s\n", .{ mode.chain, back.draft_calls - draft_before, back.draft_waits - waits_before, back.draft_seconds - draft_seconds_before, seconds }); // Admit a third prompt after a shared round, cancel its peer, and reuse the released slot. var first = try lanes.Stream.init(init.gpa, .{ .id = "survivor", .prompt = prompts[0], .max_new = n, .drafts = drafts }); defer first.deinit(init.gpa); From 39824c609214be843e90541fafbd7b7077c069d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:41:56 +0200 Subject: [PATCH 05/31] Qualify experimental batch MTP chaining on M3 Ultra --- README.md | 2 + .../flashnext-mtp-chain-2026-10-10/README.md | 94 + .../checkpoint-gate.log | 17 + .../flashnext-mtp-chain-2026-10-10/golden.log | 61 + .../flashnext-mtp-chain-2026-10-10/http.json | 1689 +++++++++++++++++ .../manifest.json | 39 + .../memory.json | 8 + .../prompts/short.json | 1 + .../prompts/sparse-thinking.json | 1 + .../prompts/sparse.json | 1 + .../prompts/thinking.json | 1 + tools/zig/flashnext_mtp_chain.py | 11 +- 12 files changed, 1921 insertions(+), 4 deletions(-) create mode 100644 docs/receipts/flashnext-mtp-chain-2026-10-10/README.md create mode 100644 docs/receipts/flashnext-mtp-chain-2026-10-10/checkpoint-gate.log create mode 100644 docs/receipts/flashnext-mtp-chain-2026-10-10/golden.log create mode 100644 docs/receipts/flashnext-mtp-chain-2026-10-10/http.json create mode 100644 docs/receipts/flashnext-mtp-chain-2026-10-10/manifest.json create mode 100644 docs/receipts/flashnext-mtp-chain-2026-10-10/memory.json create mode 100644 docs/receipts/flashnext-mtp-chain-2026-10-10/prompts/short.json create mode 100644 docs/receipts/flashnext-mtp-chain-2026-10-10/prompts/sparse-thinking.json create mode 100644 docs/receipts/flashnext-mtp-chain-2026-10-10/prompts/sparse.json create mode 100644 docs/receipts/flashnext-mtp-chain-2026-10-10/prompts/thinking.json diff --git a/README.md b/README.md index d6ae433b1d..8ccc11b51c 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,8 @@ The comparison fixes the checkpoint, backend, settings and runtime. Different quantizations and different backends can produce different outputs. Flash Next on Metal uses shared target rounds with an explicit `--parallel N` above one on a single Mac. Its default `auto`, `--parallel 1` and two-Mac speed-up mode serve one active reply; MTP heads remain serial. +In shared mode, `FZ_BATCH_MTP_CHAIN=1` experimentally submits each stream's absorb and draft chain in one Metal command buffer. +The option defaults to off. See the [MTP chain receipt](docs/receipts/flashnext-mtp-chain-2026-10-10/README.md) for qualification and measurements. On Metal, Qwen3.8-27B runs one active reply per engine; Nemotron, GLM and the 2B model use shared lane rounds. ## Models on disk diff --git a/docs/receipts/flashnext-mtp-chain-2026-10-10/README.md b/docs/receipts/flashnext-mtp-chain-2026-10-10/README.md new file mode 100644 index 0000000000..44fcbdfc89 --- /dev/null +++ b/docs/receipts/flashnext-mtp-chain-2026-10-10/README.md @@ -0,0 +1,94 @@ +# Flash Next batch MTP chaining receipt + +The shared Metal adapter can submit a stream's kept-window absorb and all chained MTP drafts in one command buffer. +`FZ_BATCH_MTP_CHAIN=1` enables the experiment; it defaults to off. Streams still draft serially. +Each encoded step owns its CPU-written position, row and selector metadata. GPU score/key scratch remains shared. +The new buffers add 17,472 bytes of payload per session; admission still checks the actual process footprint. + +## Qualified scope + +TensorFold 1.0.5, upstream `bec00ae6961a43a9ab73c4fa9338c167b3f82851`; +inference source `6dbf17059a846f2c9c40a92c79b5558d41f2960f`. +The [manifest](manifest.json) pins the source tree, compiler archive and both executed binaries. +Rebuilt binaries match the hashes of the executed artifacts. + +One 80-core M3 Ultra, 256 GiB RAM, macOS 26.7.1, system Metal, Zig 0.17.0 targeting `apple_m1`. +Checkpoint `TensorFold/Qwen3.8-Flash-Next-MLX-6bit-MTP`, download metadata revision +`45032a193a5d4126ba8534822cbaf8fb30b3e368`. +Public fixtures have 23, 66, 2,764 and 11,237 prompt tokens, with thinking off and on. +The direct gate also uses the first 2,050 tokens of the longest fixture to cross into sparse selection during drafting. + +## Correctness + +[Checkpoint gate](checkpoint-gate.log): + +- 180 paired comparisons at absorb widths 1/4/16, previous speculative tails 0/3, and draft depths 0/1/2/4/8/15. + Draft tokens, final hidden-state bytes, head positions and pooled-block counts match serial drafting exactly. + Wider absorbs use controlled repeated row inputs to exercise the head independently of the scheduler. +- Four 64-token replies match fresh legacy plain decoding in plain, serial-MTP and chained-MTP modes. +- All three modes pass cancellation, slot reuse, prompt admission, multi-chunk marks and prefix restoration with an active peer. +- [HTTP receipt](http.json): 96 responses across six blocks, four rounds each. Bodies, token fingerprints and usage match. + Every four-slot block observes four active sessions. Prefix caching is disabled. + +The executed HTTP helper is pinned to the inference-source commit. +The final helper additionally invalidates stale success before reading fixtures and rejects unobserved shared sessions. +Both failure cases were checked locally; the recorded run independently satisfies the strengthened parallelism gate. + +## Timings + +The checkpoint gate records one observation, including prefill and decode: + +| Phase | Serial MTP | Chained MTP | +| --- | ---: | ---: | +| Draft calls | 84 | 84 | +| GPU waits during drafting | 256 | 84 | +| Draft time | 0.312223 s | 0.261944 s | +| Total group time | 12.473931 s | 12.375544 s | + +Draft time fell 16.1%. This single observation does not establish a general speedup. + +HTTP uses identical public requests, greedy decoding, a 16,384-token context and 128-token reply limit. +The one-slot control runs off/on. Four-slot blocks run off/on/on/off. +Each block has one excluded warmup and three measured rounds; every group emits 459 tokens. + +| Parallel slots | Serial warm median | Chain warm median | Group throughput change | +| --- | ---: | ---: | ---: | +| 1, legacy control | 15.191969 s | 15.226884 s | -0.23% | +| 4, shared adapter | 14.599447 s | 14.496549 s | +0.71% | + +These are end-to-end group timings, including prompt processing. They are not an isolated decode benchmark. +The effect is small; the option stays experimental. The one-slot driver does not consume the switch. + +## Memory and cleanup + +A [footprint sample](memory.json) from the final four-slot serial block reports a process-lifetime physical peak +of 186,421,791,496 bytes. This is not a whole-run peak across all server restarts. +System swap usage remained zero at the probes. One model process ran at a time under the existing admission lock. +The original service was restored and verified healthy; native test processes and the lock were removed. + +## Host validation and limits + +`zig build test` and whole-tree `zig fmt --check build.zig zig` exit successfully with Zig 0.17.0. +[Golden check](golden.log): 309 equal, 10 documented differences, zero unexpected differences. +Both native builds pass. The lean checker finds 542 inherited problems on upstream and candidate, with none added. +Astra reviewed the chain, the metadata lifetime, state accounting, direct gates and benchmark failure handling. + +This receipt qualifies these fixtures on one M3 Ultra and this checkpoint. +CUDA, other Apple chips, other RAM classes/checkpoints, more than four sessions, 32K+ contexts, +separate cold-prefill sweeps, a previous-release comparison and a standard-server comparison were not run. +The patch is a shared-adapter scheduling change using the existing Metal operators and precision. + +## Reproduce + +Run on an idle GPU with no other model loaded and use the platform's admission lock and service restoration procedure. + +```sh +zig build native tf-flashnext-batch -Dcpu=apple_m1 -Doptimize=ReleaseFast +FZ_N=64 zig-out/bin/tf-flashnext-batch "$MODEL" \ + docs/receipts/flashnext-mtp-chain-2026-10-10/prompts/{short,thinking,sparse,sparse-thinking}.json +python3 tools/zig/flashnext_mtp_chain.py \ + zig-out/native/bin/tensorfold-native "$MODEL" \ + docs/receipts/flashnext-mtp-chain-2026-10-10/prompts output --tokens 128 --rounds 4 +zig build test test-golden -Dcpu=apple_m1 +zig fmt --check build.zig zig +``` diff --git a/docs/receipts/flashnext-mtp-chain-2026-10-10/checkpoint-gate.log b/docs/receipts/flashnext-mtp-chain-2026-10-10/checkpoint-gate.log new file mode 100644 index 0000000000..0c48b889e7 --- /dev/null +++ b/docs/receipts/flashnext-mtp-chain-2026-10-10/checkpoint-gate.log @@ -0,0 +1,17 @@ +PASS draft chain: prompt 0, 23 tokens, absorb 1/4/16, rewind 0/3, depths 0/1/2/4/8/15, exact drafts, hidden bytes and cache positions +PASS draft chain: prompt 1, 66 tokens, absorb 1/4/16, rewind 0/3, depths 0/1/2/4/8/15, exact drafts, hidden bytes and cache positions +PASS draft chain: prompt 2, 2764 tokens, absorb 1/4/16, rewind 0/3, depths 0/1/2/4/8/15, exact drafts, hidden bytes and cache positions +PASS draft chain: prompt 3, 11237 tokens, absorb 1/4/16, rewind 0/3, depths 0/1/2/4/8/15, exact drafts, hidden bytes and cache positions +PASS draft chain: prompt 0, 2050 tokens, absorb 1/4/16, rewind 0/3, depths 0/1/2/4/8/15, exact drafts, hidden bytes and cache positions +PASS drafts false: 4 streams, 256 exact tokens, 63 shared forwards, 12.716s, legacy serial 15.130s +MEASURE chain false: draft calls 0, waits 0, draft 0.000000s, total 12.716432s +PASS drafts false: cancellation, slot reuse and prompt admission +PASS drafts false: multi-chunk mark and restored prefix in a dynamic-capacity slot +PASS drafts true: 4 streams, 256 exact tokens, 20 shared forwards, 12.474s, legacy serial 13.090s +MEASURE chain false: draft calls 84, waits 256, draft 0.312223s, total 12.473931s +PASS drafts true: cancellation, slot reuse and prompt admission +PASS drafts true: multi-chunk mark and restored prefix in a dynamic-capacity slot +PASS drafts true: 4 streams, 256 exact tokens, 20 shared forwards, 12.376s, legacy serial 13.090s +MEASURE chain true: draft calls 84, waits 84, draft 0.261944s, total 12.375544s +PASS drafts true: cancellation, slot reuse and prompt admission +PASS drafts true: multi-chunk mark and restored prefix in a dynamic-capacity slot diff --git a/docs/receipts/flashnext-mtp-chain-2026-10-10/golden.log b/docs/receipts/flashnext-mtp-chain-2026-10-10/golden.log new file mode 100644 index 0000000000..28cecc1bdd --- /dev/null +++ b/docs/receipts/flashnext-mtp-chain-2026-10-10/golden.log @@ -0,0 +1,61 @@ +KNOWN-DIFFER tools/two_tools-stream + reply 0 body differs at 313: golden [content": "\n"}, "finish_reason": null}]} + +data: {"id": "", "type": "function"] +KNOWN-DIFFER tools/two_tools-single-stream + reply 0 body differs at 313: golden [content": "\n"}, "finish_reason": null}]} + +data: {"id": "", "type": "function"] +KNOWN-DIFFER errors/format-json + reply 0 body differs at 41: golden [needs the checkpoint's config.json vocab_size", "type": "inv] zig [(response_format and the guided_* fields) is not supported b] + reply 0 headers differ +KNOWN-DIFFER errors/stream-structured + connection after: golden closed, zig open + reply 0 status: golden HTTP/1.1 200 OK, zig HTTP/1.1 400 Bad Request + reply 0 body differs at 0: golden [data: {"id": "", "object": "chat.completion.chunk", "cr] zig [{"error": {"message": "structured output (response_format an] +KNOWN-DIFFER anthropic/tool-use-stream + reply 0 body differs at 702: golden [\n"}} + +event: content_block_stop +data: {"type": "content_blo] zig ["}} + +event: content_block_stop +data: {"type": "content_block] +KNOWN-DIFFER anthropic/refuse-format + reply 0 body differs at 91: golden [needs the checkpoint's config.json vocab_size"}}] zig [(response_format and the guided_* fields) is not supported b] + reply 0 headers differ +KNOWN-DIFFER cancel/metrics-after + reply 0 body differs at 307: golden [prompt_tokens_total Prompt tokens of finished requests. +# TY] zig [generation_tokens_running Generated tokens held by live stre] +KNOWN-DIFFER metrics-open/metrics + reply 0 body differs at 307: golden [prompt_tokens_total Prompt tokens of finished requests. +# TY] zig [generation_tokens_running Generated tokens held by live stre] +KNOWN-DIFFER metrics-open/v1-metrics + reply 0 body differs at 487: golden [prompt_tokens_total Prompt tokens of finished requests. +# TY] zig [generation_tokens_running Generated tokens held by live stre] +KNOWN-DIFFER keys/metrics-counted + reply 0 body differs at 887: golden [prompt_tokens_total Prompt tokens of finished requests. +# TY] zig [generation_tokens_running Generated tokens held by live stre] + chat 16 equal 0 differ 0 known + thinking 16 equal 0 differ 0 known + stops 13 equal 0 differ 0 known + usage 10 equal 0 differ 0 known + tools 50 equal 0 differ 2 known + errors 37 equal 0 differ 2 known + completions 15 equal 0 differ 0 known + responses 29 equal 0 differ 0 known + anthropic 27 equal 0 differ 2 known + tokens 19 equal 0 differ 0 known + status 11 equal 0 differ 0 known + framing 42 equal 0 differ 0 known + cancel 1 equal 0 differ 1 known + lifecycle 2 equal 0 differ 0 known +metrics-open 2 equal 0 differ 2 known + keys 19 equal 0 differ 1 known +309/319 equal to the frozen Python answers, 0 unexpected differences +Build Summary: 4/4 steps succeeded +test-golden success ++- run exe golden_check success 1m + +- compile exe golden_check debug native-native success 1s MaxRSS:291M + +- compile exe fake_serve debug native-native success 3s MaxRSS:588M + diff --git a/docs/receipts/flashnext-mtp-chain-2026-10-10/http.json b/docs/receipts/flashnext-mtp-chain-2026-10-10/http.json new file mode 100644 index 0000000000..9af92908f8 --- /dev/null +++ b/docs/receipts/flashnext-mtp-chain-2026-10-10/http.json @@ -0,0 +1,1689 @@ +{ + "exact": true, + "binary_sha256": "5983a5f66b4df5e218756155b6fc446f8ea570eb25b2c2934ae34c29c1a0ce83", + "tokens": 128, + "rounds": 4, + "warmup_rounds": 1, + "blocks": [ + { + "parallel": 1, + "chain": 0, + "batches": [ + { + "iteration": 0, + "seconds": 15.167411625, + "tokens": 459, + "tokens_per_second": 30.262249838558066, + "peak_running": 1, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 0.6614614580000051 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 2.180329624999999 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 5.771157625000001 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 15.165531458000004 + } + ] + }, + { + "iteration": 1, + "seconds": 15.191968916999997, + "tokens": 459, + "tokens_per_second": 30.2133319589914, + "peak_running": 1, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 0.6993732500000007 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 2.2186996249999993 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 5.812087542 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 15.191467250000002 + } + ] + }, + { + "iteration": 2, + "seconds": 15.190096166000004, + "tokens": 459, + "tokens_per_second": 30.217056889170973, + "peak_running": 1, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 0.6992123750000019 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 2.2166298340000026 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 5.811752584000004 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 15.189728959000007 + } + ] + }, + { + "iteration": 3, + "seconds": 15.195858458000004, + "tokens": 459, + "tokens_per_second": 30.205598536511445, + "peak_running": 1, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 0.6984240419999992 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 2.2160770410000055 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 5.810912999999999 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 15.195499958 + } + ] + } + ] + }, + { + "parallel": 1, + "chain": 1, + "batches": [ + { + "iteration": 0, + "seconds": 15.195600584000005, + "tokens": 459, + "tokens_per_second": 30.206111134777892, + "peak_running": 1, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 0.6617389590000187 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 2.1713265410000133 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 5.768877375000017 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 15.194293208999994 + } + ] + }, + { + "iteration": 1, + "seconds": 15.24324150000001, + "tokens": 459, + "tokens_per_second": 30.111705571285455, + "peak_running": 1, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 0.7029760829999816 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 2.2231301660000042 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 5.825613583000006 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 15.242891458000003 + } + ] + }, + { + "iteration": 2, + "seconds": 15.219083416999979, + "tokens": 459, + "tokens_per_second": 30.159503527478474, + "peak_running": 1, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 0.7033312499999909 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 2.220455708000003 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 5.8215707500000065 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 15.218772624999986 + } + ] + }, + { + "iteration": 3, + "seconds": 15.226884290999976, + "tokens": 459, + "tokens_per_second": 30.14405253419422, + "peak_running": 1, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 0.7038725419999992 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 2.221313209000016 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 5.820594207999989 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 15.22652450000001 + } + ] + } + ] + }, + { + "parallel": 4, + "chain": 0, + "batches": [ + { + "iteration": 0, + "seconds": 14.575401249999999, + "tokens": 459, + "tokens_per_second": 31.491414344424996, + "peak_running": 4, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 12.795987290999989 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 14.329476875000012 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 14.574683375000006 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 14.074014041999988 + } + ] + }, + { + "iteration": 1, + "seconds": 14.597603333000023, + "tokens": 459, + "tokens_per_second": 31.44351778365995, + "peak_running": 4, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 12.812376624999985 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 14.350047541999999 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 14.597032458000001 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 14.095057166999993 + } + ] + }, + { + "iteration": 2, + "seconds": 14.604756915999985, + "tokens": 459, + "tokens_per_second": 31.42811637605215, + "peak_running": 4, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 12.819853125000009 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 14.357231958 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 14.604170457999999 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 14.102166624999995 + } + ] + }, + { + "iteration": 3, + "seconds": 14.601289917000031, + "tokens": 459, + "tokens_per_second": 31.435578815923254, + "peak_running": 4, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 12.817181833000006 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 14.353887000000043 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 14.600695916999996 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 14.100120375000017 + } + ] + } + ] + }, + { + "parallel": 4, + "chain": 1, + "batches": [ + { + "iteration": 0, + "seconds": 14.466401125000004, + "tokens": 459, + "tokens_per_second": 31.728692992397573, + "peak_running": 4, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 12.750999500000034 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 14.219419833000018 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 14.465403707999997 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 13.977442999999994 + } + ] + }, + { + "iteration": 1, + "seconds": 14.490603332999967, + "tokens": 459, + "tokens_per_second": 31.675699724296706, + "peak_running": 4, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 12.769629000000009 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 14.243087082999978 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 14.489981 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 13.998929750000002 + } + ] + }, + { + "iteration": 2, + "seconds": 14.49711662499999, + "tokens": 459, + "tokens_per_second": 31.66146840596313, + "peak_running": 4, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 12.775831416000017 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 14.249283416999958 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 14.496512375000009 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 14.00521875000004 + } + ] + }, + { + "iteration": 3, + "seconds": 14.498926374999996, + "tokens": 459, + "tokens_per_second": 31.65751643455739, + "peak_running": 4, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 12.779494374999956 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 14.251451624999959 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 14.498408584000003 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 14.008449874999997 + } + ] + } + ] + }, + { + "parallel": 4, + "chain": 1, + "batches": [ + { + "iteration": 0, + "seconds": 14.468143209000004, + "tokens": 459, + "tokens_per_second": 31.724872595570936, + "peak_running": 4, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 12.746160792000012 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 14.220339542000033 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 14.46728595799999 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 13.976755999999966 + } + ] + }, + { + "iteration": 1, + "seconds": 14.49598050000003, + "tokens": 459, + "tokens_per_second": 31.663949879071584, + "peak_running": 4, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 12.777879291999966 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 14.248624499999949 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 14.495467209000026 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 14.005071416999954 + } + ] + }, + { + "iteration": 2, + "seconds": 14.45111637499997, + "tokens": 459, + "tokens_per_second": 31.762252001101952, + "peak_running": 4, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 12.729356499999994 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 14.202998207999997 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 14.450510916000042 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 13.959070084000018 + } + ] + }, + { + "iteration": 3, + "seconds": 14.503579874999957, + "tokens": 459, + "tokens_per_second": 31.64735906279148, + "peak_running": 4, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 12.780467250000015 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 14.255841832999977 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 14.503076749999991 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 14.011824292000028 + } + ] + } + ] + }, + { + "parallel": 4, + "chain": 0, + "batches": [ + { + "iteration": 0, + "seconds": 14.601471999999944, + "tokens": 459, + "tokens_per_second": 31.43518680856298, + "peak_running": 4, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 12.817871416999992 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 14.354455875000099 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 14.600365915999987 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 14.09945741699994 + } + ] + }, + { + "iteration": 1, + "seconds": 14.601728291999962, + "tokens": 459, + "tokens_per_second": 31.43463505285729, + "peak_running": 4, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 12.816567583000051 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 14.35420770799999 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 14.601220000000012 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 14.099293999999986 + } + ] + }, + { + "iteration": 2, + "seconds": 14.580812250000008, + "tokens": 459, + "tokens_per_second": 31.479727749734913, + "peak_running": 4, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 12.81786724999995 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 14.334535042000084 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 14.580225917000007 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 14.082230750000008 + } + ] + }, + { + "iteration": 3, + "seconds": 14.594373750000045, + "tokens": 459, + "tokens_per_second": 31.450475906854077, + "peak_running": 4, + "peak_waiting": 3, + "results": [ + { + "reply": { + "message": { + "role": "assistant", + "content": "2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71" + }, + "finish_reason": "stop" + }, + "tokens": 75, + "prompt_tokens": 23, + "cached_tokens": 0, + "token_sha": "d5242ff14875", + "seconds": 12.812724624999987 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "A rational number is", + "reasoning_content": "We need answer user: \"Explain why a square root of two cannot be a rational number.\" Need provide proof likely classic contradiction. Need final answer. Need be clear. Could include proof by contradiction: assume sqrt2 = a/b in lowest terms, square, a^2=2b^2, a even, a=2k, then 4k^2=2b^2 => b^2=2k^2 so b even, contradiction to lowest terms. Also mention infinite descent maybe. Need maybe explain rational number definition. Ensure not too terse. Final with steps." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 66, + "cached_tokens": 0, + "token_sha": "fe10693d9e4f", + "seconds": 14.348179957999946 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "Based on the data provided, here is the summary and explanation of the pattern.\n\n### Summary of the List\nThe list contains **240 items** (indexed from **Item 0** to **Item 239**). The values associated with these items follow a strict, repeating numerical sequence.\n\n### Explanation of the Repetition Pattern\n\nThe values repeat in a cycle of **97 numbers**, ranging from **0 to 96**.\n\n#### 1. The Cycle Length\nThe sequence starts at `0` and increments by `1` until it reaches `96`. Immediately after `96" + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 2764, + "cached_tokens": 0, + "token_sha": "575c66d6d071", + "seconds": 14.593745624999997 + }, + { + "reply": { + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "We need answer user: \"Row 0: ... Row 719: ... Describe an algorithm to calculate the total number of blue counters.\" Need likely infer pattern and describe algorithm. Need maybe compute total? User asks describe algorithm, not necessarily compute? Could include formula and result. Need analyze data. Rows 0-719, each row says box contains N blue counters. Pattern: counts 0..30 repeated? Let's inspect. Row 0-30: 0..30 (31 rows). Row 31-61: 0..30 (31 rows)." + }, + "finish_reason": "length" + }, + "tokens": 128, + "prompt_tokens": 11237, + "cached_tokens": 0, + "token_sha": "0b486e2f121e", + "seconds": 14.09322075 + } + ] + } + ] + } + ], + "warm_medians": { + "1": { + "serial_seconds": 15.191968916999997, + "chain_seconds": 15.226884290999976, + "throughput_gain_percent": -0.22930084272471118 + }, + "4": { + "serial_seconds": 14.599446625000027, + "chain_seconds": 14.49654856250001, + "throughput_gain_percent": 0.7098107667241393 + } + } +} diff --git a/docs/receipts/flashnext-mtp-chain-2026-10-10/manifest.json b/docs/receipts/flashnext-mtp-chain-2026-10-10/manifest.json new file mode 100644 index 0000000000..96e52eb5ed --- /dev/null +++ b/docs/receipts/flashnext-mtp-chain-2026-10-10/manifest.json @@ -0,0 +1,39 @@ +{ + "upstream_commit": "bec00ae6961a43a9ab73c4fa9338c167b3f82851", + "inference_source_commit": "6dbf17059a846f2c9c40a92c79b5558d41f2960f", + "inference_source_tree": "035e7b5232d524273818a719ef3d409102583236", + "compiler_version": "0.17.0", + "compiler_archive_sha256": "b607e9b9234790a008116ae5bdb71c6243b84b9fb42a53a9e70fde41c06c536a", + "cpu_target": "apple_m1", + "runtime": "system Metal", + "binaries": { + "tensorfold-native": "5983a5f66b4df5e218756155b6fc446f8ea570eb25b2c2934ae34c29c1a0ce83", + "tf-flashnext-batch": "053f3ba552650635ae1badd02b49aa432811718e91d569a0d62035b990d16e93" + }, + "checkpoint": "TensorFold/Qwen3.8-Flash-Next-MLX-6bit-MTP", + "checkpoint_download_metadata_revision": "45032a193a5d4126ba8534822cbaf8fb30b3e368", + "hardware": { + "soc": "M3 Ultra", + "gpu_cores": 80, + "ram_bytes": 274877906944, + "os": "macOS 26.7.1", + "os_build": "25G241" + }, + "http_helper_source_commit": "6dbf17059a846f2c9c40a92c79b5558d41f2960f", + "lifecycle": { + "one_model_process_at_a_time": true, + "admission_lock_used": true, + "original_service_restored_healthy": true, + "test_processes_and_lock_removed": true + }, + "local_checks": { + "zig_build_test_exit": 0, + "zig_fmt_whole_tree_exit": 0, + "golden_equal": 309, + "golden_known_differences": 10, + "golden_unexpected_differences": 0, + "lean_upstream_findings": 542, + "lean_candidate_findings": 542, + "helper_failure_smokes_passed": 2 + } +} diff --git a/docs/receipts/flashnext-mtp-chain-2026-10-10/memory.json b/docs/receipts/flashnext-mtp-chain-2026-10-10/memory.json new file mode 100644 index 0000000000..d48743ca58 --- /dev/null +++ b/docs/receipts/flashnext-mtp-chain-2026-10-10/memory.json @@ -0,0 +1,8 @@ +{ + "scope": "four-slot serial MTP HTTP block 5; sample after warmup", + "process": "tensorfold-native", + "page_size": 16384, + "physical_footprint_bytes": 186200771336, + "physical_peak_bytes": 186421791496, + "os_swap_used_bytes_at_probes": 0 +} diff --git a/docs/receipts/flashnext-mtp-chain-2026-10-10/prompts/short.json b/docs/receipts/flashnext-mtp-chain-2026-10-10/prompts/short.json new file mode 100644 index 0000000000..ad1581220f --- /dev/null +++ b/docs/receipts/flashnext-mtp-chain-2026-10-10/prompts/short.json @@ -0,0 +1 @@ +{"prompt": [248045, 846, 198, 826, 279, 1118, 16570, 9944, 4947, 11, 440, 874, 16308, 13, 248046, 198, 248045, 74455, 198, 248068, 271, 248069, 271], "text": "List the first twenty prime numbers, with no introduction.", "thinking": false} diff --git a/docs/receipts/flashnext-mtp-chain-2026-10-10/prompts/sparse-thinking.json b/docs/receipts/flashnext-mtp-chain-2026-10-10/prompts/sparse-thinking.json new file mode 100644 index 0000000000..46f1a5404d --- /dev/null +++ b/docs/receipts/flashnext-mtp-chain-2026-10-10/prompts/sparse-thinking.json @@ -0,0 +1 @@ +{"prompt": [248045, 8678, 198, 24342, 286, 4879, 369, 716, 310, 830, 11553, 13, 5044, 1683, 15060, 1472, 279, 3274, 11, 9307, 1328, 30800, 11, 2814, 47675, 25605, 11, 321, 60445, 55404, 11, 27224, 11, 321, 30246, 303, 279, 1534, 4087, 13, 248046, 198, 248045, 846, 198, 3002, 220, 15, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 16, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 17, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 18, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 19, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 20, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 21, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 22, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 23, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 24, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 16, 15, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 16, 16, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 16, 17, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 16, 18, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 16, 19, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 16, 20, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 16, 21, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 16, 22, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 16, 23, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 16, 24, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 17, 15, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 17, 16, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 17, 17, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 17, 18, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 17, 19, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 17, 20, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 17, 21, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 17, 22, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 17, 23, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 17, 24, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 18, 15, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 18, 16, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 18, 17, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 18, 18, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 18, 19, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 18, 20, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 18, 21, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 18, 22, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 18, 23, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 18, 24, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 19, 15, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 19, 16, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 19, 17, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 19, 18, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 19, 19, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 19, 20, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 19, 21, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 19, 22, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 19, 23, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 19, 24, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 20, 15, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 20, 16, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 20, 17, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 20, 18, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 20, 19, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 20, 20, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 20, 21, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 20, 22, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 20, 23, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 20, 24, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 21, 15, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 21, 16, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 21, 17, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 21, 18, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 21, 19, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 21, 20, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 21, 21, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 21, 22, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 21, 23, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 21, 24, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 22, 15, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 22, 16, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 22, 17, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 22, 18, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 22, 19, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 22, 20, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 22, 21, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 22, 22, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 22, 23, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 22, 24, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 23, 15, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 23, 16, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 23, 17, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 23, 18, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 23, 19, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 23, 20, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 23, 21, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 23, 22, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 23, 23, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 23, 24, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 24, 15, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 24, 16, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 24, 17, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 24, 18, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 24, 19, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 24, 20, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 24, 21, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 24, 22, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 24, 23, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 24, 24, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 16, 15, 15, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 16, 15, 16, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 16, 15, 17, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 16, 15, 18, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 16, 15, 19, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 16, 15, 20, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 16, 15, 21, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 16, 15, 22, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 16, 15, 23, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 16, 15, 24, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 16, 16, 15, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 16, 16, 16, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 16, 16, 17, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 16, 16, 18, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 16, 16, 19, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 16, 16, 20, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 16, 16, 21, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 16, 16, 22, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 16, 16, 23, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 16, 16, 24, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 16, 17, 15, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 16, 17, 16, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 16, 17, 17, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 16, 17, 18, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 16, 17, 19, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 16, 17, 20, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 16, 17, 21, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 16, 17, 22, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 16, 17, 23, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 16, 17, 24, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 16, 18, 15, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 16, 18, 16, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 16, 18, 17, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 16, 18, 18, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 16, 18, 19, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 16, 18, 20, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 16, 18, 21, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 16, 18, 22, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 16, 18, 23, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 16, 18, 24, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 16, 19, 15, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 16, 19, 16, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 16, 19, 17, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 16, 19, 18, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 16, 19, 19, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 16, 19, 20, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 16, 19, 21, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 16, 19, 22, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 16, 19, 23, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 16, 19, 24, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 16, 20, 15, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 16, 20, 16, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 16, 20, 17, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 16, 20, 18, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 16, 20, 19, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 16, 20, 20, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 16, 20, 21, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 16, 20, 22, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 16, 20, 23, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 16, 20, 24, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 16, 21, 15, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 16, 21, 16, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 16, 21, 17, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 16, 21, 18, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 16, 21, 19, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 16, 21, 20, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 16, 21, 21, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 16, 21, 22, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 16, 21, 23, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 16, 21, 24, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 16, 22, 15, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 16, 22, 16, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 16, 22, 17, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 16, 22, 18, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 16, 22, 19, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 16, 22, 20, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 16, 22, 21, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 16, 22, 22, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 16, 22, 23, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 16, 22, 24, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 16, 23, 15, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 16, 23, 16, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 16, 23, 17, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 16, 23, 18, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 16, 23, 19, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 16, 23, 20, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 16, 23, 21, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 16, 23, 22, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 16, 23, 23, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 16, 23, 24, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 16, 24, 15, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 16, 24, 16, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 16, 24, 17, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 16, 24, 18, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 16, 24, 19, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 16, 24, 20, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 16, 24, 21, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 16, 24, 22, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 16, 24, 23, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 16, 24, 24, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 17, 15, 15, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 17, 15, 16, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 17, 15, 17, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 17, 15, 18, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 17, 15, 19, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 17, 15, 20, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 17, 15, 21, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 17, 15, 22, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 17, 15, 23, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 17, 15, 24, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 17, 16, 15, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 17, 16, 16, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 17, 16, 17, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 17, 16, 18, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 17, 16, 19, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 17, 16, 20, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 17, 16, 21, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 17, 16, 22, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 17, 16, 23, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 17, 16, 24, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 17, 17, 15, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 17, 17, 16, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 17, 17, 17, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 17, 17, 18, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 17, 17, 19, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 17, 17, 20, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 17, 17, 21, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 17, 17, 22, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 17, 17, 23, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 17, 17, 24, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 17, 18, 15, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 17, 18, 16, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 17, 18, 17, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 17, 18, 18, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 17, 18, 19, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 17, 18, 20, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 17, 18, 21, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 17, 18, 22, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 17, 18, 23, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 17, 18, 24, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 17, 19, 15, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 17, 19, 16, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 17, 19, 17, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 17, 19, 18, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 17, 19, 19, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 17, 19, 20, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 17, 19, 21, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 17, 19, 22, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 17, 19, 23, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 17, 19, 24, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 17, 20, 15, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 17, 20, 16, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 17, 20, 17, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 17, 20, 18, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 17, 20, 19, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 17, 20, 20, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 17, 20, 21, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 17, 20, 22, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 17, 20, 23, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 17, 20, 24, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 17, 21, 15, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 17, 21, 16, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 17, 21, 17, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 17, 21, 18, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 17, 21, 19, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 17, 21, 20, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 17, 21, 21, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 17, 21, 22, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 17, 21, 23, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 17, 21, 24, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 17, 22, 15, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 17, 22, 16, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 17, 22, 17, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 17, 22, 18, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 17, 22, 19, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 17, 22, 20, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 17, 22, 21, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 17, 22, 22, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 17, 22, 23, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 17, 22, 24, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 17, 23, 15, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 17, 23, 16, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 17, 23, 17, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 17, 23, 18, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 17, 23, 19, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 17, 23, 20, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 17, 23, 21, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 17, 23, 22, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 17, 23, 23, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 17, 23, 24, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 17, 24, 15, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 17, 24, 16, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 17, 24, 17, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 17, 24, 18, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 17, 24, 19, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 17, 24, 20, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 17, 24, 21, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 17, 24, 22, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 17, 24, 23, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 17, 24, 24, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 18, 15, 15, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 18, 15, 16, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 18, 15, 17, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 18, 15, 18, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 18, 15, 19, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 18, 15, 20, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 18, 15, 21, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 18, 15, 22, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 18, 15, 23, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 18, 15, 24, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 18, 16, 15, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 18, 16, 16, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 18, 16, 17, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 18, 16, 18, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 18, 16, 19, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 18, 16, 20, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 18, 16, 21, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 18, 16, 22, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 18, 16, 23, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 18, 16, 24, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 18, 17, 15, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 18, 17, 16, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 18, 17, 17, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 18, 17, 18, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 18, 17, 19, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 18, 17, 20, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 18, 17, 21, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 18, 17, 22, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 18, 17, 23, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 18, 17, 24, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 18, 18, 15, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 18, 18, 16, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 18, 18, 17, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 18, 18, 18, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 18, 18, 19, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 18, 18, 20, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 18, 18, 21, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 18, 18, 22, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 18, 18, 23, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 18, 18, 24, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 18, 19, 15, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 18, 19, 16, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 18, 19, 17, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 18, 19, 18, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 18, 19, 19, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 18, 19, 20, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 18, 19, 21, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 18, 19, 22, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 18, 19, 23, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 18, 19, 24, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 18, 20, 15, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 18, 20, 16, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 18, 20, 17, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 18, 20, 18, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 18, 20, 19, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 18, 20, 20, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 18, 20, 21, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 18, 20, 22, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 18, 20, 23, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 18, 20, 24, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 18, 21, 15, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 18, 21, 16, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 18, 21, 17, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 18, 21, 18, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 18, 21, 19, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 18, 21, 20, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 18, 21, 21, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 18, 21, 22, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 18, 21, 23, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 18, 21, 24, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 18, 22, 15, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 18, 22, 16, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 18, 22, 17, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 18, 22, 18, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 18, 22, 19, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 18, 22, 20, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 18, 22, 21, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 18, 22, 22, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 18, 22, 23, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 18, 22, 24, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 18, 23, 15, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 18, 23, 16, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 18, 23, 17, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 18, 23, 18, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 18, 23, 19, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 18, 23, 20, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 18, 23, 21, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 18, 23, 22, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 18, 23, 23, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 18, 23, 24, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 18, 24, 15, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 18, 24, 16, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 18, 24, 17, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 18, 24, 18, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 18, 24, 19, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 18, 24, 20, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 18, 24, 21, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 18, 24, 22, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 18, 24, 23, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 18, 24, 24, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 19, 15, 15, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 19, 15, 16, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 19, 15, 17, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 19, 15, 18, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 19, 15, 19, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 19, 15, 20, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 19, 15, 21, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 19, 15, 22, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 19, 15, 23, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 19, 15, 24, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 19, 16, 15, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 19, 16, 16, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 19, 16, 17, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 19, 16, 18, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 19, 16, 19, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 19, 16, 20, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 19, 16, 21, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 19, 16, 22, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 19, 16, 23, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 19, 16, 24, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 19, 17, 15, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 19, 17, 16, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 19, 17, 17, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 19, 17, 18, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 19, 17, 19, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 19, 17, 20, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 19, 17, 21, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 19, 17, 22, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 19, 17, 23, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 19, 17, 24, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 19, 18, 15, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 19, 18, 16, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 19, 18, 17, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 19, 18, 18, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 19, 18, 19, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 19, 18, 20, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 19, 18, 21, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 19, 18, 22, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 19, 18, 23, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 19, 18, 24, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 19, 19, 15, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 19, 19, 16, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 19, 19, 17, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 19, 19, 18, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 19, 19, 19, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 19, 19, 20, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 19, 19, 21, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 19, 19, 22, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 19, 19, 23, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 19, 19, 24, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 19, 20, 15, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 19, 20, 16, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 19, 20, 17, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 19, 20, 18, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 19, 20, 19, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 19, 20, 20, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 19, 20, 21, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 19, 20, 22, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 19, 20, 23, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 19, 20, 24, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 19, 21, 15, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 19, 21, 16, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 19, 21, 17, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 19, 21, 18, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 19, 21, 19, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 19, 21, 20, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 19, 21, 21, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 19, 21, 22, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 19, 21, 23, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 19, 21, 24, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 19, 22, 15, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 19, 22, 16, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 19, 22, 17, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 19, 22, 18, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 19, 22, 19, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 19, 22, 20, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 19, 22, 21, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 19, 22, 22, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 19, 22, 23, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 19, 22, 24, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 19, 23, 15, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 19, 23, 16, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 19, 23, 17, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 19, 23, 18, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 19, 23, 19, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 19, 23, 20, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 19, 23, 21, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 19, 23, 22, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 19, 23, 23, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 19, 23, 24, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 19, 24, 15, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 19, 24, 16, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 19, 24, 17, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 19, 24, 18, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 19, 24, 19, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 19, 24, 20, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 19, 24, 21, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 19, 24, 22, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 19, 24, 23, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 19, 24, 24, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 20, 15, 15, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 20, 15, 16, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 20, 15, 17, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 20, 15, 18, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 20, 15, 19, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 20, 15, 20, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 20, 15, 21, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 20, 15, 22, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 20, 15, 23, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 20, 15, 24, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 20, 16, 15, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 20, 16, 16, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 20, 16, 17, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 20, 16, 18, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 20, 16, 19, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 20, 16, 20, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 20, 16, 21, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 20, 16, 22, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 20, 16, 23, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 20, 16, 24, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 20, 17, 15, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 20, 17, 16, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 20, 17, 17, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 20, 17, 18, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 20, 17, 19, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 20, 17, 20, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 20, 17, 21, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 20, 17, 22, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 20, 17, 23, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 20, 17, 24, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 20, 18, 15, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 20, 18, 16, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 20, 18, 17, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 20, 18, 18, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 20, 18, 19, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 20, 18, 20, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 20, 18, 21, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 20, 18, 22, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 20, 18, 23, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 20, 18, 24, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 20, 19, 15, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 20, 19, 16, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 20, 19, 17, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 20, 19, 18, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 20, 19, 19, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 20, 19, 20, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 20, 19, 21, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 20, 19, 22, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 20, 19, 23, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 20, 19, 24, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 20, 20, 15, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 20, 20, 16, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 20, 20, 17, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 20, 20, 18, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 20, 20, 19, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 20, 20, 20, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 20, 20, 21, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 20, 20, 22, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 20, 20, 23, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 20, 20, 24, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 20, 21, 15, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 20, 21, 16, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 20, 21, 17, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 20, 21, 18, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 20, 21, 19, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 20, 21, 20, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 20, 21, 21, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 20, 21, 22, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 20, 21, 23, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 20, 21, 24, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 20, 22, 15, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 20, 22, 16, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 20, 22, 17, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 20, 22, 18, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 20, 22, 19, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 20, 22, 20, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 20, 22, 21, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 20, 22, 22, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 20, 22, 23, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 20, 22, 24, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 20, 23, 15, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 20, 23, 16, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 20, 23, 17, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 20, 23, 18, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 20, 23, 19, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 20, 23, 20, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 20, 23, 21, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 20, 23, 22, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 20, 23, 23, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 20, 23, 24, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 20, 24, 15, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 20, 24, 16, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 20, 24, 17, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 20, 24, 18, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 20, 24, 19, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 20, 24, 20, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 20, 24, 21, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 20, 24, 22, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 20, 24, 23, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 20, 24, 24, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 21, 15, 15, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 21, 15, 16, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 21, 15, 17, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 21, 15, 18, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 21, 15, 19, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 21, 15, 20, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 21, 15, 21, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 21, 15, 22, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 21, 15, 23, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 21, 15, 24, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 21, 16, 15, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 21, 16, 16, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 21, 16, 17, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 21, 16, 18, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 21, 16, 19, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 21, 16, 20, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 21, 16, 21, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 21, 16, 22, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 21, 16, 23, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 21, 16, 24, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 21, 17, 15, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 21, 17, 16, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 21, 17, 17, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 21, 17, 18, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 21, 17, 19, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 21, 17, 20, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 21, 17, 21, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 21, 17, 22, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 21, 17, 23, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 21, 17, 24, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 21, 18, 15, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 21, 18, 16, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 21, 18, 17, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 21, 18, 18, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 21, 18, 19, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 21, 18, 20, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 21, 18, 21, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 21, 18, 22, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 21, 18, 23, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 21, 18, 24, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 21, 19, 15, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 21, 19, 16, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 21, 19, 17, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 21, 19, 18, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 21, 19, 19, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 21, 19, 20, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 21, 19, 21, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 21, 19, 22, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 21, 19, 23, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 21, 19, 24, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 21, 20, 15, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 21, 20, 16, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 21, 20, 17, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 21, 20, 18, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 21, 20, 19, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 21, 20, 20, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 21, 20, 21, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 21, 20, 22, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 21, 20, 23, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 21, 20, 24, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 21, 21, 15, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 21, 21, 16, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 21, 21, 17, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 21, 21, 18, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 21, 21, 19, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 21, 21, 20, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 21, 21, 21, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 21, 21, 22, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 21, 21, 23, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 21, 21, 24, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 21, 22, 15, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 21, 22, 16, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 21, 22, 17, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 21, 22, 18, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 21, 22, 19, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 21, 22, 20, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 21, 22, 21, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 21, 22, 22, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 21, 22, 23, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 21, 22, 24, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 21, 23, 15, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 21, 23, 16, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 21, 23, 17, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 21, 23, 18, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 21, 23, 19, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 21, 23, 20, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 21, 23, 21, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 21, 23, 22, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 21, 23, 23, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 3002, 220, 21, 23, 24, 25, 279, 3618, 5435, 220, 22, 6105, 30494, 13, 198, 3002, 220, 21, 24, 15, 25, 279, 3618, 5435, 220, 23, 6105, 30494, 13, 198, 3002, 220, 21, 24, 16, 25, 279, 3618, 5435, 220, 24, 6105, 30494, 13, 198, 3002, 220, 21, 24, 17, 25, 279, 3618, 5435, 220, 16, 15, 6105, 30494, 13, 198, 3002, 220, 21, 24, 18, 25, 279, 3618, 5435, 220, 16, 16, 6105, 30494, 13, 198, 3002, 220, 21, 24, 19, 25, 279, 3618, 5435, 220, 16, 17, 6105, 30494, 13, 198, 3002, 220, 21, 24, 20, 25, 279, 3618, 5435, 220, 16, 18, 6105, 30494, 13, 198, 3002, 220, 21, 24, 21, 25, 279, 3618, 5435, 220, 16, 19, 6105, 30494, 13, 198, 3002, 220, 21, 24, 22, 25, 279, 3618, 5435, 220, 16, 20, 6105, 30494, 13, 198, 3002, 220, 21, 24, 23, 25, 279, 3618, 5435, 220, 16, 21, 6105, 30494, 13, 198, 3002, 220, 21, 24, 24, 25, 279, 3618, 5435, 220, 16, 22, 6105, 30494, 13, 198, 3002, 220, 22, 15, 15, 25, 279, 3618, 5435, 220, 16, 23, 6105, 30494, 13, 198, 3002, 220, 22, 15, 16, 25, 279, 3618, 5435, 220, 16, 24, 6105, 30494, 13, 198, 3002, 220, 22, 15, 17, 25, 279, 3618, 5435, 220, 17, 15, 6105, 30494, 13, 198, 3002, 220, 22, 15, 18, 25, 279, 3618, 5435, 220, 17, 16, 6105, 30494, 13, 198, 3002, 220, 22, 15, 19, 25, 279, 3618, 5435, 220, 17, 17, 6105, 30494, 13, 198, 3002, 220, 22, 15, 20, 25, 279, 3618, 5435, 220, 17, 18, 6105, 30494, 13, 198, 3002, 220, 22, 15, 21, 25, 279, 3618, 5435, 220, 17, 19, 6105, 30494, 13, 198, 3002, 220, 22, 15, 22, 25, 279, 3618, 5435, 220, 17, 20, 6105, 30494, 13, 198, 3002, 220, 22, 15, 23, 25, 279, 3618, 5435, 220, 17, 21, 6105, 30494, 13, 198, 3002, 220, 22, 15, 24, 25, 279, 3618, 5435, 220, 17, 22, 6105, 30494, 13, 198, 3002, 220, 22, 16, 15, 25, 279, 3618, 5435, 220, 17, 23, 6105, 30494, 13, 198, 3002, 220, 22, 16, 16, 25, 279, 3618, 5435, 220, 17, 24, 6105, 30494, 13, 198, 3002, 220, 22, 16, 17, 25, 279, 3618, 5435, 220, 18, 15, 6105, 30494, 13, 198, 3002, 220, 22, 16, 18, 25, 279, 3618, 5435, 220, 15, 6105, 30494, 13, 198, 3002, 220, 22, 16, 19, 25, 279, 3618, 5435, 220, 16, 6105, 30494, 13, 198, 3002, 220, 22, 16, 20, 25, 279, 3618, 5435, 220, 17, 6105, 30494, 13, 198, 3002, 220, 22, 16, 21, 25, 279, 3618, 5435, 220, 18, 6105, 30494, 13, 198, 3002, 220, 22, 16, 22, 25, 279, 3618, 5435, 220, 19, 6105, 30494, 13, 198, 3002, 220, 22, 16, 23, 25, 279, 3618, 5435, 220, 20, 6105, 30494, 13, 198, 3002, 220, 22, 16, 24, 25, 279, 3618, 5435, 220, 21, 6105, 30494, 13, 198, 72240, 449, 11767, 310, 10724, 279, 2702, 1324, 314, 6105, 30494, 13, 248046, 198, 248045, 74455, 198, 248068, 198], "text": "Row 0: the box contains 0 blue counters.\nRow 1: the box contains 1 blue counters.\nRow 2: the box contains 2 blue counters.\nRow 3: the box contains 3 blue counters.\nRow 4: the box contains 4 blue counters.\nRow 5: the box contains 5 blue counters.\nRow 6: the box contains 6 blue counters.\nRow 7: the box contains 7 blue counters.\nRow 8: the box contains 8 blue counters.\nRow 9: the box contains 9 blue counters.\nRow 10: the box contains 10 blue counters.\nRow 11: the box contains 11 blue counters.\nRow 12: the box contains 12 blue counters.\nRow 13: the box contains 13 blue counters.\nRow 14: the box contains 14 blue counters.\nRow 15: the box contains 15 blue counters.\nRow 16: the box contains 16 blue counters.\nRow 17: the box contains 17 blue counters.\nRow 18: the box contains 18 blue counters.\nRow 19: the box contains 19 blue counters.\nRow 20: the box contains 20 blue counters.\nRow 21: the box contains 21 blue counters.\nRow 22: the box contains 22 blue counters.\nRow 23: the box contains 23 blue counters.\nRow 24: the box contains 24 blue counters.\nRow 25: the box contains 25 blue counters.\nRow 26: the box contains 26 blue counters.\nRow 27: the box contains 27 blue counters.\nRow 28: the box contains 28 blue counters.\nRow 29: the box contains 29 blue counters.\nRow 30: the box contains 30 blue counters.\nRow 31: the box contains 0 blue counters.\nRow 32: the box contains 1 blue counters.\nRow 33: the box contains 2 blue counters.\nRow 34: the box contains 3 blue counters.\nRow 35: the box contains 4 blue counters.\nRow 36: the box contains 5 blue counters.\nRow 37: the box contains 6 blue counters.\nRow 38: the box contains 7 blue counters.\nRow 39: the box contains 8 blue counters.\nRow 40: the box contains 9 blue counters.\nRow 41: the box contains 10 blue counters.\nRow 42: the box contains 11 blue counters.\nRow 43: the box contains 12 blue counters.\nRow 44: the box contains 13 blue counters.\nRow 45: the box contains 14 blue counters.\nRow 46: the box contains 15 blue counters.\nRow 47: the box contains 16 blue counters.\nRow 48: the box contains 17 blue counters.\nRow 49: the box contains 18 blue counters.\nRow 50: the box contains 19 blue counters.\nRow 51: the box contains 20 blue counters.\nRow 52: the box contains 21 blue counters.\nRow 53: the box contains 22 blue counters.\nRow 54: the box contains 23 blue counters.\nRow 55: the box contains 24 blue counters.\nRow 56: the box contains 25 blue counters.\nRow 57: the box contains 26 blue counters.\nRow 58: the box contains 27 blue counters.\nRow 59: the box contains 28 blue counters.\nRow 60: the box contains 29 blue counters.\nRow 61: the box contains 30 blue counters.\nRow 62: the box contains 0 blue counters.\nRow 63: the box contains 1 blue counters.\nRow 64: the box contains 2 blue counters.\nRow 65: the box contains 3 blue counters.\nRow 66: the box contains 4 blue counters.\nRow 67: the box contains 5 blue counters.\nRow 68: the box contains 6 blue counters.\nRow 69: the box contains 7 blue counters.\nRow 70: the box contains 8 blue counters.\nRow 71: the box contains 9 blue counters.\nRow 72: the box contains 10 blue counters.\nRow 73: the box contains 11 blue counters.\nRow 74: the box contains 12 blue counters.\nRow 75: the box contains 13 blue counters.\nRow 76: the box contains 14 blue counters.\nRow 77: the box contains 15 blue counters.\nRow 78: the box contains 16 blue counters.\nRow 79: the box contains 17 blue counters.\nRow 80: the box contains 18 blue counters.\nRow 81: the box contains 19 blue counters.\nRow 82: the box contains 20 blue counters.\nRow 83: the box contains 21 blue counters.\nRow 84: the box contains 22 blue counters.\nRow 85: the box contains 23 blue counters.\nRow 86: the box contains 24 blue counters.\nRow 87: the box contains 25 blue counters.\nRow 88: the box contains 26 blue counters.\nRow 89: the box contains 27 blue counters.\nRow 90: the box contains 28 blue counters.\nRow 91: the box contains 29 blue counters.\nRow 92: the box contains 30 blue counters.\nRow 93: the box contains 0 blue counters.\nRow 94: the box contains 1 blue counters.\nRow 95: the box contains 2 blue counters.\nRow 96: the box contains 3 blue counters.\nRow 97: the box contains 4 blue counters.\nRow 98: the box contains 5 blue counters.\nRow 99: the box contains 6 blue counters.\nRow 100: the box contains 7 blue counters.\nRow 101: the box contains 8 blue counters.\nRow 102: the box contains 9 blue counters.\nRow 103: the box contains 10 blue counters.\nRow 104: the box contains 11 blue counters.\nRow 105: the box contains 12 blue counters.\nRow 106: the box contains 13 blue counters.\nRow 107: the box contains 14 blue counters.\nRow 108: the box contains 15 blue counters.\nRow 109: the box contains 16 blue counters.\nRow 110: the box contains 17 blue counters.\nRow 111: the box contains 18 blue counters.\nRow 112: the box contains 19 blue counters.\nRow 113: the box contains 20 blue counters.\nRow 114: the box contains 21 blue counters.\nRow 115: the box contains 22 blue counters.\nRow 116: the box contains 23 blue counters.\nRow 117: the box contains 24 blue counters.\nRow 118: the box contains 25 blue counters.\nRow 119: the box contains 26 blue counters.\nRow 120: the box contains 27 blue counters.\nRow 121: the box contains 28 blue counters.\nRow 122: the box contains 29 blue counters.\nRow 123: the box contains 30 blue counters.\nRow 124: the box contains 0 blue counters.\nRow 125: the box contains 1 blue counters.\nRow 126: the box contains 2 blue counters.\nRow 127: the box contains 3 blue counters.\nRow 128: the box contains 4 blue counters.\nRow 129: the box contains 5 blue counters.\nRow 130: the box contains 6 blue counters.\nRow 131: the box contains 7 blue counters.\nRow 132: the box contains 8 blue counters.\nRow 133: the box contains 9 blue counters.\nRow 134: the box contains 10 blue counters.\nRow 135: the box contains 11 blue counters.\nRow 136: the box contains 12 blue counters.\nRow 137: the box contains 13 blue counters.\nRow 138: the box contains 14 blue counters.\nRow 139: the box contains 15 blue counters.\nRow 140: the box contains 16 blue counters.\nRow 141: the box contains 17 blue counters.\nRow 142: the box contains 18 blue counters.\nRow 143: the box contains 19 blue counters.\nRow 144: the box contains 20 blue counters.\nRow 145: the box contains 21 blue counters.\nRow 146: the box contains 22 blue counters.\nRow 147: the box contains 23 blue counters.\nRow 148: the box contains 24 blue counters.\nRow 149: the box contains 25 blue counters.\nRow 150: the box contains 26 blue counters.\nRow 151: the box contains 27 blue counters.\nRow 152: the box contains 28 blue counters.\nRow 153: the box contains 29 blue counters.\nRow 154: the box contains 30 blue counters.\nRow 155: the box contains 0 blue counters.\nRow 156: the box contains 1 blue counters.\nRow 157: the box contains 2 blue counters.\nRow 158: the box contains 3 blue counters.\nRow 159: the box contains 4 blue counters.\nRow 160: the box contains 5 blue counters.\nRow 161: the box contains 6 blue counters.\nRow 162: the box contains 7 blue counters.\nRow 163: the box contains 8 blue counters.\nRow 164: the box contains 9 blue counters.\nRow 165: the box contains 10 blue counters.\nRow 166: the box contains 11 blue counters.\nRow 167: the box contains 12 blue counters.\nRow 168: the box contains 13 blue counters.\nRow 169: the box contains 14 blue counters.\nRow 170: the box contains 15 blue counters.\nRow 171: the box contains 16 blue counters.\nRow 172: the box contains 17 blue counters.\nRow 173: the box contains 18 blue counters.\nRow 174: the box contains 19 blue counters.\nRow 175: the box contains 20 blue counters.\nRow 176: the box contains 21 blue counters.\nRow 177: the box contains 22 blue counters.\nRow 178: the box contains 23 blue counters.\nRow 179: the box contains 24 blue counters.\nRow 180: the box contains 25 blue counters.\nRow 181: the box contains 26 blue counters.\nRow 182: the box contains 27 blue counters.\nRow 183: the box contains 28 blue counters.\nRow 184: the box contains 29 blue counters.\nRow 185: the box contains 30 blue counters.\nRow 186: the box contains 0 blue counters.\nRow 187: the box contains 1 blue counters.\nRow 188: the box contains 2 blue counters.\nRow 189: the box contains 3 blue counters.\nRow 190: the box contains 4 blue counters.\nRow 191: the box contains 5 blue counters.\nRow 192: the box contains 6 blue counters.\nRow 193: the box contains 7 blue counters.\nRow 194: the box contains 8 blue counters.\nRow 195: the box contains 9 blue counters.\nRow 196: the box contains 10 blue counters.\nRow 197: the box contains 11 blue counters.\nRow 198: the box contains 12 blue counters.\nRow 199: the box contains 13 blue counters.\nRow 200: the box contains 14 blue counters.\nRow 201: the box contains 15 blue counters.\nRow 202: the box contains 16 blue counters.\nRow 203: the box contains 17 blue counters.\nRow 204: the box contains 18 blue counters.\nRow 205: the box contains 19 blue counters.\nRow 206: the box contains 20 blue counters.\nRow 207: the box contains 21 blue counters.\nRow 208: the box contains 22 blue counters.\nRow 209: the box contains 23 blue counters.\nRow 210: the box contains 24 blue counters.\nRow 211: the box contains 25 blue counters.\nRow 212: the box contains 26 blue counters.\nRow 213: the box contains 27 blue counters.\nRow 214: the box contains 28 blue counters.\nRow 215: the box contains 29 blue counters.\nRow 216: the box contains 30 blue counters.\nRow 217: the box contains 0 blue counters.\nRow 218: the box contains 1 blue counters.\nRow 219: the box contains 2 blue counters.\nRow 220: the box contains 3 blue counters.\nRow 221: the box contains 4 blue counters.\nRow 222: the box contains 5 blue counters.\nRow 223: the box contains 6 blue counters.\nRow 224: the box contains 7 blue counters.\nRow 225: the box contains 8 blue counters.\nRow 226: the box contains 9 blue counters.\nRow 227: the box contains 10 blue counters.\nRow 228: the box contains 11 blue counters.\nRow 229: the box contains 12 blue counters.\nRow 230: the box contains 13 blue counters.\nRow 231: the box contains 14 blue counters.\nRow 232: the box contains 15 blue counters.\nRow 233: the box contains 16 blue counters.\nRow 234: the box contains 17 blue counters.\nRow 235: the box contains 18 blue counters.\nRow 236: the box contains 19 blue counters.\nRow 237: the box contains 20 blue counters.\nRow 238: the box contains 21 blue counters.\nRow 239: the box contains 22 blue counters.\nRow 240: the box contains 23 blue counters.\nRow 241: the box contains 24 blue counters.\nRow 242: the box contains 25 blue counters.\nRow 243: the box contains 26 blue counters.\nRow 244: the box contains 27 blue counters.\nRow 245: the box contains 28 blue counters.\nRow 246: the box contains 29 blue counters.\nRow 247: the box contains 30 blue counters.\nRow 248: the box contains 0 blue counters.\nRow 249: the box contains 1 blue counters.\nRow 250: the box contains 2 blue counters.\nRow 251: the box contains 3 blue counters.\nRow 252: the box contains 4 blue counters.\nRow 253: the box contains 5 blue counters.\nRow 254: the box contains 6 blue counters.\nRow 255: the box contains 7 blue counters.\nRow 256: the box contains 8 blue counters.\nRow 257: the box contains 9 blue counters.\nRow 258: the box contains 10 blue counters.\nRow 259: the box contains 11 blue counters.\nRow 260: the box contains 12 blue counters.\nRow 261: the box contains 13 blue counters.\nRow 262: the box contains 14 blue counters.\nRow 263: the box contains 15 blue counters.\nRow 264: the box contains 16 blue counters.\nRow 265: the box contains 17 blue counters.\nRow 266: the box contains 18 blue counters.\nRow 267: the box contains 19 blue counters.\nRow 268: the box contains 20 blue counters.\nRow 269: the box contains 21 blue counters.\nRow 270: the box contains 22 blue counters.\nRow 271: the box contains 23 blue counters.\nRow 272: the box contains 24 blue counters.\nRow 273: the box contains 25 blue counters.\nRow 274: the box contains 26 blue counters.\nRow 275: the box contains 27 blue counters.\nRow 276: the box contains 28 blue counters.\nRow 277: the box contains 29 blue counters.\nRow 278: the box contains 30 blue counters.\nRow 279: the box contains 0 blue counters.\nRow 280: the box contains 1 blue counters.\nRow 281: the box contains 2 blue counters.\nRow 282: the box contains 3 blue counters.\nRow 283: the box contains 4 blue counters.\nRow 284: the box contains 5 blue counters.\nRow 285: the box contains 6 blue counters.\nRow 286: the box contains 7 blue counters.\nRow 287: the box contains 8 blue counters.\nRow 288: the box contains 9 blue counters.\nRow 289: the box contains 10 blue counters.\nRow 290: the box contains 11 blue counters.\nRow 291: the box contains 12 blue counters.\nRow 292: the box contains 13 blue counters.\nRow 293: the box contains 14 blue counters.\nRow 294: the box contains 15 blue counters.\nRow 295: the box contains 16 blue counters.\nRow 296: the box contains 17 blue counters.\nRow 297: the box contains 18 blue counters.\nRow 298: the box contains 19 blue counters.\nRow 299: the box contains 20 blue counters.\nRow 300: the box contains 21 blue counters.\nRow 301: the box contains 22 blue counters.\nRow 302: the box contains 23 blue counters.\nRow 303: the box contains 24 blue counters.\nRow 304: the box contains 25 blue counters.\nRow 305: the box contains 26 blue counters.\nRow 306: the box contains 27 blue counters.\nRow 307: the box contains 28 blue counters.\nRow 308: the box contains 29 blue counters.\nRow 309: the box contains 30 blue counters.\nRow 310: the box contains 0 blue counters.\nRow 311: the box contains 1 blue counters.\nRow 312: the box contains 2 blue counters.\nRow 313: the box contains 3 blue counters.\nRow 314: the box contains 4 blue counters.\nRow 315: the box contains 5 blue counters.\nRow 316: the box contains 6 blue counters.\nRow 317: the box contains 7 blue counters.\nRow 318: the box contains 8 blue counters.\nRow 319: the box contains 9 blue counters.\nRow 320: the box contains 10 blue counters.\nRow 321: the box contains 11 blue counters.\nRow 322: the box contains 12 blue counters.\nRow 323: the box contains 13 blue counters.\nRow 324: the box contains 14 blue counters.\nRow 325: the box contains 15 blue counters.\nRow 326: the box contains 16 blue counters.\nRow 327: the box contains 17 blue counters.\nRow 328: the box contains 18 blue counters.\nRow 329: the box contains 19 blue counters.\nRow 330: the box contains 20 blue counters.\nRow 331: the box contains 21 blue counters.\nRow 332: the box contains 22 blue counters.\nRow 333: the box contains 23 blue counters.\nRow 334: the box contains 24 blue counters.\nRow 335: the box contains 25 blue counters.\nRow 336: the box contains 26 blue counters.\nRow 337: the box contains 27 blue counters.\nRow 338: the box contains 28 blue counters.\nRow 339: the box contains 29 blue counters.\nRow 340: the box contains 30 blue counters.\nRow 341: the box contains 0 blue counters.\nRow 342: the box contains 1 blue counters.\nRow 343: the box contains 2 blue counters.\nRow 344: the box contains 3 blue counters.\nRow 345: the box contains 4 blue counters.\nRow 346: the box contains 5 blue counters.\nRow 347: the box contains 6 blue counters.\nRow 348: the box contains 7 blue counters.\nRow 349: the box contains 8 blue counters.\nRow 350: the box contains 9 blue counters.\nRow 351: the box contains 10 blue counters.\nRow 352: the box contains 11 blue counters.\nRow 353: the box contains 12 blue counters.\nRow 354: the box contains 13 blue counters.\nRow 355: the box contains 14 blue counters.\nRow 356: the box contains 15 blue counters.\nRow 357: the box contains 16 blue counters.\nRow 358: the box contains 17 blue counters.\nRow 359: the box contains 18 blue counters.\nRow 360: the box contains 19 blue counters.\nRow 361: the box contains 20 blue counters.\nRow 362: the box contains 21 blue counters.\nRow 363: the box contains 22 blue counters.\nRow 364: the box contains 23 blue counters.\nRow 365: the box contains 24 blue counters.\nRow 366: the box contains 25 blue counters.\nRow 367: the box contains 26 blue counters.\nRow 368: the box contains 27 blue counters.\nRow 369: the box contains 28 blue counters.\nRow 370: the box contains 29 blue counters.\nRow 371: the box contains 30 blue counters.\nRow 372: the box contains 0 blue counters.\nRow 373: the box contains 1 blue counters.\nRow 374: the box contains 2 blue counters.\nRow 375: the box contains 3 blue counters.\nRow 376: the box contains 4 blue counters.\nRow 377: the box contains 5 blue counters.\nRow 378: the box contains 6 blue counters.\nRow 379: the box contains 7 blue counters.\nRow 380: the box contains 8 blue counters.\nRow 381: the box contains 9 blue counters.\nRow 382: the box contains 10 blue counters.\nRow 383: the box contains 11 blue counters.\nRow 384: the box contains 12 blue counters.\nRow 385: the box contains 13 blue counters.\nRow 386: the box contains 14 blue counters.\nRow 387: the box contains 15 blue counters.\nRow 388: the box contains 16 blue counters.\nRow 389: the box contains 17 blue counters.\nRow 390: the box contains 18 blue counters.\nRow 391: the box contains 19 blue counters.\nRow 392: the box contains 20 blue counters.\nRow 393: the box contains 21 blue counters.\nRow 394: the box contains 22 blue counters.\nRow 395: the box contains 23 blue counters.\nRow 396: the box contains 24 blue counters.\nRow 397: the box contains 25 blue counters.\nRow 398: the box contains 26 blue counters.\nRow 399: the box contains 27 blue counters.\nRow 400: the box contains 28 blue counters.\nRow 401: the box contains 29 blue counters.\nRow 402: the box contains 30 blue counters.\nRow 403: the box contains 0 blue counters.\nRow 404: the box contains 1 blue counters.\nRow 405: the box contains 2 blue counters.\nRow 406: the box contains 3 blue counters.\nRow 407: the box contains 4 blue counters.\nRow 408: the box contains 5 blue counters.\nRow 409: the box contains 6 blue counters.\nRow 410: the box contains 7 blue counters.\nRow 411: the box contains 8 blue counters.\nRow 412: the box contains 9 blue counters.\nRow 413: the box contains 10 blue counters.\nRow 414: the box contains 11 blue counters.\nRow 415: the box contains 12 blue counters.\nRow 416: the box contains 13 blue counters.\nRow 417: the box contains 14 blue counters.\nRow 418: the box contains 15 blue counters.\nRow 419: the box contains 16 blue counters.\nRow 420: the box contains 17 blue counters.\nRow 421: the box contains 18 blue counters.\nRow 422: the box contains 19 blue counters.\nRow 423: the box contains 20 blue counters.\nRow 424: the box contains 21 blue counters.\nRow 425: the box contains 22 blue counters.\nRow 426: the box contains 23 blue counters.\nRow 427: the box contains 24 blue counters.\nRow 428: the box contains 25 blue counters.\nRow 429: the box contains 26 blue counters.\nRow 430: the box contains 27 blue counters.\nRow 431: the box contains 28 blue counters.\nRow 432: the box contains 29 blue counters.\nRow 433: the box contains 30 blue counters.\nRow 434: the box contains 0 blue counters.\nRow 435: the box contains 1 blue counters.\nRow 436: the box contains 2 blue counters.\nRow 437: the box contains 3 blue counters.\nRow 438: the box contains 4 blue counters.\nRow 439: the box contains 5 blue counters.\nRow 440: the box contains 6 blue counters.\nRow 441: the box contains 7 blue counters.\nRow 442: the box contains 8 blue counters.\nRow 443: the box contains 9 blue counters.\nRow 444: the box contains 10 blue counters.\nRow 445: the box contains 11 blue counters.\nRow 446: the box contains 12 blue counters.\nRow 447: the box contains 13 blue counters.\nRow 448: the box contains 14 blue counters.\nRow 449: the box contains 15 blue counters.\nRow 450: the box contains 16 blue counters.\nRow 451: the box contains 17 blue counters.\nRow 452: the box contains 18 blue counters.\nRow 453: the box contains 19 blue counters.\nRow 454: the box contains 20 blue counters.\nRow 455: the box contains 21 blue counters.\nRow 456: the box contains 22 blue counters.\nRow 457: the box contains 23 blue counters.\nRow 458: the box contains 24 blue counters.\nRow 459: the box contains 25 blue counters.\nRow 460: the box contains 26 blue counters.\nRow 461: the box contains 27 blue counters.\nRow 462: the box contains 28 blue counters.\nRow 463: the box contains 29 blue counters.\nRow 464: the box contains 30 blue counters.\nRow 465: the box contains 0 blue counters.\nRow 466: the box contains 1 blue counters.\nRow 467: the box contains 2 blue counters.\nRow 468: the box contains 3 blue counters.\nRow 469: the box contains 4 blue counters.\nRow 470: the box contains 5 blue counters.\nRow 471: the box contains 6 blue counters.\nRow 472: the box contains 7 blue counters.\nRow 473: the box contains 8 blue counters.\nRow 474: the box contains 9 blue counters.\nRow 475: the box contains 10 blue counters.\nRow 476: the box contains 11 blue counters.\nRow 477: the box contains 12 blue counters.\nRow 478: the box contains 13 blue counters.\nRow 479: the box contains 14 blue counters.\nRow 480: the box contains 15 blue counters.\nRow 481: the box contains 16 blue counters.\nRow 482: the box contains 17 blue counters.\nRow 483: the box contains 18 blue counters.\nRow 484: the box contains 19 blue counters.\nRow 485: the box contains 20 blue counters.\nRow 486: the box contains 21 blue counters.\nRow 487: the box contains 22 blue counters.\nRow 488: the box contains 23 blue counters.\nRow 489: the box contains 24 blue counters.\nRow 490: the box contains 25 blue counters.\nRow 491: the box contains 26 blue counters.\nRow 492: the box contains 27 blue counters.\nRow 493: the box contains 28 blue counters.\nRow 494: the box contains 29 blue counters.\nRow 495: the box contains 30 blue counters.\nRow 496: the box contains 0 blue counters.\nRow 497: the box contains 1 blue counters.\nRow 498: the box contains 2 blue counters.\nRow 499: the box contains 3 blue counters.\nRow 500: the box contains 4 blue counters.\nRow 501: the box contains 5 blue counters.\nRow 502: the box contains 6 blue counters.\nRow 503: the box contains 7 blue counters.\nRow 504: the box contains 8 blue counters.\nRow 505: the box contains 9 blue counters.\nRow 506: the box contains 10 blue counters.\nRow 507: the box contains 11 blue counters.\nRow 508: the box contains 12 blue counters.\nRow 509: the box contains 13 blue counters.\nRow 510: the box contains 14 blue counters.\nRow 511: the box contains 15 blue counters.\nRow 512: the box contains 16 blue counters.\nRow 513: the box contains 17 blue counters.\nRow 514: the box contains 18 blue counters.\nRow 515: the box contains 19 blue counters.\nRow 516: the box contains 20 blue counters.\nRow 517: the box contains 21 blue counters.\nRow 518: the box contains 22 blue counters.\nRow 519: the box contains 23 blue counters.\nRow 520: the box contains 24 blue counters.\nRow 521: the box contains 25 blue counters.\nRow 522: the box contains 26 blue counters.\nRow 523: the box contains 27 blue counters.\nRow 524: the box contains 28 blue counters.\nRow 525: the box contains 29 blue counters.\nRow 526: the box contains 30 blue counters.\nRow 527: the box contains 0 blue counters.\nRow 528: the box contains 1 blue counters.\nRow 529: the box contains 2 blue counters.\nRow 530: the box contains 3 blue counters.\nRow 531: the box contains 4 blue counters.\nRow 532: the box contains 5 blue counters.\nRow 533: the box contains 6 blue counters.\nRow 534: the box contains 7 blue counters.\nRow 535: the box contains 8 blue counters.\nRow 536: the box contains 9 blue counters.\nRow 537: the box contains 10 blue counters.\nRow 538: the box contains 11 blue counters.\nRow 539: the box contains 12 blue counters.\nRow 540: the box contains 13 blue counters.\nRow 541: the box contains 14 blue counters.\nRow 542: the box contains 15 blue counters.\nRow 543: the box contains 16 blue counters.\nRow 544: the box contains 17 blue counters.\nRow 545: the box contains 18 blue counters.\nRow 546: the box contains 19 blue counters.\nRow 547: the box contains 20 blue counters.\nRow 548: the box contains 21 blue counters.\nRow 549: the box contains 22 blue counters.\nRow 550: the box contains 23 blue counters.\nRow 551: the box contains 24 blue counters.\nRow 552: the box contains 25 blue counters.\nRow 553: the box contains 26 blue counters.\nRow 554: the box contains 27 blue counters.\nRow 555: the box contains 28 blue counters.\nRow 556: the box contains 29 blue counters.\nRow 557: the box contains 30 blue counters.\nRow 558: the box contains 0 blue counters.\nRow 559: the box contains 1 blue counters.\nRow 560: the box contains 2 blue counters.\nRow 561: the box contains 3 blue counters.\nRow 562: the box contains 4 blue counters.\nRow 563: the box contains 5 blue counters.\nRow 564: the box contains 6 blue counters.\nRow 565: the box contains 7 blue counters.\nRow 566: the box contains 8 blue counters.\nRow 567: the box contains 9 blue counters.\nRow 568: the box contains 10 blue counters.\nRow 569: the box contains 11 blue counters.\nRow 570: the box contains 12 blue counters.\nRow 571: the box contains 13 blue counters.\nRow 572: the box contains 14 blue counters.\nRow 573: the box contains 15 blue counters.\nRow 574: the box contains 16 blue counters.\nRow 575: the box contains 17 blue counters.\nRow 576: the box contains 18 blue counters.\nRow 577: the box contains 19 blue counters.\nRow 578: the box contains 20 blue counters.\nRow 579: the box contains 21 blue counters.\nRow 580: the box contains 22 blue counters.\nRow 581: the box contains 23 blue counters.\nRow 582: the box contains 24 blue counters.\nRow 583: the box contains 25 blue counters.\nRow 584: the box contains 26 blue counters.\nRow 585: the box contains 27 blue counters.\nRow 586: the box contains 28 blue counters.\nRow 587: the box contains 29 blue counters.\nRow 588: the box contains 30 blue counters.\nRow 589: the box contains 0 blue counters.\nRow 590: the box contains 1 blue counters.\nRow 591: the box contains 2 blue counters.\nRow 592: the box contains 3 blue counters.\nRow 593: the box contains 4 blue counters.\nRow 594: the box contains 5 blue counters.\nRow 595: the box contains 6 blue counters.\nRow 596: the box contains 7 blue counters.\nRow 597: the box contains 8 blue counters.\nRow 598: the box contains 9 blue counters.\nRow 599: the box contains 10 blue counters.\nRow 600: the box contains 11 blue counters.\nRow 601: the box contains 12 blue counters.\nRow 602: the box contains 13 blue counters.\nRow 603: the box contains 14 blue counters.\nRow 604: the box contains 15 blue counters.\nRow 605: the box contains 16 blue counters.\nRow 606: the box contains 17 blue counters.\nRow 607: the box contains 18 blue counters.\nRow 608: the box contains 19 blue counters.\nRow 609: the box contains 20 blue counters.\nRow 610: the box contains 21 blue counters.\nRow 611: the box contains 22 blue counters.\nRow 612: the box contains 23 blue counters.\nRow 613: the box contains 24 blue counters.\nRow 614: the box contains 25 blue counters.\nRow 615: the box contains 26 blue counters.\nRow 616: the box contains 27 blue counters.\nRow 617: the box contains 28 blue counters.\nRow 618: the box contains 29 blue counters.\nRow 619: the box contains 30 blue counters.\nRow 620: the box contains 0 blue counters.\nRow 621: the box contains 1 blue counters.\nRow 622: the box contains 2 blue counters.\nRow 623: the box contains 3 blue counters.\nRow 624: the box contains 4 blue counters.\nRow 625: the box contains 5 blue counters.\nRow 626: the box contains 6 blue counters.\nRow 627: the box contains 7 blue counters.\nRow 628: the box contains 8 blue counters.\nRow 629: the box contains 9 blue counters.\nRow 630: the box contains 10 blue counters.\nRow 631: the box contains 11 blue counters.\nRow 632: the box contains 12 blue counters.\nRow 633: the box contains 13 blue counters.\nRow 634: the box contains 14 blue counters.\nRow 635: the box contains 15 blue counters.\nRow 636: the box contains 16 blue counters.\nRow 637: the box contains 17 blue counters.\nRow 638: the box contains 18 blue counters.\nRow 639: the box contains 19 blue counters.\nRow 640: the box contains 20 blue counters.\nRow 641: the box contains 21 blue counters.\nRow 642: the box contains 22 blue counters.\nRow 643: the box contains 23 blue counters.\nRow 644: the box contains 24 blue counters.\nRow 645: the box contains 25 blue counters.\nRow 646: the box contains 26 blue counters.\nRow 647: the box contains 27 blue counters.\nRow 648: the box contains 28 blue counters.\nRow 649: the box contains 29 blue counters.\nRow 650: the box contains 30 blue counters.\nRow 651: the box contains 0 blue counters.\nRow 652: the box contains 1 blue counters.\nRow 653: the box contains 2 blue counters.\nRow 654: the box contains 3 blue counters.\nRow 655: the box contains 4 blue counters.\nRow 656: the box contains 5 blue counters.\nRow 657: the box contains 6 blue counters.\nRow 658: the box contains 7 blue counters.\nRow 659: the box contains 8 blue counters.\nRow 660: the box contains 9 blue counters.\nRow 661: the box contains 10 blue counters.\nRow 662: the box contains 11 blue counters.\nRow 663: the box contains 12 blue counters.\nRow 664: the box contains 13 blue counters.\nRow 665: the box contains 14 blue counters.\nRow 666: the box contains 15 blue counters.\nRow 667: the box contains 16 blue counters.\nRow 668: the box contains 17 blue counters.\nRow 669: the box contains 18 blue counters.\nRow 670: the box contains 19 blue counters.\nRow 671: the box contains 20 blue counters.\nRow 672: the box contains 21 blue counters.\nRow 673: the box contains 22 blue counters.\nRow 674: the box contains 23 blue counters.\nRow 675: the box contains 24 blue counters.\nRow 676: the box contains 25 blue counters.\nRow 677: the box contains 26 blue counters.\nRow 678: the box contains 27 blue counters.\nRow 679: the box contains 28 blue counters.\nRow 680: the box contains 29 blue counters.\nRow 681: the box contains 30 blue counters.\nRow 682: the box contains 0 blue counters.\nRow 683: the box contains 1 blue counters.\nRow 684: the box contains 2 blue counters.\nRow 685: the box contains 3 blue counters.\nRow 686: the box contains 4 blue counters.\nRow 687: the box contains 5 blue counters.\nRow 688: the box contains 6 blue counters.\nRow 689: the box contains 7 blue counters.\nRow 690: the box contains 8 blue counters.\nRow 691: the box contains 9 blue counters.\nRow 692: the box contains 10 blue counters.\nRow 693: the box contains 11 blue counters.\nRow 694: the box contains 12 blue counters.\nRow 695: the box contains 13 blue counters.\nRow 696: the box contains 14 blue counters.\nRow 697: the box contains 15 blue counters.\nRow 698: the box contains 16 blue counters.\nRow 699: the box contains 17 blue counters.\nRow 700: the box contains 18 blue counters.\nRow 701: the box contains 19 blue counters.\nRow 702: the box contains 20 blue counters.\nRow 703: the box contains 21 blue counters.\nRow 704: the box contains 22 blue counters.\nRow 705: the box contains 23 blue counters.\nRow 706: the box contains 24 blue counters.\nRow 707: the box contains 25 blue counters.\nRow 708: the box contains 26 blue counters.\nRow 709: the box contains 27 blue counters.\nRow 710: the box contains 28 blue counters.\nRow 711: the box contains 29 blue counters.\nRow 712: the box contains 30 blue counters.\nRow 713: the box contains 0 blue counters.\nRow 714: the box contains 1 blue counters.\nRow 715: the box contains 2 blue counters.\nRow 716: the box contains 3 blue counters.\nRow 717: the box contains 4 blue counters.\nRow 718: the box contains 5 blue counters.\nRow 719: the box contains 6 blue counters.\nDescribe an algorithm to calculate the total number of blue counters.", "thinking": true} diff --git a/docs/receipts/flashnext-mtp-chain-2026-10-10/prompts/sparse.json b/docs/receipts/flashnext-mtp-chain-2026-10-10/prompts/sparse.json new file mode 100644 index 0000000000..65b13239d4 --- /dev/null +++ b/docs/receipts/flashnext-mtp-chain-2026-10-10/prompts/sparse.json @@ -0,0 +1 @@ +{"prompt": [248045, 846, 198, 1192, 220, 15, 25, 869, 220, 15, 13, 198, 1192, 220, 16, 25, 869, 220, 16, 13, 198, 1192, 220, 17, 25, 869, 220, 17, 13, 198, 1192, 220, 18, 25, 869, 220, 18, 13, 198, 1192, 220, 19, 25, 869, 220, 19, 13, 198, 1192, 220, 20, 25, 869, 220, 20, 13, 198, 1192, 220, 21, 25, 869, 220, 21, 13, 198, 1192, 220, 22, 25, 869, 220, 22, 13, 198, 1192, 220, 23, 25, 869, 220, 23, 13, 198, 1192, 220, 24, 25, 869, 220, 24, 13, 198, 1192, 220, 16, 15, 25, 869, 220, 16, 15, 13, 198, 1192, 220, 16, 16, 25, 869, 220, 16, 16, 13, 198, 1192, 220, 16, 17, 25, 869, 220, 16, 17, 13, 198, 1192, 220, 16, 18, 25, 869, 220, 16, 18, 13, 198, 1192, 220, 16, 19, 25, 869, 220, 16, 19, 13, 198, 1192, 220, 16, 20, 25, 869, 220, 16, 20, 13, 198, 1192, 220, 16, 21, 25, 869, 220, 16, 21, 13, 198, 1192, 220, 16, 22, 25, 869, 220, 16, 22, 13, 198, 1192, 220, 16, 23, 25, 869, 220, 16, 23, 13, 198, 1192, 220, 16, 24, 25, 869, 220, 16, 24, 13, 198, 1192, 220, 17, 15, 25, 869, 220, 17, 15, 13, 198, 1192, 220, 17, 16, 25, 869, 220, 17, 16, 13, 198, 1192, 220, 17, 17, 25, 869, 220, 17, 17, 13, 198, 1192, 220, 17, 18, 25, 869, 220, 17, 18, 13, 198, 1192, 220, 17, 19, 25, 869, 220, 17, 19, 13, 198, 1192, 220, 17, 20, 25, 869, 220, 17, 20, 13, 198, 1192, 220, 17, 21, 25, 869, 220, 17, 21, 13, 198, 1192, 220, 17, 22, 25, 869, 220, 17, 22, 13, 198, 1192, 220, 17, 23, 25, 869, 220, 17, 23, 13, 198, 1192, 220, 17, 24, 25, 869, 220, 17, 24, 13, 198, 1192, 220, 18, 15, 25, 869, 220, 18, 15, 13, 198, 1192, 220, 18, 16, 25, 869, 220, 18, 16, 13, 198, 1192, 220, 18, 17, 25, 869, 220, 18, 17, 13, 198, 1192, 220, 18, 18, 25, 869, 220, 18, 18, 13, 198, 1192, 220, 18, 19, 25, 869, 220, 18, 19, 13, 198, 1192, 220, 18, 20, 25, 869, 220, 18, 20, 13, 198, 1192, 220, 18, 21, 25, 869, 220, 18, 21, 13, 198, 1192, 220, 18, 22, 25, 869, 220, 18, 22, 13, 198, 1192, 220, 18, 23, 25, 869, 220, 18, 23, 13, 198, 1192, 220, 18, 24, 25, 869, 220, 18, 24, 13, 198, 1192, 220, 19, 15, 25, 869, 220, 19, 15, 13, 198, 1192, 220, 19, 16, 25, 869, 220, 19, 16, 13, 198, 1192, 220, 19, 17, 25, 869, 220, 19, 17, 13, 198, 1192, 220, 19, 18, 25, 869, 220, 19, 18, 13, 198, 1192, 220, 19, 19, 25, 869, 220, 19, 19, 13, 198, 1192, 220, 19, 20, 25, 869, 220, 19, 20, 13, 198, 1192, 220, 19, 21, 25, 869, 220, 19, 21, 13, 198, 1192, 220, 19, 22, 25, 869, 220, 19, 22, 13, 198, 1192, 220, 19, 23, 25, 869, 220, 19, 23, 13, 198, 1192, 220, 19, 24, 25, 869, 220, 19, 24, 13, 198, 1192, 220, 20, 15, 25, 869, 220, 20, 15, 13, 198, 1192, 220, 20, 16, 25, 869, 220, 20, 16, 13, 198, 1192, 220, 20, 17, 25, 869, 220, 20, 17, 13, 198, 1192, 220, 20, 18, 25, 869, 220, 20, 18, 13, 198, 1192, 220, 20, 19, 25, 869, 220, 20, 19, 13, 198, 1192, 220, 20, 20, 25, 869, 220, 20, 20, 13, 198, 1192, 220, 20, 21, 25, 869, 220, 20, 21, 13, 198, 1192, 220, 20, 22, 25, 869, 220, 20, 22, 13, 198, 1192, 220, 20, 23, 25, 869, 220, 20, 23, 13, 198, 1192, 220, 20, 24, 25, 869, 220, 20, 24, 13, 198, 1192, 220, 21, 15, 25, 869, 220, 21, 15, 13, 198, 1192, 220, 21, 16, 25, 869, 220, 21, 16, 13, 198, 1192, 220, 21, 17, 25, 869, 220, 21, 17, 13, 198, 1192, 220, 21, 18, 25, 869, 220, 21, 18, 13, 198, 1192, 220, 21, 19, 25, 869, 220, 21, 19, 13, 198, 1192, 220, 21, 20, 25, 869, 220, 21, 20, 13, 198, 1192, 220, 21, 21, 25, 869, 220, 21, 21, 13, 198, 1192, 220, 21, 22, 25, 869, 220, 21, 22, 13, 198, 1192, 220, 21, 23, 25, 869, 220, 21, 23, 13, 198, 1192, 220, 21, 24, 25, 869, 220, 21, 24, 13, 198, 1192, 220, 22, 15, 25, 869, 220, 22, 15, 13, 198, 1192, 220, 22, 16, 25, 869, 220, 22, 16, 13, 198, 1192, 220, 22, 17, 25, 869, 220, 22, 17, 13, 198, 1192, 220, 22, 18, 25, 869, 220, 22, 18, 13, 198, 1192, 220, 22, 19, 25, 869, 220, 22, 19, 13, 198, 1192, 220, 22, 20, 25, 869, 220, 22, 20, 13, 198, 1192, 220, 22, 21, 25, 869, 220, 22, 21, 13, 198, 1192, 220, 22, 22, 25, 869, 220, 22, 22, 13, 198, 1192, 220, 22, 23, 25, 869, 220, 22, 23, 13, 198, 1192, 220, 22, 24, 25, 869, 220, 22, 24, 13, 198, 1192, 220, 23, 15, 25, 869, 220, 23, 15, 13, 198, 1192, 220, 23, 16, 25, 869, 220, 23, 16, 13, 198, 1192, 220, 23, 17, 25, 869, 220, 23, 17, 13, 198, 1192, 220, 23, 18, 25, 869, 220, 23, 18, 13, 198, 1192, 220, 23, 19, 25, 869, 220, 23, 19, 13, 198, 1192, 220, 23, 20, 25, 869, 220, 23, 20, 13, 198, 1192, 220, 23, 21, 25, 869, 220, 23, 21, 13, 198, 1192, 220, 23, 22, 25, 869, 220, 23, 22, 13, 198, 1192, 220, 23, 23, 25, 869, 220, 23, 23, 13, 198, 1192, 220, 23, 24, 25, 869, 220, 23, 24, 13, 198, 1192, 220, 24, 15, 25, 869, 220, 24, 15, 13, 198, 1192, 220, 24, 16, 25, 869, 220, 24, 16, 13, 198, 1192, 220, 24, 17, 25, 869, 220, 24, 17, 13, 198, 1192, 220, 24, 18, 25, 869, 220, 24, 18, 13, 198, 1192, 220, 24, 19, 25, 869, 220, 24, 19, 13, 198, 1192, 220, 24, 20, 25, 869, 220, 24, 20, 13, 198, 1192, 220, 24, 21, 25, 869, 220, 24, 21, 13, 198, 1192, 220, 24, 22, 25, 869, 220, 15, 13, 198, 1192, 220, 24, 23, 25, 869, 220, 16, 13, 198, 1192, 220, 24, 24, 25, 869, 220, 17, 13, 198, 1192, 220, 16, 15, 15, 25, 869, 220, 18, 13, 198, 1192, 220, 16, 15, 16, 25, 869, 220, 19, 13, 198, 1192, 220, 16, 15, 17, 25, 869, 220, 20, 13, 198, 1192, 220, 16, 15, 18, 25, 869, 220, 21, 13, 198, 1192, 220, 16, 15, 19, 25, 869, 220, 22, 13, 198, 1192, 220, 16, 15, 20, 25, 869, 220, 23, 13, 198, 1192, 220, 16, 15, 21, 25, 869, 220, 24, 13, 198, 1192, 220, 16, 15, 22, 25, 869, 220, 16, 15, 13, 198, 1192, 220, 16, 15, 23, 25, 869, 220, 16, 16, 13, 198, 1192, 220, 16, 15, 24, 25, 869, 220, 16, 17, 13, 198, 1192, 220, 16, 16, 15, 25, 869, 220, 16, 18, 13, 198, 1192, 220, 16, 16, 16, 25, 869, 220, 16, 19, 13, 198, 1192, 220, 16, 16, 17, 25, 869, 220, 16, 20, 13, 198, 1192, 220, 16, 16, 18, 25, 869, 220, 16, 21, 13, 198, 1192, 220, 16, 16, 19, 25, 869, 220, 16, 22, 13, 198, 1192, 220, 16, 16, 20, 25, 869, 220, 16, 23, 13, 198, 1192, 220, 16, 16, 21, 25, 869, 220, 16, 24, 13, 198, 1192, 220, 16, 16, 22, 25, 869, 220, 17, 15, 13, 198, 1192, 220, 16, 16, 23, 25, 869, 220, 17, 16, 13, 198, 1192, 220, 16, 16, 24, 25, 869, 220, 17, 17, 13, 198, 1192, 220, 16, 17, 15, 25, 869, 220, 17, 18, 13, 198, 1192, 220, 16, 17, 16, 25, 869, 220, 17, 19, 13, 198, 1192, 220, 16, 17, 17, 25, 869, 220, 17, 20, 13, 198, 1192, 220, 16, 17, 18, 25, 869, 220, 17, 21, 13, 198, 1192, 220, 16, 17, 19, 25, 869, 220, 17, 22, 13, 198, 1192, 220, 16, 17, 20, 25, 869, 220, 17, 23, 13, 198, 1192, 220, 16, 17, 21, 25, 869, 220, 17, 24, 13, 198, 1192, 220, 16, 17, 22, 25, 869, 220, 18, 15, 13, 198, 1192, 220, 16, 17, 23, 25, 869, 220, 18, 16, 13, 198, 1192, 220, 16, 17, 24, 25, 869, 220, 18, 17, 13, 198, 1192, 220, 16, 18, 15, 25, 869, 220, 18, 18, 13, 198, 1192, 220, 16, 18, 16, 25, 869, 220, 18, 19, 13, 198, 1192, 220, 16, 18, 17, 25, 869, 220, 18, 20, 13, 198, 1192, 220, 16, 18, 18, 25, 869, 220, 18, 21, 13, 198, 1192, 220, 16, 18, 19, 25, 869, 220, 18, 22, 13, 198, 1192, 220, 16, 18, 20, 25, 869, 220, 18, 23, 13, 198, 1192, 220, 16, 18, 21, 25, 869, 220, 18, 24, 13, 198, 1192, 220, 16, 18, 22, 25, 869, 220, 19, 15, 13, 198, 1192, 220, 16, 18, 23, 25, 869, 220, 19, 16, 13, 198, 1192, 220, 16, 18, 24, 25, 869, 220, 19, 17, 13, 198, 1192, 220, 16, 19, 15, 25, 869, 220, 19, 18, 13, 198, 1192, 220, 16, 19, 16, 25, 869, 220, 19, 19, 13, 198, 1192, 220, 16, 19, 17, 25, 869, 220, 19, 20, 13, 198, 1192, 220, 16, 19, 18, 25, 869, 220, 19, 21, 13, 198, 1192, 220, 16, 19, 19, 25, 869, 220, 19, 22, 13, 198, 1192, 220, 16, 19, 20, 25, 869, 220, 19, 23, 13, 198, 1192, 220, 16, 19, 21, 25, 869, 220, 19, 24, 13, 198, 1192, 220, 16, 19, 22, 25, 869, 220, 20, 15, 13, 198, 1192, 220, 16, 19, 23, 25, 869, 220, 20, 16, 13, 198, 1192, 220, 16, 19, 24, 25, 869, 220, 20, 17, 13, 198, 1192, 220, 16, 20, 15, 25, 869, 220, 20, 18, 13, 198, 1192, 220, 16, 20, 16, 25, 869, 220, 20, 19, 13, 198, 1192, 220, 16, 20, 17, 25, 869, 220, 20, 20, 13, 198, 1192, 220, 16, 20, 18, 25, 869, 220, 20, 21, 13, 198, 1192, 220, 16, 20, 19, 25, 869, 220, 20, 22, 13, 198, 1192, 220, 16, 20, 20, 25, 869, 220, 20, 23, 13, 198, 1192, 220, 16, 20, 21, 25, 869, 220, 20, 24, 13, 198, 1192, 220, 16, 20, 22, 25, 869, 220, 21, 15, 13, 198, 1192, 220, 16, 20, 23, 25, 869, 220, 21, 16, 13, 198, 1192, 220, 16, 20, 24, 25, 869, 220, 21, 17, 13, 198, 1192, 220, 16, 21, 15, 25, 869, 220, 21, 18, 13, 198, 1192, 220, 16, 21, 16, 25, 869, 220, 21, 19, 13, 198, 1192, 220, 16, 21, 17, 25, 869, 220, 21, 20, 13, 198, 1192, 220, 16, 21, 18, 25, 869, 220, 21, 21, 13, 198, 1192, 220, 16, 21, 19, 25, 869, 220, 21, 22, 13, 198, 1192, 220, 16, 21, 20, 25, 869, 220, 21, 23, 13, 198, 1192, 220, 16, 21, 21, 25, 869, 220, 21, 24, 13, 198, 1192, 220, 16, 21, 22, 25, 869, 220, 22, 15, 13, 198, 1192, 220, 16, 21, 23, 25, 869, 220, 22, 16, 13, 198, 1192, 220, 16, 21, 24, 25, 869, 220, 22, 17, 13, 198, 1192, 220, 16, 22, 15, 25, 869, 220, 22, 18, 13, 198, 1192, 220, 16, 22, 16, 25, 869, 220, 22, 19, 13, 198, 1192, 220, 16, 22, 17, 25, 869, 220, 22, 20, 13, 198, 1192, 220, 16, 22, 18, 25, 869, 220, 22, 21, 13, 198, 1192, 220, 16, 22, 19, 25, 869, 220, 22, 22, 13, 198, 1192, 220, 16, 22, 20, 25, 869, 220, 22, 23, 13, 198, 1192, 220, 16, 22, 21, 25, 869, 220, 22, 24, 13, 198, 1192, 220, 16, 22, 22, 25, 869, 220, 23, 15, 13, 198, 1192, 220, 16, 22, 23, 25, 869, 220, 23, 16, 13, 198, 1192, 220, 16, 22, 24, 25, 869, 220, 23, 17, 13, 198, 1192, 220, 16, 23, 15, 25, 869, 220, 23, 18, 13, 198, 1192, 220, 16, 23, 16, 25, 869, 220, 23, 19, 13, 198, 1192, 220, 16, 23, 17, 25, 869, 220, 23, 20, 13, 198, 1192, 220, 16, 23, 18, 25, 869, 220, 23, 21, 13, 198, 1192, 220, 16, 23, 19, 25, 869, 220, 23, 22, 13, 198, 1192, 220, 16, 23, 20, 25, 869, 220, 23, 23, 13, 198, 1192, 220, 16, 23, 21, 25, 869, 220, 23, 24, 13, 198, 1192, 220, 16, 23, 22, 25, 869, 220, 24, 15, 13, 198, 1192, 220, 16, 23, 23, 25, 869, 220, 24, 16, 13, 198, 1192, 220, 16, 23, 24, 25, 869, 220, 24, 17, 13, 198, 1192, 220, 16, 24, 15, 25, 869, 220, 24, 18, 13, 198, 1192, 220, 16, 24, 16, 25, 869, 220, 24, 19, 13, 198, 1192, 220, 16, 24, 17, 25, 869, 220, 24, 20, 13, 198, 1192, 220, 16, 24, 18, 25, 869, 220, 24, 21, 13, 198, 1192, 220, 16, 24, 19, 25, 869, 220, 15, 13, 198, 1192, 220, 16, 24, 20, 25, 869, 220, 16, 13, 198, 1192, 220, 16, 24, 21, 25, 869, 220, 17, 13, 198, 1192, 220, 16, 24, 22, 25, 869, 220, 18, 13, 198, 1192, 220, 16, 24, 23, 25, 869, 220, 19, 13, 198, 1192, 220, 16, 24, 24, 25, 869, 220, 20, 13, 198, 1192, 220, 17, 15, 15, 25, 869, 220, 21, 13, 198, 1192, 220, 17, 15, 16, 25, 869, 220, 22, 13, 198, 1192, 220, 17, 15, 17, 25, 869, 220, 23, 13, 198, 1192, 220, 17, 15, 18, 25, 869, 220, 24, 13, 198, 1192, 220, 17, 15, 19, 25, 869, 220, 16, 15, 13, 198, 1192, 220, 17, 15, 20, 25, 869, 220, 16, 16, 13, 198, 1192, 220, 17, 15, 21, 25, 869, 220, 16, 17, 13, 198, 1192, 220, 17, 15, 22, 25, 869, 220, 16, 18, 13, 198, 1192, 220, 17, 15, 23, 25, 869, 220, 16, 19, 13, 198, 1192, 220, 17, 15, 24, 25, 869, 220, 16, 20, 13, 198, 1192, 220, 17, 16, 15, 25, 869, 220, 16, 21, 13, 198, 1192, 220, 17, 16, 16, 25, 869, 220, 16, 22, 13, 198, 1192, 220, 17, 16, 17, 25, 869, 220, 16, 23, 13, 198, 1192, 220, 17, 16, 18, 25, 869, 220, 16, 24, 13, 198, 1192, 220, 17, 16, 19, 25, 869, 220, 17, 15, 13, 198, 1192, 220, 17, 16, 20, 25, 869, 220, 17, 16, 13, 198, 1192, 220, 17, 16, 21, 25, 869, 220, 17, 17, 13, 198, 1192, 220, 17, 16, 22, 25, 869, 220, 17, 18, 13, 198, 1192, 220, 17, 16, 23, 25, 869, 220, 17, 19, 13, 198, 1192, 220, 17, 16, 24, 25, 869, 220, 17, 20, 13, 198, 1192, 220, 17, 17, 15, 25, 869, 220, 17, 21, 13, 198, 1192, 220, 17, 17, 16, 25, 869, 220, 17, 22, 13, 198, 1192, 220, 17, 17, 17, 25, 869, 220, 17, 23, 13, 198, 1192, 220, 17, 17, 18, 25, 869, 220, 17, 24, 13, 198, 1192, 220, 17, 17, 19, 25, 869, 220, 18, 15, 13, 198, 1192, 220, 17, 17, 20, 25, 869, 220, 18, 16, 13, 198, 1192, 220, 17, 17, 21, 25, 869, 220, 18, 17, 13, 198, 1192, 220, 17, 17, 22, 25, 869, 220, 18, 18, 13, 198, 1192, 220, 17, 17, 23, 25, 869, 220, 18, 19, 13, 198, 1192, 220, 17, 17, 24, 25, 869, 220, 18, 20, 13, 198, 1192, 220, 17, 18, 15, 25, 869, 220, 18, 21, 13, 198, 1192, 220, 17, 18, 16, 25, 869, 220, 18, 22, 13, 198, 1192, 220, 17, 18, 17, 25, 869, 220, 18, 23, 13, 198, 1192, 220, 17, 18, 18, 25, 869, 220, 18, 24, 13, 198, 1192, 220, 17, 18, 19, 25, 869, 220, 19, 15, 13, 198, 1192, 220, 17, 18, 20, 25, 869, 220, 19, 16, 13, 198, 1192, 220, 17, 18, 21, 25, 869, 220, 19, 17, 13, 198, 1192, 220, 17, 18, 22, 25, 869, 220, 19, 18, 13, 198, 1192, 220, 17, 18, 23, 25, 869, 220, 19, 19, 13, 198, 1192, 220, 17, 18, 24, 25, 869, 220, 19, 20, 13, 198, 8917, 5437, 537, 279, 1103, 321, 10033, 1204, 279, 2663, 12771, 13, 248046, 198, 248045, 74455, 198, 248068, 271, 248069, 271], "text": "Item 0: value 0.\nItem 1: value 1.\nItem 2: value 2.\nItem 3: value 3.\nItem 4: value 4.\nItem 5: value 5.\nItem 6: value 6.\nItem 7: value 7.\nItem 8: value 8.\nItem 9: value 9.\nItem 10: value 10.\nItem 11: value 11.\nItem 12: value 12.\nItem 13: value 13.\nItem 14: value 14.\nItem 15: value 15.\nItem 16: value 16.\nItem 17: value 17.\nItem 18: value 18.\nItem 19: value 19.\nItem 20: value 20.\nItem 21: value 21.\nItem 22: value 22.\nItem 23: value 23.\nItem 24: value 24.\nItem 25: value 25.\nItem 26: value 26.\nItem 27: value 27.\nItem 28: value 28.\nItem 29: value 29.\nItem 30: value 30.\nItem 31: value 31.\nItem 32: value 32.\nItem 33: value 33.\nItem 34: value 34.\nItem 35: value 35.\nItem 36: value 36.\nItem 37: value 37.\nItem 38: value 38.\nItem 39: value 39.\nItem 40: value 40.\nItem 41: value 41.\nItem 42: value 42.\nItem 43: value 43.\nItem 44: value 44.\nItem 45: value 45.\nItem 46: value 46.\nItem 47: value 47.\nItem 48: value 48.\nItem 49: value 49.\nItem 50: value 50.\nItem 51: value 51.\nItem 52: value 52.\nItem 53: value 53.\nItem 54: value 54.\nItem 55: value 55.\nItem 56: value 56.\nItem 57: value 57.\nItem 58: value 58.\nItem 59: value 59.\nItem 60: value 60.\nItem 61: value 61.\nItem 62: value 62.\nItem 63: value 63.\nItem 64: value 64.\nItem 65: value 65.\nItem 66: value 66.\nItem 67: value 67.\nItem 68: value 68.\nItem 69: value 69.\nItem 70: value 70.\nItem 71: value 71.\nItem 72: value 72.\nItem 73: value 73.\nItem 74: value 74.\nItem 75: value 75.\nItem 76: value 76.\nItem 77: value 77.\nItem 78: value 78.\nItem 79: value 79.\nItem 80: value 80.\nItem 81: value 81.\nItem 82: value 82.\nItem 83: value 83.\nItem 84: value 84.\nItem 85: value 85.\nItem 86: value 86.\nItem 87: value 87.\nItem 88: value 88.\nItem 89: value 89.\nItem 90: value 90.\nItem 91: value 91.\nItem 92: value 92.\nItem 93: value 93.\nItem 94: value 94.\nItem 95: value 95.\nItem 96: value 96.\nItem 97: value 0.\nItem 98: value 1.\nItem 99: value 2.\nItem 100: value 3.\nItem 101: value 4.\nItem 102: value 5.\nItem 103: value 6.\nItem 104: value 7.\nItem 105: value 8.\nItem 106: value 9.\nItem 107: value 10.\nItem 108: value 11.\nItem 109: value 12.\nItem 110: value 13.\nItem 111: value 14.\nItem 112: value 15.\nItem 113: value 16.\nItem 114: value 17.\nItem 115: value 18.\nItem 116: value 19.\nItem 117: value 20.\nItem 118: value 21.\nItem 119: value 22.\nItem 120: value 23.\nItem 121: value 24.\nItem 122: value 25.\nItem 123: value 26.\nItem 124: value 27.\nItem 125: value 28.\nItem 126: value 29.\nItem 127: value 30.\nItem 128: value 31.\nItem 129: value 32.\nItem 130: value 33.\nItem 131: value 34.\nItem 132: value 35.\nItem 133: value 36.\nItem 134: value 37.\nItem 135: value 38.\nItem 136: value 39.\nItem 137: value 40.\nItem 138: value 41.\nItem 139: value 42.\nItem 140: value 43.\nItem 141: value 44.\nItem 142: value 45.\nItem 143: value 46.\nItem 144: value 47.\nItem 145: value 48.\nItem 146: value 49.\nItem 147: value 50.\nItem 148: value 51.\nItem 149: value 52.\nItem 150: value 53.\nItem 151: value 54.\nItem 152: value 55.\nItem 153: value 56.\nItem 154: value 57.\nItem 155: value 58.\nItem 156: value 59.\nItem 157: value 60.\nItem 158: value 61.\nItem 159: value 62.\nItem 160: value 63.\nItem 161: value 64.\nItem 162: value 65.\nItem 163: value 66.\nItem 164: value 67.\nItem 165: value 68.\nItem 166: value 69.\nItem 167: value 70.\nItem 168: value 71.\nItem 169: value 72.\nItem 170: value 73.\nItem 171: value 74.\nItem 172: value 75.\nItem 173: value 76.\nItem 174: value 77.\nItem 175: value 78.\nItem 176: value 79.\nItem 177: value 80.\nItem 178: value 81.\nItem 179: value 82.\nItem 180: value 83.\nItem 181: value 84.\nItem 182: value 85.\nItem 183: value 86.\nItem 184: value 87.\nItem 185: value 88.\nItem 186: value 89.\nItem 187: value 90.\nItem 188: value 91.\nItem 189: value 92.\nItem 190: value 93.\nItem 191: value 94.\nItem 192: value 95.\nItem 193: value 96.\nItem 194: value 0.\nItem 195: value 1.\nItem 196: value 2.\nItem 197: value 3.\nItem 198: value 4.\nItem 199: value 5.\nItem 200: value 6.\nItem 201: value 7.\nItem 202: value 8.\nItem 203: value 9.\nItem 204: value 10.\nItem 205: value 11.\nItem 206: value 12.\nItem 207: value 13.\nItem 208: value 14.\nItem 209: value 15.\nItem 210: value 16.\nItem 211: value 17.\nItem 212: value 18.\nItem 213: value 19.\nItem 214: value 20.\nItem 215: value 21.\nItem 216: value 22.\nItem 217: value 23.\nItem 218: value 24.\nItem 219: value 25.\nItem 220: value 26.\nItem 221: value 27.\nItem 222: value 28.\nItem 223: value 29.\nItem 224: value 30.\nItem 225: value 31.\nItem 226: value 32.\nItem 227: value 33.\nItem 228: value 34.\nItem 229: value 35.\nItem 230: value 36.\nItem 231: value 37.\nItem 232: value 38.\nItem 233: value 39.\nItem 234: value 40.\nItem 235: value 41.\nItem 236: value 42.\nItem 237: value 43.\nItem 238: value 44.\nItem 239: value 45.\nSummarize the list and explain how the values repeat.", "thinking": false} diff --git a/docs/receipts/flashnext-mtp-chain-2026-10-10/prompts/thinking.json b/docs/receipts/flashnext-mtp-chain-2026-10-10/prompts/thinking.json new file mode 100644 index 0000000000..acbf44651e --- /dev/null +++ b/docs/receipts/flashnext-mtp-chain-2026-10-10/prompts/thinking.json @@ -0,0 +1 @@ +{"prompt": [248045, 8678, 198, 24342, 286, 4879, 369, 716, 310, 830, 11553, 13, 5044, 1683, 15060, 1472, 279, 3274, 11, 9307, 1328, 30800, 11, 2814, 47675, 25605, 11, 321, 60445, 55404, 11, 27224, 11, 321, 30246, 303, 279, 1534, 4087, 13, 248046, 198, 248045, 846, 198, 814, 20139, 3069, 264, 9057, 3578, 314, 1330, 4021, 381, 264, 23665, 1324, 13, 248046, 198, 248045, 74455, 198, 248068, 198], "text": "Explain why a square root of two cannot be a rational number.", "thinking": true} diff --git a/tools/zig/flashnext_mtp_chain.py b/tools/zig/flashnext_mtp_chain.py index 8513130d16..59b174bb16 100644 --- a/tools/zig/flashnext_mtp_chain.py +++ b/tools/zig/flashnext_mtp_chain.py @@ -22,15 +22,16 @@ def main(): if args.tokens < 1 or args.rounds < 3: parser.error("tokens must be positive; rounds must include one warmup and at least two measured rounds") args.output.mkdir(parents=True, exist_ok=True) - cases = [json.loads((args.fixtures / f"{name}.json").read_text()) for name in - ("short", "thinking", "sparse", "sparse-thinking")] - receipt = {"exact": False, "binary_sha256": hashlib.sha256(args.binary.read_bytes()).hexdigest(), - "tokens": args.tokens, "rounds": args.rounds, "warmup_rounds": 1, "blocks": []} + receipt = {"exact": False, "tokens": args.tokens, "rounds": args.rounds, + "warmup_rounds": 1, "blocks": []} def save(): (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") save() + receipt["binary_sha256"] = hashlib.sha256(args.binary.read_bytes()).hexdigest() + cases = [json.loads((args.fixtures / f"{name}.json").read_text()) for name in + ("short", "thinking", "sparse", "sparse-thinking")] reference = None previous = os.environ.get("FZ_BATCH_MTP_CHAIN") try: @@ -41,6 +42,8 @@ def save(): batches = gate.run(args, slots, cases, block) receipt["blocks"].append({"parallel": slots, "chain": chain, "batches": batches}) save() + if slots > 1 and not any(batch["peak_running"] > 1 for batch in batches): + raise RuntimeError(f"shared sessions were not observed in block {block}") for batch in batches: replies = [{key: result[key] for key in ("reply", "tokens", "prompt_tokens", "token_sha")} for result in batch["results"]] From 77ebc3fc05443bc9908a13f456b067b5d7fb10b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:46:40 +0200 Subject: [PATCH 06/31] Attach final current-base host validation receipt --- .../flashnext-mtp-chain-2026-10-10/README.md | 3 +- .../flashnext-mtp-chain-2026-10-10/golden.log | 1 - .../flashnext-mtp-chain-2026-10-10/host.log | 161 ++++++++++++++++++ .../manifest.json | 8 +- 4 files changed, 170 insertions(+), 3 deletions(-) create mode 100644 docs/receipts/flashnext-mtp-chain-2026-10-10/host.log diff --git a/docs/receipts/flashnext-mtp-chain-2026-10-10/README.md b/docs/receipts/flashnext-mtp-chain-2026-10-10/README.md index 44fcbdfc89..d5eb7cb806 100644 --- a/docs/receipts/flashnext-mtp-chain-2026-10-10/README.md +++ b/docs/receipts/flashnext-mtp-chain-2026-10-10/README.md @@ -68,7 +68,8 @@ The original service was restored and verified healthy; native test processes an ## Host validation and limits -`zig build test` and whole-tree `zig fmt --check build.zig zig` exit successfully with Zig 0.17.0. +[Host check](host.log): 137/137 build steps succeed; 351 tests pass, two are skipped and none fail. +Other test steps are cached. Whole-tree `zig fmt --check build.zig zig` passes with Zig 0.17.0. [Golden check](golden.log): 309 equal, 10 documented differences, zero unexpected differences. Both native builds pass. The lean checker finds 542 inherited problems on upstream and candidate, with none added. Astra reviewed the chain, the metadata lifetime, state accounting, direct gates and benchmark failure handling. diff --git a/docs/receipts/flashnext-mtp-chain-2026-10-10/golden.log b/docs/receipts/flashnext-mtp-chain-2026-10-10/golden.log index 28cecc1bdd..a82b1308a3 100644 --- a/docs/receipts/flashnext-mtp-chain-2026-10-10/golden.log +++ b/docs/receipts/flashnext-mtp-chain-2026-10-10/golden.log @@ -58,4 +58,3 @@ test-golden success +- run exe golden_check success 1m +- compile exe golden_check debug native-native success 1s MaxRSS:291M +- compile exe fake_serve debug native-native success 3s MaxRSS:588M - diff --git a/docs/receipts/flashnext-mtp-chain-2026-10-10/host.log b/docs/receipts/flashnext-mtp-chain-2026-10-10/host.log new file mode 100644 index 0000000000..3d528cc1aa --- /dev/null +++ b/docs/receipts/flashnext-mtp-chain-2026-10-10/host.log @@ -0,0 +1,161 @@ +Build Summary: 137/137 steps succeeded; 351/353 tests passed (2 skipped) +test success ++- run test cached +| +- compile test debug native cached 60ms MaxRSS:31M ++- run test cached +| +- compile test debug native cached 178ms MaxRSS:31M ++- run test cached +| +- compile test fast native cached 122ms MaxRSS:31M +| +- WriteFile draft_ids.txt cached +| +- WriteFile nemotron_packed.metallib cached +| +- run xcrun (nemotron_packed.metallib) cached +| +- run xcrun (coop_down_1_0.air) cached +| +- run xcrun (coop_down_2_0.air) cached +| +- run xcrun (coop_down_2_1.air) cached +| +- run xcrun (coop_draft_1_0.air) cached +| +- run xcrun (coop_draft_2_0.air) cached +| +- run xcrun (coop_draft_2_1.air) cached +| +- run xcrun (coop_eh_1_0.air) cached +| +- run xcrun (coop_eh_1_0_sk.air) cached +| +- run xcrun (coop_eh_2_0.air) cached +| +- run xcrun (coop_eh_2_0_sk.air) cached +| +- run xcrun (coop_eh_2_1.air) cached +| +- run xcrun (coop_eh_2_1_sk.air) cached +| +- run xcrun (coop_head_1_0.air) cached +| +- run xcrun (coop_head_2_0.air) cached +| +- run xcrun (coop_head_2_1.air) cached +| +- run xcrun (coop_in_1_0.air) cached +| +- run xcrun (coop_in_1_0_sk.air) cached +| +- run xcrun (coop_in_2_0.air) cached +| +- run xcrun (coop_in_2_0_sk.air) cached +| +- run xcrun (coop_in_2_1.air) cached +| +- run xcrun (coop_in_2_1_sk.air) cached +| +- run xcrun (coop_out_1_0.air) cached +| +- run xcrun (coop_out_1_0_sk.air) cached +| +- run xcrun (coop_out_2_0.air) cached +| +- run xcrun (coop_out_2_0_sk.air) cached +| +- run xcrun (coop_out_2_1.air) cached +| +- run xcrun (coop_out_2_1_sk.air) cached +| +- run xcrun (coop_qkv_1_0.air) cached +| +- run xcrun (coop_qkv_2_0.air) cached +| +- run xcrun (coop_qkv_2_1.air) cached +| +- run xcrun (coop_up_1_0.air) cached +| +- run xcrun (coop_up_2_0.air) cached +| +- run xcrun (coop_up_2_1.air) cached +| +- run xcrun (up_relu2_1_0.air) cached +| +- run xcrun (up_relu2_2_0.air) cached +| +- run xcrun (up_relu2_2_1.air) cached ++- run test cached +| +- compile test fast native cached 178ms MaxRSS:31M ++- run test cached +| +- compile test fast native cached 114ms MaxRSS:31M ++- run test cached +| +- compile test fast native cached 90ms MaxRSS:31M ++- run test cached +| +- compile test fast native cached 115ms MaxRSS:31M +| +- WriteFile nemotron_packed.metallib (+1 more reused dependencies) ++- run test cached +| +- compile test fast native cached 123ms MaxRSS:31M +| +- WriteFile draft_ids.txt (reused) +| +- WriteFile xsum.metal cached +| +- WriteFile nemotron_packed.metallib (+1 more reused dependencies) ++- run test cached +| +- compile test safe native cached 113ms MaxRSS:31M +| +- WriteFile nemotron_packed.metallib (+1 more reused dependencies) ++- run test cached +| +- compile test fast native cached 111ms MaxRSS:31M +| +- WriteFile nemotron_packed.metallib (+1 more reused dependencies) ++- run test cached +| +- compile test fast native cached 115ms MaxRSS:31M +| +- WriteFile nemotron_packed.metallib (+1 more reused dependencies) ++- run test cached +| +- compile test safe native cached 178ms MaxRSS:31M ++- run test cached +| +- compile test safe native cached 113ms MaxRSS:31M +| +- WriteFile nemotron_packed.metallib (+1 more reused dependencies) ++- run test cached +| +- compile test safe native cached 164ms MaxRSS:31M ++- run test cached +| +- compile test safe native cached 180ms MaxRSS:31M ++- run test cached +| +- compile test safe native cached 64ms MaxRSS:31M ++- run test 93 pass (93 total) 227ms MaxRSS:7M +| +- compile test safe native success 8s MaxRSS:611M +| +- WriteFile xsum.metal (reused) +| +- WriteFile xsum.metal (reused) +| +- WriteFile nemotron_packed.metallib (+1 more reused dependencies) ++- run test cached +| +- compile test safe native cached 104ms MaxRSS:31M ++- run test cached +| +- compile test safe native cached 119ms MaxRSS:31M ++- run sh success 37ms +| +- compile exe tensorfold-native safe native success 50s MaxRSS:2G +| +- options cached +| +- WriteFile draft_ids.txt (reused) +| +- WriteFile xsum.metal (reused) +| +- WriteFile nemotron_packed.metallib (+1 more reused dependencies) ++- run test 108 pass (108 total) 1s MaxRSS:8M +| +- compile test debug native success 3s MaxRSS:666M ++- run test cached +| +- compile test debug native cached 99ms MaxRSS:31M ++- run test cached +| +- compile test fast native cached 186ms MaxRSS:31M ++- run test cached +| +- compile test fast native cached 106ms MaxRSS:31M ++- run test cached +| +- compile test fast native cached 124ms MaxRSS:31M +| +- WriteFile tensorfold.metallib cached +| +- run xcrun (tensorfold.metallib) cached +| +- run xcrun (runtime_tests.air) cached +| +- run xcrun (nemotron_expert_up.air) cached +| +- run xcrun (nemotron_expert_down.air) cached ++- run test cached +| +- compile test fast native cached 130ms MaxRSS:31M +| +- WriteFile draft_ids.txt (reused) +| +- WriteFile xsum.metal (reused) +| +- WriteFile nemotron_packed.metallib (+1 more reused dependencies) ++- run test cached +| +- compile test fast native cached 60ms MaxRSS:31M ++- run test cached +| +- compile test fast native cached 114ms MaxRSS:31M ++- run test cached +| +- compile test fast native cached 105ms MaxRSS:31M ++- run test cached +| +- compile test fast native cached 171ms MaxRSS:31M ++- test-cuda-host success +| +- run test 108 pass (108 total) 1s MaxRSS:8M +| | +- compile test debug native success 4s MaxRSS:657M +| +- run test cached +| | +- compile test debug native cached 74ms MaxRSS:31M +| | +- options cached +| +- run test 21 pass, 1 skip (22 total) 1s MaxRSS:5M +| | +- compile test debug native success 1s MaxRSS:329M +| +- run test cached +| | +- compile test debug native cached 178ms MaxRSS:31M +| +- run test 15 pass (15 total) 290ms MaxRSS:3M +| | +- compile test debug native success 1s MaxRSS:305M +| | +- WriteFile draft_ids.txt (reused) +| | +- options (reused) +| +- run test cached +| | +- compile test debug native cached 179ms MaxRSS:31M +| +- run test 5 pass, 1 skip (6 total) 370ms MaxRSS:3M +| | +- compile test debug native success 1s MaxRSS:355M +| | +- options (reused) +| | +- WriteFile draft_ids.txt (reused) +| +- run test cached +| | +- compile test debug native cached 180ms MaxRSS:31M +| +- run test 1 pass (1 total) 227ms MaxRSS:2M +| +- compile test debug native success 1s MaxRSS:270M +| +- options (reused) +| +- options (reused) +| +- WriteFile draft_ids.txt (reused) ++- run test cached +| +- compile test debug native cached 48ms MaxRSS:31M +| +- options cached +| +- compile lib hip-mock-success debug native cached 3s MaxRSS:30M +| +- compile lib hip-mock-failed debug native cached 98ms MaxRSS:30M +| +- compile lib hip-mock-missing debug native cached 102ms MaxRSS:30M ++- run test cached +| +- compile test debug native cached 60ms MaxRSS:31M ++- run exe learned-warning-test success 10ms + +- compile exe learned-warning-test debug native cached 100ms MaxRSS:31M diff --git a/docs/receipts/flashnext-mtp-chain-2026-10-10/manifest.json b/docs/receipts/flashnext-mtp-chain-2026-10-10/manifest.json index 96e52eb5ed..875352b77a 100644 --- a/docs/receipts/flashnext-mtp-chain-2026-10-10/manifest.json +++ b/docs/receipts/flashnext-mtp-chain-2026-10-10/manifest.json @@ -34,6 +34,12 @@ "golden_unexpected_differences": 0, "lean_upstream_findings": 542, "lean_candidate_findings": 542, - "helper_failure_smokes_passed": 2 + "helper_failure_smokes_passed": 2, + "host_build_steps_succeeded": 137, + "host_build_steps_total": 137, + "host_tests_passed": 351, + "host_tests_skipped": 2, + "host_tests_failed": 0, + "host_other_steps_cached": true } } From a5e409e3b23f01a25bd02b79046e7adcb15219b3 Mon Sep 17 00:00:00 2001 From: BobClawblaw <288991054+BobClawblaw@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:34:50 -0500 Subject: [PATCH 07/31] Flash Next shared sessions: FLASHNEXT_LOAD_LIMIT_GB lets the slots take the server past 70% of RAM, as GLM_LOAD_LIMIT_GB does for GLM --- zig/src/native/cache_fit.zig | 31 +++++++++++++++++++++++++ zig/src/native/flashnext_batch_host.zig | 19 ++++++++++++--- zig/src/native/metal.zig | 2 +- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/zig/src/native/cache_fit.zig b/zig/src/native/cache_fit.zig index 12f26ddefc..f19dacf707 100644 --- a/zig/src/native/cache_fit.zig +++ b/zig/src/native/cache_fit.zig @@ -22,6 +22,20 @@ pub fn fit(total: u64, ready: u64, gib: ?f64, over: bool) error{CacheOverCap}!Fi return .{ .budget = want, .room = room, .cap = cap }; } +/// The ceiling a served model may hold: 70% of `total`, or `text` (a load limit in GB, as GLM_LOAD_LIMIT_GB) up to +/// `most` (the smaller of RAM and the GPU's recommended working set); text that is not a number above 0 is refused. +pub const Ceiling = struct { bytes: u64, default: u64, asked: bool = false, capped: bool = false }; + +pub fn ceiling(total: u64, most: u64, text: ?[]const u8) error{BadLoadLimit}!Ceiling { + const default = total / 100 * SHARE_PERCENT; + const t = std.mem.trim(u8, text orelse "", " \t"); + if (t.len == 0) return .{ .bytes = default, .default = default }; + const gb = std.fmt.parseFloat(f64, t) catch return error.BadLoadLimit; + if (!std.math.isFinite(gb) or gb <= 0) return error.BadLoadLimit; + const top: f64 = @floatFromInt(@min(total, most)); + return .{ .bytes = @intFromFloat(@min(gb * 1e9, top)), .default = default, .asked = true, .capped = gb * 1e9 > top }; +} + /// This machine's RAM (null where the OS does not say). pub fn ram() ?u64 { return std.process.totalSystemMemory() catch null; @@ -91,6 +105,23 @@ test "a given budget that fits is kept, a larger one is refused unless the overr try std.testing.expectEqual(@as(u64, 64 * GiB), (try fit(512 * GiB, 172 * GiB, 64, false)).budget); // a 512 GB Mac has room } +test "the ceiling: 70% of RAM, or a load limit in GB up to RAM and the GPU's working set; bad text refused" { + const ram256: u64 = 256 * GiB; + const ws: u64 = 239_000_000_000; + const d = try ceiling(ram256, ws, null); + try std.testing.expectEqual(ram256 / 100 * 70, d.bytes); + try std.testing.expect(!d.asked and !d.capped); + const asked = try ceiling(ram256, ws, "215"); + try std.testing.expectEqual(@as(u64, 215_000_000_000), asked.bytes); + try std.testing.expect(asked.asked and !asked.capped and asked.bytes > asked.default); + try std.testing.expect((try ceiling(ram256, ws, " 300 ")).capped); + try std.testing.expectEqual(ws, (try ceiling(ram256, ws, "300")).bytes); + try std.testing.expectEqual(ram256 / 100 * 70, (try ceiling(ram256, ws, " ")).bytes); + try std.testing.expectError(error.BadLoadLimit, ceiling(ram256, ws, "lots")); + try std.testing.expectError(error.BadLoadLimit, ceiling(ram256, ws, "0")); + try std.testing.expectError(error.BadLoadLimit, ceiling(ram256, ws, "-4")); +} + test "the OS readings: RAM and this process's footprint" { if (builtin.os.tag != .macos) return error.SkipZigTest; const r = ram() orelse return error.NoRam; diff --git a/zig/src/native/flashnext_batch_host.zig b/zig/src/native/flashnext_batch_host.zig index 4956950c60..22ca2ec48c 100644 --- a/zig/src/native/flashnext_batch_host.zig +++ b/zig/src/native/flashnext_batch_host.zig @@ -21,6 +21,19 @@ const Host = struct { warm: mtl.keepalive.Target, }; +/// What the sessions may take the server to: 70% of RAM, or FLASHNEXT_LOAD_LIMIT_GB (GB, as GLM_LOAD_LIMIT_GB) up to the +/// GPU's recommended working set; past the default the log names the risk, as GLM's loader does. +fn ceiling(total: u64, working_set: u64) error{BadLoadLimit}!u64 { + const text: ?[]const u8 = if (std.c.getenv("FLASHNEXT_LOAD_LIMIT_GB")) |v| std.mem.span(v) else null; + const c = fit.ceiling(total, working_set, text) catch |err| { + std.log.err("flash next: FLASHNEXT_LOAD_LIMIT_GB={s} is not a load limit: give a number of GB above 0, or unset it for 70% of RAM", .{text.?}); + return err; + }; + if (c.capped) std.log.info("flash next: FLASHNEXT_LOAD_LIMIT_GB={s} passes this GPU's recommended working set; the limit is {d:.1} GB", .{ text.?, fit.gibs(c.bytes) * 1.073741824 }); + if (c.bytes > c.default) std.log.warn("flash next: FLASHNEXT_LOAD_LIMIT_GB lets the sessions take this Mac to {d:.1} GiB, past the default {d:.1} GiB (70% of RAM): a load that wires most of the RAM can stall macOS until its watchdog kills the server", .{ fit.gibs(c.bytes), fit.gibs(c.default) }); + return c.bytes; +} + pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const u8, window: i64, streams: u32, cache_gib: ?f64, over: bool, a: std.mem.Allocator, why: *[]const u8) !api.Opened { const pool = mtl.objc.Pool.push(); defer pool.pop(); @@ -32,9 +45,10 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const eng.hostMode(); const ready = fit.footprint() orelse return error.NoMemoryReading; const total = fit.ram() orelse return error.NoMemoryReading; + const cap = try ceiling(total, eng.r.device.maxWorkingSet()); const reserved = if (!over) (if (cache_gib) |g| (if (g > 0) std.math.lossyCast(u64, g * fit.GiB) else 0) else 0) else 0; const fixed = Session.overhead(tf.flashnext_replay.CAP) + tf.flashnext_replay.MAXR * tf.flashnext_replay.WIDE * 2; - const available = total / 100 * fit.SHARE_PERCENT -| ready -| fit.MARGIN -| reserved; + const available = cap -| ready -| fit.MARGIN -| reserved; if (available < fixed) return error.OverMemoryLimit; const room = available - fixed; const slots = try tf.flashnext_batch_meta.fit(streams, true, room, Session.bytes(capacity)); @@ -45,8 +59,7 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const h.back = try Backend.init(gpa, eng, slots, capacity); errdefer h.back.deinit(); const allocated = fit.footprint() orelse return error.NoMemoryReading; - if (allocated +| fit.MARGIN +| reserved > total / 100 * fit.SHARE_PERCENT) - return error.OverMemoryLimit; + if (allocated +| fit.MARGIN +| reserved > cap) return error.OverMemoryLimit; h.cfg = try lanes.Config.init(gpa, h.back.facts(), tf.flashnext_replay.MAXR, tf.flashnext_replay.MAXR - 1); errdefer h.cfg.deinit(gpa); h.clock = .{ .io = io }; diff --git a/zig/src/native/metal.zig b/zig/src/native/metal.zig index a419977be5..22f5b148b3 100644 --- a/zig/src/native/metal.zig +++ b/zig/src/native/metal.zig @@ -307,7 +307,7 @@ fn openFlashNext(a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: } var why: []const u8 = ""; return @import("flashnext_batch_host.zig").open(gpa, io, o.dir, dump, if (window > 0) @min(window, tf.flashnext_replay.CAP - tf.flashnext_engine.MARGIN) else tf.flashnext_replay.CAP - tf.flashnext_engine.MARGIN, o.lanes, o.prompt_cache_gib, o.prompt_cache_over_cap, a, &why) catch |err| { - problem.* = if (err == error.CacheOverCap) why else try std.fmt.allocPrint(a, "the native Flash Next batching engine cannot open ({s})", .{@errorName(err)}); + problem.* = if (err == error.CacheOverCap) why else if (err == error.BadLoadLimit) "FLASHNEXT_LOAD_LIMIT_GB is not a load limit: give a number of GB above 0, or unset it for 70% of RAM" else if (err == error.OverMemoryLimit) "the native Flash Next batching engine cannot open (OverMemoryLimit): the sessions do not fit under 70% of RAM past the loaded model; fewer streams, a smaller --context, or FLASHNEXT_LOAD_LIMIT_GB past the share" else try std.fmt.allocPrint(a, "the native Flash Next batching engine cannot open ({s})", .{@errorName(err)}); return null; }; } From 1874ffe0b29f849909fd143e7a38983ca2b2d014 Mon Sep 17 00:00:00 2001 From: BobClawblaw <288991054+BobClawblaw@users.noreply.github.com> Date: Mon, 12 Oct 2026 04:17:05 -0500 Subject: [PATCH 08/31] Flash Next shared sessions sample: a request's temperature draws each token on the host from the head's logits The first token from the prompt pass's last row, each verify row from the window's logits, with the lane core's keyed sampler (top-k in one pass over bf16 keys, equal to qwen3_5's draw; families/flashnext/sample.zig, the same file as #719). Sampled replies are the same drafted and undrafted, alone and beside other streams; greedy replies are unchanged. --- docs/recipes/qwen3.8-flash-next.md | 4 +- zig/flashnext_host.zig | 1 + zig/src/families/flashnext/backend.zig | 21 ++++- zig/src/families/flashnext/sample.zig | 107 ++++++++++++++++++++++++ zig/src/native/flashnext_batch_host.zig | 3 +- zig/src/tensorfold.zig | 1 + 6 files changed, 133 insertions(+), 4 deletions(-) create mode 100644 zig/src/families/flashnext/sample.zig diff --git a/docs/recipes/qwen3.8-flash-next.md b/docs/recipes/qwen3.8-flash-next.md index 613ae04d5e..9dc40db99c 100644 --- a/docs/recipes/qwen3.8-flash-next.md +++ b/docs/recipes/qwen3.8-flash-next.md @@ -77,7 +77,9 @@ New requests join between rounds, and cancelling a request releases its slot for Additional slots allocate caches for the selected context. Startup reserves their metadata and the prompt-cache budget, then checks the physical footprint before accepting requests. A fixed slot count that does not fit is refused; lower the count, context or prompt-cache budget. `--prompt-cache-over-cap` overrides only the prompt-cache -budget. Greedy decoding remains required, so clients must send temperature 0. +budget. Shared sessions sample: a request with a temperature draws each token on the host from the head's logits +with the server's keyed sampler (top-k, top-p, min-p, a `seed` or one derived from the prompt), so a sampled reply is +the same with drafting on and off and alone or beside other streams. `--parallel 1` still decodes greedily only. The checkpoint gate compares shared replies with fresh legacy replies, with drafting off and on, then checks prompt admission, cancellation, slot reuse and a restored prefix while another request remains active: diff --git a/zig/flashnext_host.zig b/zig/flashnext_host.zig index 2091e630e6..328f8b04f1 100644 --- a/zig/flashnext_host.zig +++ b/zig/flashnext_host.zig @@ -2,5 +2,6 @@ test { _ = @import("src/families/flashnext/host_test.zig"); _ = @import("src/families/flashnext/pack.zig"); + _ = @import("src/families/flashnext/sample.zig"); _ = @import("src/families/flashnext/batch_meta_test.zig"); } diff --git a/zig/src/families/flashnext/backend.zig b/zig/src/families/flashnext/backend.zig index e4c02f2591..e5b2c74b58 100644 --- a/zig/src/families/flashnext/backend.zig +++ b/zig/src/families/flashnext/backend.zig @@ -12,6 +12,19 @@ const Stream = lanes.Stream; const Buf = fz.Buf; const mtl = @import("metal"); const segments = @import("../../core/segments.zig"); +const sample = @import("sample.zig"); + +/// A stream's sampling when it samples (a temperature above 0); null: the GPU's argmax stands. +fn sampling(stream: *const Stream) ?lanes.Sampling { + const p = stream.sampling orelse return null; + return if (p.temperature > 0) p else null; +} + +/// Row `row` of a [rows, VOCAB] bf16 logits buffer, drawn with `p` keyed at `position`. +fn drawRow(gpa: std.mem.Allocator, logits: Buf, row: usize, p: lanes.Sampling, position: u64) !u32 { + const bytes = logits.b.contents()[logits.off + row * fz.VOCAB * 2 ..][0 .. fz.VOCAB * 2]; + return sample.draw(gpa, @as([*]const u16, @ptrCast(@alignCast(bytes.ptr)))[0..fz.VOCAB], p, position); +} pub const Backend = struct { gpa: std.mem.Allocator, @@ -120,7 +133,6 @@ pub const Backend = struct { const pool = mtl.objc.Pool.push(); defer pool.pop(); const b = self(ptr); - if (stream.sampling) |p| if (p.temperature > 0) return error.GreedyOnly; const prompt = stream.prompt(); if (prompt.len == 0) return error.EmptyPrompt; try b.settle(); @@ -162,6 +174,10 @@ pub const Backend = struct { end = at + call.rows; const ps = [2]*fz.Prompt{ b.e.pr, b.e.pr2 }; s.first = try fz.Prompt.chunkN(ps[0..call.parts], &s.m, b.gpa, prompt[at..end]); + // the pass's last row's logits are in the model's head buffer until the next forward: a sampled first token now + if (end == prompt.len) if (sampling(stream)) |p| { + s.first = try drawRow(b.gpa, s.m.t.logits, 0, p, prompt.len); + }; for (ps[0..call.parts], 0..) |p, k| { const start = at + segments.start(call.rows, call.parts, k); const rows = segments.rows(call.rows, call.parts, k); @@ -243,6 +259,9 @@ pub const Backend = struct { try b.primary.finish(cb); Session.copy(b.primary.last, b.hidden, plan.total * fz.WIDE * 2); for (out, 0..) |o, i| @memcpy(o.sampled, t.picks.b.slice(u32, plan.total)[plan.starts[i]..][0..o.sampled.len]); + for (windows, out, 0..) |w, o, i| if (sampling(w.stream)) |p| { + for (o.sampled, w.positions, 0..) |*tok, pos, j| tok.* = try drawRow(b.gpa, t.logits, plan.starts[i] + j, p, pos); + }; b.window_count += 1; if (windows.len > 1) b.shared_count += 1; if (std.c.getenv("FZ_BATCH_LOG") != null) diff --git a/zig/src/families/flashnext/sample.zig b/zig/src/families/flashnext/sample.zig new file mode 100644 index 0000000000..3fbe8e1981 --- /dev/null +++ b/zig/src/families/flashnext/sample.zig @@ -0,0 +1,107 @@ +//! The keyed host sampler over a bf16 logits row, in one pass: the top-k candidates by an order-preserving integer key +//! of each bf16 word, then lanes.sampling.choose. The candidates, their order and the draw equal `reference`, which is +//! qwen3_5/backend.zig's draw without its mask (two passes in f64), kept here for wide top-k and the tests. +const std = @import("std"); +const lanes = @import("lanes"); + +/// k at most this takes the one-pass path; a wider (or unlimited) top-k takes the reference's. +pub const MAX_K = 256; + +/// A bf16 word's order as an unsigned key: larger floats give larger keys, -0 and +0 the same. +fn key(word: u16) u16 { + const w = if (word == 0x8000) 0 else word; + return if (w & 0x8000 != 0) ~w else w | 0x8000; +} + +fn value(word: u16) f64 { + return @as(f32, @bitCast(@as(u32, word) << 16)); +} + +/// qwen3_5/backend.zig's draw without a mask: the unchanged top-k candidates for the keyed sampler. +pub fn reference(gpa: std.mem.Allocator, logits: []const u16, p: lanes.Sampling, position: u64) !u32 { + var best: ?usize = null; + for (logits, 0..) |word, id| { + const x = value(word); + if (!std.math.isFinite(x)) return error.NonfiniteQwenLogits; + if (best == null or x > value(logits[best.?])) best = id; + } + if (p.temperature <= 0) return @intCast(best.?); + const allowed = logits.len; + const k = if (p.top_k == 0) allowed else @min(allowed, p.top_k); + const values = try gpa.alloc(f64, k); + defer gpa.free(values); + const ids = try gpa.alloc(u64, k); + defer gpa.free(ids); + var filled: usize = 0; + for (logits, 0..) |word, id| { + const x = value(word); + if (k == allowed) { + values[filled] = x; + ids[filled] = id; + filled += 1; + continue; + } + if (filled == k and x <= values[k - 1]) continue; + var at = @min(filled, k - 1); + while (at > 0 and x > values[at - 1]) : (at -= 1) { + values[at] = values[at - 1]; + ids[at] = ids[at - 1]; + } + values[at] = x; + ids[at] = id; + filled = @min(filled + 1, k); + } + return @intCast(try lanes.sampling.choose(gpa, values, ids, position, p)); +} + +/// Row `row` of a [rows, vocab] bf16 logits buffer whose bytes start at `base`. +pub fn row(base: [*]const u8, vocab: usize, at: usize) []const u16 { + return @as([*]const u16, @ptrCast(@alignCast(base + at * vocab * 2)))[0..vocab]; +} + +/// The draw for `logits` with `p` keyed at `position` (p.temperature above 0). +pub fn draw(gpa: std.mem.Allocator, logits: []const u16, p: lanes.Sampling, position: u64) !u32 { + const k: usize = p.top_k; + if (k == 0 or k > MAX_K or k >= logits.len) return reference(gpa, logits, p, position); + var keys: [MAX_K]u16 = undefined; + var ids: [MAX_K]u64 = undefined; + var filled: usize = 0; + for (logits, 0..) |word, id| { + if (word & 0x7f80 == 0x7f80) return error.NonfiniteQwenLogits; // exponent all ones: inf or nan + const x = key(word); + if (filled == k and x <= keys[k - 1]) continue; + var at = @min(filled, k - 1); + while (at > 0 and x > keys[at - 1]) : (at -= 1) { + keys[at] = keys[at - 1]; + ids[at] = ids[at - 1]; + } + keys[at] = x; + ids[at] = id; + filled = @min(filled + 1, k); + } + var values: [MAX_K]f64 = undefined; + for (ids[0..filled], values[0..filled]) |id, *v| v.* = value(logits[id]); + return @intCast(try lanes.sampling.choose(gpa, values[0..filled], ids[0..filled], position, p)); +} + +test "the one-pass draw equals the reference on random rows with ties, signed zeros and every top-k" { + const gpa = std.testing.allocator; + var rng = std.Random.DefaultPrng.init(7); + const r = rng.random(); + const words = try gpa.alloc(u16, 4096); + defer gpa.free(words); + for (0..40) |trial| { + for (words) |*w| { + const f: f32 = @floatCast(r.floatNorm(f64) * 3.0); + w.* = @intCast(@as(u32, @bitCast(f)) >> 16); + } + for (0..64) |j| words[r.uintLessThan(usize, words.len)] = words[j]; // ties + words[r.uintLessThan(usize, words.len)] = 0x8000; + words[r.uintLessThan(usize, words.len)] = 0; + for ([_]u32{ 1, 2, 20, 64, 256 }) |k| for ([_]f64{ 0.6, 1.0, 1.5 }) |t| { + const p: lanes.Sampling = .{ .seed = trial * 31 + k, .temperature = t, .top_k = k, .top_p = 0.95 }; + const pos: u64 = trial * 7 + k; + try std.testing.expectEqual(try reference(gpa, words, p, pos), try draw(gpa, words, p, pos)); + }; + } +} diff --git a/zig/src/native/flashnext_batch_host.zig b/zig/src/native/flashnext_batch_host.zig index 4956950c60..a54afd2030 100644 --- a/zig/src/native/flashnext_batch_host.zig +++ b/zig/src/native/flashnext_batch_host.zig @@ -54,7 +54,7 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const errdefer h.core.deinit(); const plan = try fit.budget(cache_gib, over, 0, a, why, ""); eng.snap_pool.max = plan.budget_bytes; - h.host = api.LaneHost.init(gpa, io, &h.core, .{ .name = "flashnext-zig", .lanes = slots, .greedy_only = true, .context_window = @intCast(window), .prefill_step = @intCast(eng.pr.step), .prompt_cache_plan = plan }); + h.host = api.LaneHost.init(gpa, io, &h.core, .{ .name = "flashnext-zig", .lanes = slots, .context_window = @intCast(window), .prefill_step = @intCast(eng.pr.step), .prompt_cache_plan = plan }); h.cache = null; if (plan.budget_bytes > 0) { h.cache = api.prompt_cache.Store.init(gpa, .{ .ptr = &h.back, .vtable = &.{ .bytes = Backend.snapBytes, .save = Backend.snapSave, .restore = Backend.snapRestore, .drop = Backend.snapDrop, .charged = Backend.snapCharged, .spare = Backend.snapSpare, .trim = Backend.snapTrim, .reuses = Backend.snapReuses } }, .{ .lookahead = 1, .planned = true }, plan.budget_bytes); @@ -72,7 +72,6 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const fn words(_: ?*anyopaque, err: anyerror) ?[]const u8 { return switch (err) { - error.GreedyOnly => "the native Flash Next engine decodes greedily only: send temperature 0", error.ContextFull => "the prompt and max_tokens do not fit this server's --context", else => null, }; diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index bdb31c39d7..1804603c1d 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -15,6 +15,7 @@ pub const segments = @import("core/segments.zig"); pub const row_projection = @import("core/row_projection.zig"); pub const nemotron = @import("families/nemotron/nemotron.zig"); pub const flashnext_replay = @import("families/flashnext/replay.zig"); +pub const flashnext_sample = @import("families/flashnext/sample.zig"); pub const flashnext_engine = @import("families/flashnext/engine.zig"); pub const flashnext_snapshot = @import("families/flashnext/snapshot.zig"); pub const flashnext_session = @import("families/flashnext/session.zig"); From b7372ac847224ca90180c310da9d479d2757bc0a Mon Sep 17 00:00:00 2001 From: BobClawblaw <288991054+BobClawblaw@users.noreply.github.com> Date: Mon, 12 Oct 2026 12:02:28 -0500 Subject: [PATCH 09/31] Flash Next's host sampler: an unlimited or wide top-k by a counting sort of bf16 keys, not a full sort, the same draws top_k 0 (or above 256) went through the reference: every candidate in f64, sorted, then choose. Now a counting sort of the 65,536 bf16 keys gives choose's order (ids ascending within a key), top-p and min-p cut it as choose does with libm's exp and the same pairwise sum, and the keyed Gumbel draw runs in that order, passing over candidates whose noise is provably a nat short of the best score (one hash, no logs) and stopping once none can reach it. 248k vocabulary: 13.7-16.8 ms a row to 0.33-0.65 ms; the host test now also compares top_k 0, 300 and the whole row, top-p 1, min-p and wide logits. --- zig/src/families/flashnext/sample.zig | 102 ++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 7 deletions(-) diff --git a/zig/src/families/flashnext/sample.zig b/zig/src/families/flashnext/sample.zig index 3fbe8e1981..7763842f90 100644 --- a/zig/src/families/flashnext/sample.zig +++ b/zig/src/families/flashnext/sample.zig @@ -4,7 +4,10 @@ const std = @import("std"); const lanes = @import("lanes"); -/// k at most this takes the one-pass path; a wider (or unlimited) top-k takes the reference's. +extern "c" fn exp(x: f64) f64; // libm's, as lanes.sampling's +extern "c" fn log(x: f64) f64; + +/// k at most this takes the one-pass path; a wider (or unlimited) top-k takes `wide`. pub const MAX_K = 256; /// A bf16 word's order as an unsigned key: larger floats give larger keys, -0 and +0 the same. @@ -62,7 +65,7 @@ pub fn row(base: [*]const u8, vocab: usize, at: usize) []const u16 { /// The draw for `logits` with `p` keyed at `position` (p.temperature above 0). pub fn draw(gpa: std.mem.Allocator, logits: []const u16, p: lanes.Sampling, position: u64) !u32 { const k: usize = p.top_k; - if (k == 0 or k > MAX_K or k >= logits.len) return reference(gpa, logits, p, position); + if (k == 0 or k > MAX_K or k >= logits.len) return wide(gpa, logits, p, position); var keys: [MAX_K]u16 = undefined; var ids: [MAX_K]u64 = undefined; var filled: usize = 0; @@ -84,6 +87,89 @@ pub fn draw(gpa: std.mem.Allocator, logits: []const u16, p: lanes.Sampling, posi return @intCast(try lanes.sampling.choose(gpa, values[0..filled], ids[0..filled], position, p)); } +/// A wide or unlimited top-k as `reference` draws it: the candidates in choose's order by a counting sort of their +/// keys (ids ascending within one), top-p and min-p cut as choose cuts them, then the keyed Gumbel draw in that order, +/// passing over candidates whose noise cannot lift them past the best score so far (most, without their two logs). +fn wide(gpa: std.mem.Allocator, logits: []const u16, p: lanes.Sampling, position: u64) !u32 { + const starts = try gpa.alloc(u32, 1 << 16); + defer gpa.free(starts); + @memset(starts, 0); + for (logits) |word| { + if (word & 0x7f80 == 0x7f80) return error.NonfiniteQwenLogits; // exponent all ones: inf or nan + starts[0xffff - key(word)] += 1; // larger keys first + } + var at: u32 = 0; + for (starts) |*c| { + const n = c.*; + c.* = at; + at += n; + } + const order = try gpa.alloc(u32, logits.len); + defer gpa.free(order); + for (logits, 0..) |word, id| { + const b = 0xffff - @as(usize, key(word)); + order[starts[b]] = @intCast(id); + starts[b] += 1; + } + const t = @max(p.temperature, 1e-6); + const k = @max(1, @min(if (p.top_k != 0) @as(usize, p.top_k) else logits.len, logits.len)); + const x0 = value(logits[order[0]]) / t; + var keep = k; + if (p.top_p > 0.0 and p.top_p < 1.0) { + const probs = try gpa.alloc(f64, k); + defer gpa.free(probs); + for (probs, order[0..k]) |*q, id| q.* = exp(value(logits[id]) / t - x0); + const total = lanes.sampling.pairwiseSum(probs); + var cum: f64 = 0.0; + var below: usize = 0; + for (probs) |q| { // cum only grows: past top_p no later one counts + cum += q / total; + if (cum < p.top_p) below += 1 else break; + } + keep = @min(keep, below + 1); + } + if (p.min_p > 0.0) { + const floor = x0 + p.minLog(); + var n: usize = 0; + for (order[0..keep]) |id| { + if (value(logits[id]) / t >= floor) n += 1; + } + keep = n; + } + var best: usize = 0; + var best_score = -std.math.inf(f64); + for (order[0..keep], 0..) |id, j| { + const x = value(logits[id]) / t; + if (j > 0) { + const d = best_score - x; + if (d > 38) break; // a draw's noise is at most -log(-log(1 - 2^-54)) = 37.43, and x only falls from here + if (d > 1 and beyondReach(p.seed, position, id, d)) continue; + } + const score = x - log(-log(lanes.sampling.uniform(p.seed, position, id))); + if (j == 0 or score > best_score) { + best = j; + best_score = score; + } + } + return order[best]; +} + +/// exp(-j) for the whole nats below a candidate's shortfall, for beyondReach. +const below_e = blk: { + var e: [38]f64 = undefined; + for (&e, 0..) |*v, j| v.* = @exp(-@as(f64, @floatFromInt(j))); + break :blk e; +}; + +/// Whether the noise for `id` stays a nat short of `d`: -log(-log u) <= -log(1 - u), so 1 - u >= exp(-(d - 1)) +/// bounds it by d - 1, read from the uniform's integer bits against exp of a whole nat at or below d - 1. +fn beyondReach(seed: u64, position: u64, id: u64, d: f64) bool { + const m = lanes.sampling.keyBits(seed, position, id) >> 11; + const one_minus_u = @as(f64, @floatFromInt((@as(u64, 1) << 53) - m)) * 0x1p-53 - 0x1p-54; + const j: usize = @intFromFloat(@floor(d) - 1); + return one_minus_u >= below_e[j] * (1 + 1e-9); +} + test "the one-pass draw equals the reference on random rows with ties, signed zeros and every top-k" { const gpa = std.testing.allocator; var rng = std.Random.DefaultPrng.init(7); @@ -92,16 +178,18 @@ test "the one-pass draw equals the reference on random rows with ties, signed ze defer gpa.free(words); for (0..40) |trial| { for (words) |*w| { - const f: f32 = @floatCast(r.floatNorm(f64) * 3.0); + const f: f32 = @floatCast(r.floatNorm(f64) * @as(f64, if (trial % 2 == 0) 3.0 else 20.0)); // narrow, then wide w.* = @intCast(@as(u32, @bitCast(f)) >> 16); } for (0..64) |j| words[r.uintLessThan(usize, words.len)] = words[j]; // ties words[r.uintLessThan(usize, words.len)] = 0x8000; words[r.uintLessThan(usize, words.len)] = 0; - for ([_]u32{ 1, 2, 20, 64, 256 }) |k| for ([_]f64{ 0.6, 1.0, 1.5 }) |t| { - const p: lanes.Sampling = .{ .seed = trial * 31 + k, .temperature = t, .top_k = k, .top_p = 0.95 }; - const pos: u64 = trial * 7 + k; - try std.testing.expectEqual(try reference(gpa, words, p, pos), try draw(gpa, words, p, pos)); + for ([_]u32{ 0, 1, 2, 20, 64, 256, 300, 4096 }) |k| for ([_]f64{ 0.6, 1.0, 1.5, 4.0 }) |t| { + for ([_][2]f64{ .{ 0.95, 0 }, .{ 1.0, 0 }, .{ 0.5, 0.05 }, .{ 1.0, 0.2 } }) |cut| { + const p: lanes.Sampling = .{ .seed = trial * 31 + k, .temperature = t, .top_k = k, .top_p = cut[0], .min_p = cut[1] }; + const pos: u64 = trial * 7 + k; + try std.testing.expectEqual(try reference(gpa, words, p, pos), try draw(gpa, words, p, pos)); + } }; } } From 03cc8741a95d85d630fbec8b7ec123b9f1ef4b36 Mon Sep 17 00:00:00 2001 From: BobClawblaw <288991054+BobClawblaw@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:12:02 -0500 Subject: [PATCH 10/31] FLASHNEXT_LOAD_LIMIT_GB covers the prompt cache too: the automatic budget is what the sessions leave under it The slot checks reserved a given cache against the selected limit while fit.budget still planned the cache against 70% of RAM, so an automatic cache could take the server past the limit and a raised limit did not raise a given cache's own gate. cache_fit.budgetUnder plans the cache under the same ceiling (fitUnder; budget and fit keep 70% for the other hosts), with a test for the automatic and the given budget under a limit. The recipe doc, the limit's comment and the OverMemoryLimit message name the contract. --- docs/recipes/qwen3.8-flash-next.md | 5 ++++ zig/src/native/cache_fit.zig | 40 +++++++++++++++++++++---- zig/src/native/flashnext_batch_host.zig | 8 ++--- zig/src/native/metal.zig | 2 +- 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/docs/recipes/qwen3.8-flash-next.md b/docs/recipes/qwen3.8-flash-next.md index 613ae04d5e..5c0d2699d5 100644 --- a/docs/recipes/qwen3.8-flash-next.md +++ b/docs/recipes/qwen3.8-flash-next.md @@ -74,6 +74,11 @@ New requests join between rounds, and cancelling a request releases its slot for `auto` and `--parallel 1` retain the existing single-reply driver. Shared sessions currently run on one Mac; `--parallel N` above one with `--speed-up` is refused. Use `--parallel 1` for the two-Mac link. +The ceiling is 70% of RAM, or `FLASHNEXT_LOAD_LIMIT_GB` (GB, up to the GPU's recommended working set). It covers the +server's whole footprint: the model, the sessions and the prompt cache. Left out, `--prompt-cache-gib` takes what the +sessions leave under it; a given budget must fit under the same ceiling, and only `--prompt-cache-over-cap` lets it +pass. + Additional slots allocate caches for the selected context. Startup reserves their metadata and the prompt-cache budget, then checks the physical footprint before accepting requests. A fixed slot count that does not fit is refused; lower the count, context or prompt-cache budget. `--prompt-cache-over-cap` overrides only the prompt-cache diff --git a/zig/src/native/cache_fit.zig b/zig/src/native/cache_fit.zig index f19dacf707..3b4e78dafc 100644 --- a/zig/src/native/cache_fit.zig +++ b/zig/src/native/cache_fit.zig @@ -15,7 +15,11 @@ pub const Fit = struct { budget: u64, room: u64, cap: u64 }; /// The budget for `total` bytes of RAM and a footprint of `ready` bytes once loaded: `gib` when given and it fits (or `over`), /// else all that fits; a given budget past the room is refused. pub fn fit(total: u64, ready: u64, gib: ?f64, over: bool) error{CacheOverCap}!Fit { - const cap = total / 100 * SHARE_PERCENT; + return fitUnder(total / 100 * SHARE_PERCENT, ready, gib, over); +} + +/// `fit` under a given ceiling `cap` (bytes) instead of 70% of RAM: a load limit covers the cache as well. +pub fn fitUnder(cap: u64, ready: u64, gib: ?f64, over: bool) error{CacheOverCap}!Fit { const room = cap -| ready -| MARGIN; const want = if (gib) |g| (if (g > 0) std.math.lossyCast(u64, g * GiB) else 0) else room; if (want > room and !over) return error.CacheOverCap; @@ -54,6 +58,11 @@ pub fn footprint() ?u64 { /// The cache plan: what 70% of RAM leaves past this process, or `gib` if it fits (else refused unless `over`). pub fn budget(gib: ?f64, over: bool, spare: u64, a: std.mem.Allocator, why: *[]const u8, rank: []const u8) !api.PromptCachePlan { + return budgetUnder(null, gib, over, spare, a, why, rank); +} + +/// `budget` under a load limit `limit` (bytes; null: 70% of RAM), so the cache and the rest stay under one ceiling. +pub fn budgetUnder(limit: ?u64, gib: ?f64, over: bool, spare: u64, a: std.mem.Allocator, why: *[]const u8, rank: []const u8) !api.PromptCachePlan { const total = ram() orelse 0; const ready = footprint() orelse 0; if (total == 0 or ready == 0) { @@ -61,12 +70,15 @@ pub fn budget(gib: ?f64, over: bool, spare: u64, a: std.mem.Allocator, why: *[]c std.log.info("prompt cache: {d:.1} GiB for kept prompt states{s} (no memory reading)", .{ gibs(b), rank }); return .{ .source = if (gib != null) .explicit else .metal_working_set, .budget_bytes = b, .explicit_budget = gib != null }; } - const f = fit(total, ready, gib, over) catch |e| { - const left = (fit(total, ready, null, false) catch unreachable).room; // without a given budget it never refuses - why.* = try std.fmt.allocPrint(a, "--prompt-cache-gib {d} would take this server past 70% of this Mac's memory: it holds {d:.1} GiB once loaded, 70% of {d:.0} GiB is {d:.1} GiB, and {d} GiB stays free for prompt buffers, so {d:.1} GiB is left for kept prompt states. Leave --prompt-cache-gib out to use that, pass a smaller one, or add --prompt-cache-over-cap to keep {d} GiB anyway.", .{ gib.?, gibs(ready), gibs(total), gibs(total / 100 * SHARE_PERCENT), MARGIN >> 30, gibs(left), gib.? }); + const cap = limit orelse total / 100 * SHARE_PERCENT; + const f = fitUnder(cap, ready, gib, over) catch |e| { + const left = (fitUnder(cap, ready, null, false) catch unreachable).room; // without a given budget it never refuses + if (limit != null) { + why.* = try std.fmt.allocPrint(a, "--prompt-cache-gib {d} would take this server past its load limit: it holds {d:.1} GiB once loaded, the limit is {d:.1} GiB, and {d} GiB stays free for prompt buffers, so {d:.1} GiB is left for kept prompt states. Leave --prompt-cache-gib out to use that, pass a smaller one, raise the limit, or add --prompt-cache-over-cap to keep {d} GiB anyway.", .{ gib.?, gibs(ready), gibs(cap), MARGIN >> 30, gibs(left), gib.? }); + } else why.* = try std.fmt.allocPrint(a, "--prompt-cache-gib {d} would take this server past 70% of this Mac's memory: it holds {d:.1} GiB once loaded, 70% of {d:.0} GiB is {d:.1} GiB, and {d} GiB stays free for prompt buffers, so {d:.1} GiB is left for kept prompt states. Leave --prompt-cache-gib out to use that, pass a smaller one, or add --prompt-cache-over-cap to keep {d} GiB anyway.", .{ gib.?, gibs(ready), gibs(total), gibs(cap), MARGIN >> 30, gibs(left), gib.? }); return e; }; - std.log.info("prompt cache: {d:.1} GiB from {d:.1} GiB free under the 70% cap ({d:.1} GiB in use once loaded, {d:.0} GiB of RAM, {d} GiB kept for prompt buffers){s}{s}", .{ gibs(f.budget), gibs(f.room), gibs(ready), gibs(total), MARGIN >> 30, if (f.budget > f.room) ", past the cap by --prompt-cache-over-cap" else "", rank }); + std.log.info("prompt cache: {d:.1} GiB from {d:.1} GiB free under the {s} ({d:.1} GiB in use once loaded, {d:.0} GiB of RAM, {d} GiB kept for prompt buffers){s}{s}", .{ gibs(f.budget), gibs(f.room), if (limit != null) "load limit" else "70% cap", gibs(ready), gibs(total), MARGIN >> 30, if (f.budget > f.room) ", past the cap by --prompt-cache-over-cap" else "", rank }); return .{ .source = .physical_footprint, .budget_bytes = f.budget, @@ -105,6 +117,24 @@ test "a given budget that fits is kept, a larger one is refused unless the overr try std.testing.expectEqual(@as(u64, 64 * GiB), (try fit(512 * GiB, 172 * GiB, 64, false)).budget); // a 512 GB Mac has room } +test "under a load limit the automatic cache gets what the limit leaves, and a given one is held to the same limit" { + // 100 GiB of RAM, a 60 GiB load limit, 50 GiB in use once the sessions are allocated (the case in review) + const cap: u64 = 60 * GiB; + const ready: u64 = 50 * GiB; + const auto = try fitUnder(cap, ready, null, false); + try std.testing.expectEqual(@as(u64, 8 * GiB), auto.budget); // 60 - 50 - 2, not the 18 GiB 70% of RAM leaves + try std.testing.expect(ready + MARGIN + auto.budget <= cap); + try std.testing.expectEqual(@as(u64, 18 * GiB), (try fit(100 * GiB, ready, null, false)).budget); // 70% of RAM: past the limit + try std.testing.expectEqual(@as(u64, 8 * GiB), (try fitUnder(cap, ready, 8, false)).budget); + try std.testing.expectError(error.CacheOverCap, fitUnder(cap, ready, 16, false)); + try std.testing.expectEqual(@as(u64, 16 * GiB), (try fitUnder(cap, ready, 16, true)).budget); // the override still passes it + // a raised limit raises both: past 70% of RAM a given cache that fits under the limit needs no override + const raised: u64 = 80 * GiB; + try std.testing.expectEqual(@as(u64, 16 * GiB), (try fitUnder(raised, ready, 16, false)).budget); + try std.testing.expectError(error.CacheOverCap, fit(100 * GiB, 60 * GiB, 16, false)); + try std.testing.expectEqual(@as(u64, 18 * GiB), (try fitUnder(raised, 60 * GiB, null, false)).budget); +} + test "the ceiling: 70% of RAM, or a load limit in GB up to RAM and the GPU's working set; bad text refused" { const ram256: u64 = 256 * GiB; const ws: u64 = 239_000_000_000; diff --git a/zig/src/native/flashnext_batch_host.zig b/zig/src/native/flashnext_batch_host.zig index 22ca2ec48c..360e3dac21 100644 --- a/zig/src/native/flashnext_batch_host.zig +++ b/zig/src/native/flashnext_batch_host.zig @@ -21,8 +21,8 @@ const Host = struct { warm: mtl.keepalive.Target, }; -/// What the sessions may take the server to: 70% of RAM, or FLASHNEXT_LOAD_LIMIT_GB (GB, as GLM_LOAD_LIMIT_GB) up to the -/// GPU's recommended working set; past the default the log names the risk, as GLM's loader does. +/// What the server may hold in all, sessions and prompt cache: 70% of RAM, or FLASHNEXT_LOAD_LIMIT_GB (GB, as +/// GLM_LOAD_LIMIT_GB) up to the GPU's recommended working set; past the default the log names the risk, as GLM's does. fn ceiling(total: u64, working_set: u64) error{BadLoadLimit}!u64 { const text: ?[]const u8 = if (std.c.getenv("FLASHNEXT_LOAD_LIMIT_GB")) |v| std.mem.span(v) else null; const c = fit.ceiling(total, working_set, text) catch |err| { @@ -30,7 +30,7 @@ fn ceiling(total: u64, working_set: u64) error{BadLoadLimit}!u64 { return err; }; if (c.capped) std.log.info("flash next: FLASHNEXT_LOAD_LIMIT_GB={s} passes this GPU's recommended working set; the limit is {d:.1} GB", .{ text.?, fit.gibs(c.bytes) * 1.073741824 }); - if (c.bytes > c.default) std.log.warn("flash next: FLASHNEXT_LOAD_LIMIT_GB lets the sessions take this Mac to {d:.1} GiB, past the default {d:.1} GiB (70% of RAM): a load that wires most of the RAM can stall macOS until its watchdog kills the server", .{ fit.gibs(c.bytes), fit.gibs(c.default) }); + if (c.bytes > c.default) std.log.warn("flash next: FLASHNEXT_LOAD_LIMIT_GB lets the server (sessions and prompt cache) take this Mac to {d:.1} GiB, past the default {d:.1} GiB (70% of RAM): a load that wires most of the RAM can stall macOS until its watchdog kills the server", .{ fit.gibs(c.bytes), fit.gibs(c.default) }); return c.bytes; } @@ -65,7 +65,7 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const h.clock = .{ .io = io }; h.core = lanes.Engine.init(gpa, &h.cfg, h.back.backend(), h.clock.clock()); errdefer h.core.deinit(); - const plan = try fit.budget(cache_gib, over, 0, a, why, ""); + const plan = try fit.budgetUnder(cap, cache_gib, over, 0, a, why, ""); // the cache under the same limit as the sessions eng.snap_pool.max = plan.budget_bytes; h.host = api.LaneHost.init(gpa, io, &h.core, .{ .name = "flashnext-zig", .lanes = slots, .greedy_only = true, .context_window = @intCast(window), .prefill_step = @intCast(eng.pr.step), .prompt_cache_plan = plan }); h.cache = null; diff --git a/zig/src/native/metal.zig b/zig/src/native/metal.zig index 22f5b148b3..66bbe94d71 100644 --- a/zig/src/native/metal.zig +++ b/zig/src/native/metal.zig @@ -307,7 +307,7 @@ fn openFlashNext(a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: } var why: []const u8 = ""; return @import("flashnext_batch_host.zig").open(gpa, io, o.dir, dump, if (window > 0) @min(window, tf.flashnext_replay.CAP - tf.flashnext_engine.MARGIN) else tf.flashnext_replay.CAP - tf.flashnext_engine.MARGIN, o.lanes, o.prompt_cache_gib, o.prompt_cache_over_cap, a, &why) catch |err| { - problem.* = if (err == error.CacheOverCap) why else if (err == error.BadLoadLimit) "FLASHNEXT_LOAD_LIMIT_GB is not a load limit: give a number of GB above 0, or unset it for 70% of RAM" else if (err == error.OverMemoryLimit) "the native Flash Next batching engine cannot open (OverMemoryLimit): the sessions do not fit under 70% of RAM past the loaded model; fewer streams, a smaller --context, or FLASHNEXT_LOAD_LIMIT_GB past the share" else try std.fmt.allocPrint(a, "the native Flash Next batching engine cannot open ({s})", .{@errorName(err)}); + problem.* = if (err == error.CacheOverCap) why else if (err == error.BadLoadLimit) "FLASHNEXT_LOAD_LIMIT_GB is not a load limit: give a number of GB above 0, or unset it for 70% of RAM" else if (err == error.OverMemoryLimit) "the native Flash Next batching engine cannot open (OverMemoryLimit): the sessions and the given prompt cache do not fit under the load limit (70% of RAM, or FLASHNEXT_LOAD_LIMIT_GB) past the loaded model; fewer streams, a smaller --context or --prompt-cache-gib, or a higher FLASHNEXT_LOAD_LIMIT_GB" else try std.fmt.allocPrint(a, "the native Flash Next batching engine cannot open ({s})", .{@errorName(err)}); return null; }; } From d519c08573e1e089d9655460bb99bf7fb3b4c74c Mon Sep 17 00:00:00 2001 From: BobClawblaw <288991054+BobClawblaw@users.noreply.github.com> Date: Mon, 12 Oct 2026 14:35:59 -0500 Subject: [PATCH 11/31] Flash Next's wide top-k draw keeps a uniform that rounds to 1: its +inf noise wins as in the reference lanes.sampling.uniform's largest value, (2^53 - 1) 2^-53 + 2^-54, rounds to exactly 1.0 in f64, so the reference's Gumbel score is +inf there; the early stop assumed noise of at most 37.43 and could pass that candidate over (a seed that hits it: expected 4, found 0). Past a 38-nat shortfall the draw now still asks each candidate whether its uniform is 1 (one hash, no logs) and stops only once the best score is +inf; the pruning bound reads 1 - u from the f64 uniform itself (exact), so a u of 1 is never passed over. The reviewer's case is a host test, with the same candidate inside the bound and behind another leader. Speed unchanged (0.33-0.63 ms a row at 248k). --- zig/src/families/flashnext/sample.zig | 29 ++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/zig/src/families/flashnext/sample.zig b/zig/src/families/flashnext/sample.zig index 7763842f90..c388bb4fc2 100644 --- a/zig/src/families/flashnext/sample.zig +++ b/zig/src/families/flashnext/sample.zig @@ -142,8 +142,10 @@ fn wide(gpa: std.mem.Allocator, logits: []const u16, p: lanes.Sampling, position const x = value(logits[id]) / t; if (j > 0) { const d = best_score - x; - if (d > 38) break; // a draw's noise is at most -log(-log(1 - 2^-54)) = 37.43, and x only falls from here - if (d > 1 and beyondReach(p.seed, position, id, d)) continue; + if (d > 38) { // finite noise is at most -log(-log(1 - 2^-52)) = 36.04; a uniform that rounds to 1 gives +inf + if (best_score == std.math.inf(f64)) break; // the first +inf stands: a later one ties, never passes + if (lanes.sampling.uniform(p.seed, position, id) != 1.0) continue; + } else if (d > 1 and beyondReach(p.seed, position, id, d)) continue; } const score = x - log(-log(lanes.sampling.uniform(p.seed, position, id))); if (j == 0 or score > best_score) { @@ -162,10 +164,10 @@ const below_e = blk: { }; /// Whether the noise for `id` stays a nat short of `d`: -log(-log u) <= -log(1 - u), so 1 - u >= exp(-(d - 1)) -/// bounds it by d - 1, read from the uniform's integer bits against exp of a whole nat at or below d - 1. +/// bounds it by d - 1, against exp of a whole nat at or below d - 1. 1 - u is exact for the f64 u the draw uses +/// (Sterbenz past 0.5; below it the bound cannot hold), and a u rounded to 1 gives 0: never passed over. fn beyondReach(seed: u64, position: u64, id: u64, d: f64) bool { - const m = lanes.sampling.keyBits(seed, position, id) >> 11; - const one_minus_u = @as(f64, @floatFromInt((@as(u64, 1) << 53) - m)) * 0x1p-53 - 0x1p-54; + const one_minus_u = 1.0 - lanes.sampling.uniform(seed, position, id); const j: usize = @intFromFloat(@floor(d) - 1); return one_minus_u >= below_e[j] * (1 + 1e-9); } @@ -193,3 +195,20 @@ test "the one-pass draw equals the reference on random rows with ties, signed ze }; } } + +test "wide draw preserves reference when uniform rounds to one" { + const gpa = std.testing.allocator; + var words: [300]u16 = @splat(0); + words[0] = 0x4280; // 64.0; other logits are zero + const p: lanes.Sampling = .{ .seed = 2870505676250484375, .temperature = 1, .top_k = 0, .top_p = 1, .min_p = 0 }; + try std.testing.expectEqual(@as(f64, 1), lanes.sampling.uniform(p.seed, 0, 4)); + const expected = try reference(gpa, &words, p, 0); + try std.testing.expectEqual(@as(u32, 4), expected); + try std.testing.expectEqual(expected, try draw(gpa, &words, p, 0)); + // the same candidate within the pruning bound (shortfall under 38), and behind a different leader + words[0] = 0x41a0; // 20.0 + try std.testing.expectEqual(try reference(gpa, &words, p, 0), try draw(gpa, &words, p, 0)); + words[0] = 0; + words[2] = 0x4280; + try std.testing.expectEqual(try reference(gpa, &words, p, 0), try draw(gpa, &words, p, 0)); +} From 3f24b0091da67e42f7d42d8f21147bd8d22dcbaf Mon Sep 17 00:00:00 2001 From: Seth For Privacy <40500387+sethforprivacy@users.noreply.github.com> Date: Mon, 12 Oct 2026 14:09:43 -0400 Subject: [PATCH 12/31] Flash Next shared rounds priced from forwards timed at load; every stream's drafts in one command buffer The batch backend gave the lane core no costs, so each shared round filled up to 16 rows with drafts however few landed, and each stream's head ran in its own command buffers (a wait per draft). - batch_costs.zig times, on the last session before any stream holds it, a shared forward at 1..16 rows (the fastest of four, after six warm-up windows) and a chained head step (absorb plus 8 drafts against absorb plus 1, in one command buffer). Backend.facts reports them: shared_costs for the allocator, and a lone stream's window_costs and mtp_step_ms for its depth (FZ_LONE_UNPRICED=1: the old heuristic). - The lane core prices each draft past a stream's first at that step (Model.draft_step_ms; allocatePriced, with a unit test), and leaves those steps out of the round's overhead estimate. - Backend.draftTogether encodes every request's absorb and chained drafts into one command buffer (Session encodeDrafts / heldDrafts) and waits once; FZ_BATCH_MTP_SERIAL=1 keeps the per-stream path. Co-Authored-By: Claude Opus 5.5 --- zig/src/core/lanes/allocate.zig | 22 ++++- zig/src/core/lanes/config.zig | 3 + zig/src/core/lanes/depth.zig | 3 +- zig/src/core/lanes/engine.zig | 6 +- zig/src/families/flashnext/backend.zig | 83 ++++++++++++++++- zig/src/families/flashnext/batch_costs.zig | 103 +++++++++++++++++++++ zig/src/families/flashnext/session.zig | 30 ++++++ zig/src/native/flashnext_batch_host.zig | 3 +- 8 files changed, 244 insertions(+), 9 deletions(-) create mode 100644 zig/src/families/flashnext/batch_costs.zig diff --git a/zig/src/core/lanes/allocate.zig b/zig/src/core/lanes/allocate.zig index ae4fe80cfa..8ec9598dca 100644 --- a/zig/src/core/lanes/allocate.zig +++ b/zig/src/core/lanes/allocate.zig @@ -7,6 +7,11 @@ pub const Cost = struct { width: u32, ms: f64 }; /// Draft prefixes per stream with the most expected tokens a ms; fixed rows always run. Caller frees. pub fn allocate(gpa: Allocator, fixed: []const u32, probs: []const []const f64, costs: *const Table, overhead_ms: f64, max_rows: u64) ![]u32 { + return allocatePriced(gpa, fixed, probs, costs, overhead_ms, max_rows, 0); +} + +/// allocate, each stream's nodes past its first costing `step_ms` more (a head step drafts each of them). +pub fn allocatePriced(gpa: Allocator, fixed: []const u32, probs: []const []const f64, costs: *const Table, overhead_ms: f64, max_rows: u64, step_ms: f64) ![]u32 { const counts = try gpa.alloc(u32, probs.len); defer gpa.free(counts); const best_counts = try gpa.alloc(u32, probs.len); @@ -14,6 +19,7 @@ pub fn allocate(gpa: Allocator, fixed: []const u32, probs: []const []const f64, @memset(best_counts, 0); var rows: u64 = 0; for (fixed) |f| rows += f; + var steps: f64 = 0; var expected: f64 = @floatFromInt(rows); const timed = !costs.empty(); var best = rate(costs, timed, rows, expected, overhead_ms); @@ -31,9 +37,10 @@ pub fn allocate(gpa: Allocator, fixed: []const u32, probs: []const []const f64, } const s = pick orelse break; counts[s] += 1; + if (counts[s] > 1) steps += 1; rows += 1; expected += pick_p; - const now = rate(costs, timed, rows, expected, overhead_ms); + const now = rate(costs, timed, rows, expected, overhead_ms + step_ms * steps); if (now > best) { best = now; @memcpy(best_counts, counts); @@ -114,6 +121,19 @@ test "allocate takes the widest round when no cost is timed" { try std.testing.expectEqualSlices(u32, &.{ 2, 1 }, counts); } +test "a priced draft step keeps a stream's later drafts out when they land too seldom to pay" { + const gpa = std.testing.allocator; + var costs = try extendCosts(gpa, &.{ .{ .width = 1, .ms = 10.0 }, .{ .width = 8, .ms = 12.0 } }, 8); + defer costs.deinit(gpa); + const probs = [_][]const f64{ &.{ 0.6, 0.3, 0.15 }, &.{ 0.6, 0.3, 0.15 } }; + const free = try allocate(gpa, &.{ 1, 1 }, &probs, &costs, 2.0, 8); + defer gpa.free(free); + try std.testing.expectEqualSlices(u32, &.{ 3, 3 }, free); + const priced = try allocatePriced(gpa, &.{ 1, 1 }, &probs, &costs, 2.0, 8, 1.5); + defer gpa.free(priced); + try std.testing.expectEqualSlices(u32, &.{ 1, 1 }, priced); +} + test "extendCosts interpolates and extrapolates" { const gpa = std.testing.allocator; var t = try extendCosts(gpa, &.{ .{ .width = 4, .ms = 8.0 }, .{ .width = 2, .ms = 4.0 } }, 6); diff --git a/zig/src/core/lanes/config.zig b/zig/src/core/lanes/config.zig index 85361bd839..470c4f5099 100644 --- a/zig/src/core/lanes/config.zig +++ b/zig/src/core/lanes/config.zig @@ -24,6 +24,7 @@ pub const Model = struct { drafts: u32 = 1, // head drafts a round at most window_costs: []const Cost = &.{}, // forward ms by exact window width (timed at load) mtp_step_ms: f64 = 0.0, // one chained head step (timed at load) + draft_step_ms: f64 = 0.0, // a shared round's draft past a stream's first, as the backend's head runs it (0: priced free) streams_exact: bool = true, // shared forwards keep each stream's bits hidden_rows: bool = false, // the backend runs several streams' windows in one forward batch_rows: u32 = 0, // a shared forward's rows at most (0: 32) @@ -63,6 +64,7 @@ pub const Config = struct { shared_costs: Table, lane_costs: Table, // window ms by rows to 64 (lanes/plan_lanes.zig) mtp_step_ms: f64, + draft_step_ms: f64 = 0, // Model.draft_step_ms enter_match: i64 = 8, // matching tokens a copy needs (rejects coincidental indentation) copy_rate: f64 = 0.94, // a copied token's chance of landing in allocation lone_copies: bool = true, // Model.lone_copies @@ -121,6 +123,7 @@ pub const Config = struct { .lane_costs = try alloc.extendCosts(gpa, m.lane_costs, 64), .mtp_step_ms = m.mtp_step_ms, .k = .{ .depth_rate = if (m.depth_rate > 0) m.depth_rate else (Constants{}).depth_rate }, + .draft_step_ms = m.draft_step_ms, }; } diff --git a/zig/src/core/lanes/depth.zig b/zig/src/core/lanes/depth.zig index db51281770..3918f4b031 100644 --- a/zig/src/core/lanes/depth.zig +++ b/zig/src/core/lanes/depth.zig @@ -54,6 +54,7 @@ pub const Rule = struct { family_costs: *const Table, shared_costs: *const Table, mtp_step_ms: f64, + draft_step_ms: f64 = 0, // a shared round's draft past a stream's first (budgets price it) plain_guard: bool, node_probabilities: bool, batch_rows: u32, @@ -220,7 +221,7 @@ pub const Rule = struct { const ones = try r.gpa.alloc(u32, whos.len); defer r.gpa.free(ones); @memset(ones, 1); - const counts = try alloc.allocate(r.gpa, ones, probs, r.shared_costs, r.overheadMs(whos.len, sampled), @max(whos.len, r.batch_rows)); + const counts = try alloc.allocatePriced(r.gpa, ones, probs, r.shared_costs, r.overheadMs(whos.len, sampled), @max(whos.len, r.batch_rows), r.draft_step_ms); defer r.gpa.free(counts); if (r.plain_guard) { // no draft where none pays; a probe after a growing run of plain rounds diff --git a/zig/src/core/lanes/engine.zig b/zig/src/core/lanes/engine.zig index 0efdfc0067..368e69b890 100644 --- a/zig/src/core/lanes/engine.zig +++ b/zig/src/core/lanes/engine.zig @@ -56,6 +56,7 @@ pub const Engine = struct { .family_costs = &cfg.family_costs, .shared_costs = &cfg.shared_costs, .mtp_step_ms = cfg.mtp_step_ms, + .draft_step_ms = cfg.draft_step_ms, .plain_guard = cfg.plain_guard, .node_probabilities = cfg.node_probabilities, .batch_rows = cfg.batch_rows, @@ -293,7 +294,10 @@ pub const Engine = struct { for (heads.items, budgets) |i, d| try e.draftLate(plans[i].stream, plans[i].position, outcomes[i].follow, outcomes[i].path, d); } try e.backend.keep(windows, paths); - const ms = e.clock.elapsedMs(.overhead); + // the drafts budgets priced past each stream's first are no part of the round's overhead + var steps: i64 = 0; + for (budgets) |d| steps += @max(0, d - 1); + const ms = e.clock.elapsedMs(.overhead) - e.cfg.draft_step_ms * @as(f64, @floatFromInt(steps)); try trail.event(e, &.{ f("ev", str("overhead")), f("streams", int(plans.len)), f("rows", int(total)), f("ms", ev.bits(ms)) }); try e.rule.observeOverhead(@intCast(plans.len), total, ms, sampled); } diff --git a/zig/src/families/flashnext/backend.zig b/zig/src/families/flashnext/backend.zig index e5b2c74b58..92162b8bf7 100644 --- a/zig/src/families/flashnext/backend.zig +++ b/zig/src/families/flashnext/backend.zig @@ -13,6 +13,7 @@ const Buf = fz.Buf; const mtl = @import("metal"); const segments = @import("../../core/segments.zig"); const sample = @import("sample.zig"); +const batch_costs = @import("batch_costs.zig"); /// A stream's sampling when it samples (a temperature above 0); null: the GPU's argmax stands. fn sampling(stream: *const Stream) ?lanes.Sampling { @@ -39,6 +40,8 @@ pub const Backend = struct { draft_calls: u64 = 0, draft_waits: u64 = 0, draft_seconds: f64 = 0, + serial_drafts: bool = false, // FZ_BATCH_MTP_SERIAL=1: each stream's drafts in their own command buffers + costs: ?batch_costs.Costs = null, // timeCosts: the lane allocator's prices pub fn init(gpa: std.mem.Allocator, e: *Engine, n: u32, capacity: usize) !Backend { if (n < 2 or n > fz.MAXR or e.r.tp != null) return error.BadBatchSlots; @@ -51,7 +54,19 @@ pub const Backend = struct { made += 1; } const hidden = try e.r.device.buffer(fz.MAXR * fz.WIDE * 2, fz.opts); - return .{ .gpa = gpa, .e = e, .primary = e.m, .sessions = sessions, .hidden = .{ .b = hidden }, .chain_drafts = if (std.c.getenv("FZ_BATCH_MTP_CHAIN")) |v| v[0] == '1' else false }; + const env = struct { + fn on(name: [*:0]const u8) bool { + return if (std.c.getenv(name)) |v| v[0] == '1' else false; + } + }; + return .{ .gpa = gpa, .e = e, .primary = e.m, .sessions = sessions, .hidden = .{ .b = hidden }, .chain_drafts = env.on("FZ_BATCH_MTP_CHAIN"), .serial_drafts = env.on("FZ_BATCH_MTP_SERIAL") }; + } + + /// The lane allocator's prices, timed on the last session before any stream holds it. + pub fn timeCosts(b: *Backend) !void { + const c = try batch_costs.time(b, &b.sessions[b.sessions.len - 1]); + b.costs = c; + std.log.info("flash next: shared forwards {d:.2} {d:.2} {d:.2} {d:.2} {d:.2} ms at 1, 2, 4, 8, 16 rows; a chained draft {d:.2} ms", .{ c.shared[0].ms, c.shared[1].ms, c.shared[3].ms, c.shared[7].ms, c.shared[15].ms, c.step_ms }); } pub fn deinit(b: *Backend) void { @@ -67,7 +82,11 @@ pub const Backend = struct { } pub fn facts(b: *Backend) lanes.Model { - return .{ .exact_width = fz.MAXR, .first_copy_rows = 4, .mtp = true, .speculate = true, .speculate_early = false, .drafts = 8, .streams_exact = true, .hidden_rows = true, .batch_rows = fz.MAXR, .max_streams = @intCast(b.sessions.len), .draft_streams = false }; + const c: ?*const batch_costs.Costs = if (b.costs) |*x| x else null; + // shared rounds priced from the timed forwards, each draft past a stream's first at a chained step's time; a + // lone stream's depth from the same (FZ_LONE_UNPRICED=1: the heuristic) + const lone = c != null and std.c.getenv("FZ_LONE_UNPRICED") == null; + return .{ .exact_width = fz.MAXR, .first_copy_rows = 4, .mtp = true, .speculate = true, .speculate_early = false, .drafts = 8, .streams_exact = true, .hidden_rows = true, .batch_rows = fz.MAXR, .max_streams = @intCast(b.sessions.len), .draft_streams = !b.serial_drafts, .shared_costs = if (c) |x| &x.shared else &.{}, .draft_step_ms = if (c != null and !b.serial_drafts) c.?.step_ms else 0, .window_costs = if (lone) &c.?.shared else &.{}, .mtp_step_ms = if (lone) c.?.step_ms else 0 }; } fn self(ptr: *anyopaque) *Backend { @@ -106,7 +125,7 @@ pub const Backend = struct { select: ?fz.Select, shape: @TypeOf(@as(fz.Select, undefined).q_shape), passed: @FieldType(Engine, "passed"), - fn restore(old: Binding, b: *Backend) void { + pub fn restore(old: Binding, b: *Backend) void { b.e.m = old.model; b.e.host = old.host; b.e.cins = old.cins; @@ -116,7 +135,7 @@ pub const Backend = struct { } }; - fn bind(b: *Backend, s: *Session) !Binding { + pub fn bind(b: *Backend, s: *Session) !Binding { const e = b.e; const old: Binding = .{ .model = e.m, .host = e.host, .cins = e.cins, .select = e.r.sel, .shape = e.r.shapes.get("Kc_shape").?, .passed = e.passed }; try e.r.shapes.put(e.r.arena, "Kc_shape", s.shape); @@ -206,7 +225,7 @@ pub const Backend = struct { return @intCast(handle); } - fn outputs(b: *Backend, s: *Session, row: usize) void { + pub fn outputs(b: *Backend, s: *Session, row: usize) void { var li: usize = 0; for (&s.m.layers) |*l| if (l.linear) { l.cs[1] = .{ .b = b.e.o_cs, .off = (li * fz.MAXR + row) * fz.CS_ROW }; @@ -278,6 +297,7 @@ pub const Backend = struct { fn draft(ptr: *anyopaque, requests: []const be.DraftRequest) anyerror!void { const b = self(ptr); + if (!b.serial_drafts) return b.draftTogether(requests); for (requests) |r| { const pool = mtl.objc.Pool.push(); defer pool.pop(); @@ -316,6 +336,59 @@ pub const Backend = struct { } } + /// Every request's absorb and chained drafts in one command buffer, one wait for the round's drafts (a follow + /// past MAXR rows, the prompt's first, absorbs alone first). + fn draftTogether(b: *Backend, requests: []const be.DraftRequest) !void { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const start = mtl.clock.seconds(); + var follows: [fz.MAXR][]const u32 = undefined; + var hiddens: [fz.MAXR]Buf = undefined; + var ones: [fz.MAXR][1]u32 = undefined; + if (requests.len > fz.MAXR) return error.DraftOutOfStep; + for (requests, 0..) |r, k| { + if (r.early or r.lanes != null or r.ranks or r.depth >= fz.MAXR) return error.DraftOutOfStep; + const s = try b.session(r.stream); + follows[k] = if (r.first) |f| blk: { + ones[k][0] = switch (f) { + .value => |v| v, + .handle => |h| @intCast(h), + }; + break :blk ones[k][0..]; + } else r.follow; + if (follows[k].len == 0) return error.DraftOutOfStep; + hiddens[k] = if (r.rows) |path| blk: { + try s.round.draft(r.start, path, follows[k].len); + break :blk b.hidden.at(s.row0 * fz.WIDE * 2); + } else s.last; + if (follows[k].len > fz.MAXR) { // all but the last MAXR rows absorbed first, on their own + const cut = follows[k].len - fz.MAXR; + const old = try b.bind(s); + defer old.restore(b); + _ = try s.m.mtpAbsorb(follows[k][0..cut], hiddens[k]); + hiddens[k] = hiddens[k].at(cut * fz.WIDE * 2); + follows[k] = follows[k][cut..]; + b.draft_waits += (cut + fz.MAXR - 1) / fz.MAXR; + } + } + const cb = b.e.r.queue.commandBuffer(); + b.e.r.enc = cb.compute(if (b.e.r.serial) .serial else .concurrent); + for (requests, 0..) |r, k| { + const s = try b.session(r.stream); + const old = try b.bind(s); + defer old.restore(b); + s.encodeDrafts(follows[k], hiddens[k], r.depth) catch |err| { + b.e.r.enc.end(); + return err; + }; + } + try b.primary.finish(cb); + for (requests) |r| (try b.session(r.stream)).heldDrafts(r.depth); + b.draft_waits += 1; + b.draft_calls += requests.len; + b.draft_seconds += mtl.clock.seconds() - start; + } + fn release(ptr: *anyopaque, stream: *Stream) void { const b = self(ptr); const kv = b.by.fetchRemove(stream) orelse return; diff --git a/zig/src/families/flashnext/batch_costs.zig b/zig/src/families/flashnext/batch_costs.zig new file mode 100644 index 0000000000..fcf148cb29 --- /dev/null +++ b/zig/src/families/flashnext/batch_costs.zig @@ -0,0 +1,103 @@ +//! The lane allocator's prices for shared rounds, timed at load on a free session: a shared forward's ms by rows +//! (the fastest of a few at each width, the widest first while the GPU's clocks ramp up) and a chained head step's ms +//! (a draft past a stream's first, in the one command buffer that takes a round's drafts). +const std = @import("std"); +const lanes = @import("lanes"); +const mtl = @import("metal"); +const fz = @import("replay.zig"); +const forward = @import("batch_forward.zig"); +const Session = @import("session.zig").Session; + +pub const Costs = struct { + shared: [fz.MAXR]lanes.config.Cost = undefined, + step_ms: f64 = 0, // a chained head step +}; + +const PROMPT = 192; // the scratch prompt the windows follow +const TRIES = 4; // the fastest of these at each width +const WARM = 6; // widest windows first + +fn tokenAt(i: usize) u32 { + return @intCast((37 * i + 11) % 50_000 + 1000); +} + +/// The costs timed on `s` (a free session, empty again after), `b` a *Backend. +pub fn time(b: anytype, s: *Session) !Costs { + const e = b.e; + var prompt: [PROMPT]u32 = undefined; + for (&prompt, 0..) |*t, i| t.* = tokenAt(i); + { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const old = try b.bind(s); + defer old.restore(b); + s.m.reset(); + const ps = [1]*fz.Prompt{e.pr}; + _ = try fz.Prompt.chunkN(&ps, &s.m, b.gpa, &prompt); + try e.pr.mtpKeys(&s.m, 0, prompt[1..], e.pr.last); + Session.copy(e.pr.last.at((PROMPT - 1) * fz.WIDE * 2), s.last, fz.WIDE * 2); + s.normalize(); + } + defer s.m.reset(); + var c: Costs = .{}; + for (0..WARM) |_| _ = try windowMs(b, s, fz.MAXR); + for (&c.shared, 0..) |*x, i| { + var best = std.math.inf(f64); + for (0..TRIES) |_| best = @min(best, try windowMs(b, s, i + 1)); + x.* = .{ .width = @intCast(i + 1), .ms = best }; + } + var one = std.math.inf(f64); + var eight = std.math.inf(f64); + for (0..TRIES) |_| { + one = @min(one, try chainMs(b, s, 1)); + eight = @min(eight, try chainMs(b, s, 8)); + } + c.step_ms = @max(0, (eight - one) / 7); + return c; +} + +/// One shared forward of `rows` rows on `s` from its position (nothing kept: the next window overwrites them). +fn windowMs(b: anytype, s: *Session, rows: usize) !f64 { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const t = &b.primary.t; + var toks: [fz.MAXR]u32 = undefined; + for (&toks, 0..) |*x, i| x.* = tokenAt(PROMPT + i); + const t0 = mtl.clock.seconds(); + s.m.windowMeta(rows); + s.m.pleIds(toks[0..rows]); + @memcpy(t.ids8.b.slice(u32, fz.MAXR)[0..rows], toks[0..rows]); + @memcpy(t.ple_ids.b.slice(u32, fz.MAXR * 16)[0 .. rows * 16], s.m.t.ple_ids.b.slice(u32, rows * 16)); + b.outputs(s, 0); + t.rows.b.slice(i32, 1)[0] = @intCast(rows); + t.mdims.b.slice(i32, 2)[0] = @intCast(rows); + const pos8 = t.pos8.b.slice(u32, fz.MAXR); + for (pos8[0..rows], 0..) |*p, j| p.* = @intCast(s.m.pos + j); + const segs = [1]forward.Seg{.{ .s = s, .row0 = 0, .rows = rows }}; + const cb = b.e.r.queue.commandBuffer(); + b.e.r.enc = cb.compute(.serial); + try forward.encode(b.e, &segs, rows, t.ids8); + try b.primary.finish(cb); + return (mtl.clock.seconds() - t0) * 1e3; +} + +/// The head's absorb of one row and `depth` chained drafts in one command buffer, from the prompt's end each time. +fn chainMs(b: anytype, s: *Session, depth: usize) !f64 { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const old = try b.bind(s); + defer old.restore(b); + s.m.mtp.pos = PROMPT - 1; + s.m.mtp.drafted = 0; + const follow = [1]u32{tokenAt(PROMPT)}; + const t0 = mtl.clock.seconds(); + const cb = b.e.r.queue.commandBuffer(); + b.e.r.enc = cb.compute(if (b.e.r.serial) .serial else .concurrent); + s.encodeDrafts(&follow, s.last, depth) catch |err| { + b.e.r.enc.end(); + return err; + }; + try s.m.finish(cb); + s.heldDrafts(depth); + return (mtl.clock.seconds() - t0) * 1e3; +} diff --git a/zig/src/families/flashnext/session.zig b/zig/src/families/flashnext/session.zig index 612b37e434..4b851fddd4 100644 --- a/zig/src/families/flashnext/session.zig +++ b/zig/src/families/flashnext/session.zig @@ -142,6 +142,36 @@ pub const Session = struct { @memcpy(s.held[0..depth], s.draft_ids.b.slice(u32, depth)); } + /// Encode the head's absorb of a kept window and `depth` chained drafts into the open encoder, unsubmitted (one + /// command buffer takes every stream's drafts: heldDrafts reads them once it is done); depth 0 absorbs only. + pub fn encodeDrafts(s: *Session, follow: []const u32, hidden: Buf, depth: usize) !void { + if (follow.len == 0 or follow.len > fz.MAXR or depth >= fz.MAXR) return error.DraftOutOfStep; + const m = &s.m; + const h = &m.mtp; + const r = m.r; + const old_select = r.sel; + defer r.sel = old_select; + h.pos -= h.drafted; + h.drafted = 0; + h.pooled_n = @min(h.pooled_n, h.pos / 4); + @memcpy(h.slots[0].ids8.b.slice(u32, follow.len), follow); + for (0..@max(depth, 1)) |i| { + r.sel = s.draft_select[i]; + const rows = if (i == 0) follow.len else 1; + const input = if (i == 0) h.slots[0].ids8 else s.draft_ids.at((i - 1) * 4); + const streams = if (i == 0) hidden else h.h[1]; + try m.mtpEncode(i, rows, input, streams, s.draft_ids.at(i * 4)); + h.pos += rows; + } + } + + /// encodeDrafts' drafts once their command buffer is done: held, the head's chained rows trimmed at its next absorb. + pub fn heldDrafts(s: *Session, depth: usize) void { + s.m.mtp.drafted = depth -| 1; + @memcpy(s.held[0..depth], s.draft_ids.b.slice(u32, depth)); + s.held_n = depth; + } + pub fn deinit(s: *Session, gpa: std.mem.Allocator) void { for (s.owned.items) |b| b.deinit(); s.owned.deinit(gpa); diff --git a/zig/src/native/flashnext_batch_host.zig b/zig/src/native/flashnext_batch_host.zig index 92a1da42f5..b0deee75e7 100644 --- a/zig/src/native/flashnext_batch_host.zig +++ b/zig/src/native/flashnext_batch_host.zig @@ -58,6 +58,7 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const h.eng = eng; h.back = try Backend.init(gpa, eng, slots, capacity); errdefer h.back.deinit(); + try h.back.timeCosts(); const allocated = fit.footprint() orelse return error.NoMemoryReading; if (allocated +| fit.MARGIN +| reserved > cap) return error.OverMemoryLimit; h.cfg = try lanes.Config.init(gpa, h.back.facts(), tf.flashnext_replay.MAXR, tf.flashnext_replay.MAXR - 1); @@ -79,7 +80,7 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const h.warm = .{ .queue = eng.r.queue }; h.host.keepalive_target = .{ .ctx = &h.warm, .tick = mtl.keepalive.Target.tick }; try h.host.start(); - std.log.info("Flash Next: {d} session slots, shared target forwards, serial MTP drafts", .{slots}); + std.log.info("Flash Next: {d} session slots, shared target forwards, {s}", .{ slots, if (h.back.serial_drafts) "serial MTP drafts" else "every stream's drafts in one command buffer" }); return .{ .engine = h.host.engine(), .close = close, .ctx = h }; } From 5551aba71e589c3076253ea05f3485c0d3711981 Mon Sep 17 00:00:00 2001 From: Seth For Privacy <40500387+sethforprivacy@users.noreply.github.com> Date: Mon, 12 Oct 2026 14:12:44 -0400 Subject: [PATCH 13/31] Flash Next shared sessions sized to the traffic: FLASHNEXT_SLOT_CONTEXTS, and a stream that outgrows its session moves Every session held the whole window, so few fit beside the model (three 262K sessions in 15.4 GiB on a 256 GB Mac). - FLASHNEXT_SLOT_CONTEXTS lists the sessions past the engine's own (token counts, CONTEXTxN for N alike, e.g. "65536,32768x3,16384x11": 16 streams in 11.6 GiB); unset, nothing changes. - A new stream takes the smallest free session that holds its prompt and its reply's next FLASHNEXT_SLOT_GROW tokens (default 8192); LaneHost's admit hook (as on main since #639) holds a request until one is free. - Before a round, a stream left too little room moves to the smallest free larger session that holds its next FLASHNEXT_SLOT_GROW tokens (Session.moveTo: its cache rows, DeltaNet states, n-gram tail, head rows and pooled blocks copied on the GPU at a round's end, states settled first), or pauses (Stream.paused) until one frees. Co-Authored-By: Claude Opus 5.5 --- zig/src/families/flashnext/backend.zig | 107 ++++++++++++++++++++++-- zig/src/families/flashnext/session.zig | 53 ++++++++++++ zig/src/native/flashnext_batch_host.zig | 59 ++++++++++++- 3 files changed, 207 insertions(+), 12 deletions(-) diff --git a/zig/src/families/flashnext/backend.zig b/zig/src/families/flashnext/backend.zig index e5b2c74b58..9a169ade92 100644 --- a/zig/src/families/flashnext/backend.zig +++ b/zig/src/families/flashnext/backend.zig @@ -39,14 +39,18 @@ pub const Backend = struct { draft_calls: u64 = 0, draft_waits: u64 = 0, draft_seconds: f64 = 0, + grow: usize = std.math.maxInt(usize), // reply tokens a session must hold at admission past the prompt (the rest: move) + paused: std.ArrayList(*Stream) = .empty, // streams that outgrew their session and wait for a larger one to free + moves: u64 = 0, - pub fn init(gpa: std.mem.Allocator, e: *Engine, n: u32, capacity: usize) !Backend { - if (n < 2 or n > fz.MAXR or e.r.tp != null) return error.BadBatchSlots; - const sessions = try gpa.alloc(Session, n); + /// Sessions of the given capacities (the first, the engine's own, holds its whole window whatever is given). + pub fn init(gpa: std.mem.Allocator, e: *Engine, capacities: []const usize) !Backend { + if (capacities.len < 2 or capacities.len > fz.MAXR or e.r.tp != null) return error.BadBatchSlots; + const sessions = try gpa.alloc(Session, capacities.len); errdefer gpa.free(sessions); var made: usize = 0; errdefer for (sessions[0..made]) |*s| s.deinit(gpa); - for (sessions, 0..) |*s, i| { + for (sessions, capacities, 0..) |*s, capacity, i| { s.* = try Session.init(gpa, e, capacity, i == 0); made += 1; } @@ -56,6 +60,7 @@ pub const Backend = struct { pub fn deinit(b: *Backend) void { b.e.m = b.primary; + b.paused.deinit(b.gpa); b.by.deinit(b.gpa); for (b.sessions) |*s| s.deinit(b.gpa); b.gpa.free(b.sessions); @@ -129,6 +134,88 @@ pub const Backend = struct { return old; } + /// The free session with the smallest capacity that holds a stream's prompt and its reply's first `grow` tokens + /// (slot 0 holds the longest window: keep it for those that need it; a stream that outgrows its session moves: + /// makeRoom); error.LaneMemory when only a busy one would (the stream waits), ContextFull when none holds it whole. + fn slotFor(b: *Backend, prompt: usize, max_new: usize) !usize { + const need = prompt + @min(max_new, b.grow) + fz.MAXR + 1; + var best: ?usize = null; + var most: usize = 0; + for (b.sessions, 0..) |s, j| { + most = @max(most, s.m.cap); + if (s.m.cap >= need and !s.used and (best == null or s.m.cap < b.sessions[best.?].m.cap)) best = j; + } + if (prompt + max_new + fz.MAXR + 1 > most) return error.ContextFull; + return best orelse error.LaneMemory; + } + + /// Before a round: a stream left too little room for the next one moves to a free larger session; with none free + /// it pauses after this round (its window fits) until one frees (release). + fn makeRoom(b: *Backend, stream: *Stream, rows: usize) !void { + const i = b.by.get(stream) orelse return error.UnknownStream; + const s = &b.sessions[i]; + if (s.m.pos + rows + 2 * (fz.MAXR + 1) <= s.m.cap) return; + b.move(stream, i) catch |err| switch (err) { + error.LaneMemory => { + if (s.m.pos + rows + fz.MAXR + 1 > s.m.cap) return error.ContextFull; + if (!stream.paused) { + try b.paused.append(b.gpa, stream); + std.log.info("flash next: a stream at {d} tokens waits for a session larger than {d} tokens to free", .{ s.m.pos, s.m.cap }); + } + stream.paused = true; + }, + else => return err, + }; + } + + /// The stream in session `i` into the smallest free larger session that holds its reply's next `grow` tokens (the + /// largest free one when none does); LaneMemory when no larger session is free. + fn move(b: *Backend, stream: *Stream, i: usize) !void { + const from = &b.sessions[i]; + const left: usize = @intCast(@max(stream.budgetLeft(), 0)); + const need = from.m.pos + @min(left, b.grow) + 2 * (fz.MAXR + 1); + var best: ?usize = null; // the smallest free larger session that holds `need` + var most: ?usize = null; // else the largest free larger one + for (b.sessions, 0..) |o, j| if (!o.used and o.m.cap > from.m.cap) { + if (o.m.cap >= need and (best == null or o.m.cap < b.sessions[best.?].m.cap)) best = j; + if (most == null or o.m.cap > b.sessions[most.?].m.cap) most = j; + }; + const j = best orelse most orelse return error.LaneMemory; + try b.settle(); + try from.moveTo(&b.sessions[j], b.e); + try b.by.put(b.gpa, stream, j); + from.used = false; + from.round = .{}; + from.held_n = 0; + b.moves += 1; + std.log.info("flash next: a stream at {d} tokens moved from a {d}-token session to a {d}-token one ({d} moves)", .{ from.m.pos, from.m.cap, b.sessions[j].m.cap, b.moves }); + } + + /// Paused streams into sessions that freed, oldest first; each goes on once moved. + fn resumePaused(b: *Backend) void { + var k: usize = 0; + while (k < b.paused.items.len) { + const stream = b.paused.items[k]; + const i = b.by.get(stream) orelse { + _ = b.paused.orderedRemove(k); + continue; + }; + b.move(stream, i) catch |err| { + if (err != error.LaneMemory) std.log.err("flash next: a paused stream could not move: {s}", .{@errorName(err)}); + k += 1; + continue; + }; + stream.paused = false; + _ = b.paused.orderedRemove(k); + } + } + + /// LaneHost's admission check: a free session holds the request now. + pub fn fits(ptr: *anyopaque, prompt: u32, max_new: u32) bool { + _ = self(ptr).slotFor(prompt, max_new) catch return false; + return true; + } + fn prefill(ptr: *anyopaque, stream: *Stream) anyerror!void { const pool = mtl.objc.Pool.push(); defer pool.pop(); @@ -136,12 +223,13 @@ pub const Backend = struct { const prompt = stream.prompt(); if (prompt.len == 0) return error.EmptyPrompt; try b.settle(); - const i = for (b.sessions, 0..) |s, j| { - if (!s.used) break j; - } else return error.NoFreeSlot; + const i = try b.slotFor(prompt.len, stream.max_new); const s = &b.sessions[i]; - if (prompt.len + stream.max_new + fz.MAXR + 1 > s.m.cap) return error.ContextFull; try b.by.put(b.gpa, stream, i); + errdefer { + _ = b.by.remove(stream); + s.used = false; + } s.used = true; s.held_n = 0; s.round = .{}; @@ -221,6 +309,7 @@ pub const Backend = struct { const b = self(ptr); try b.settle(); if (windows.len == 0 or windows.len > fz.MAXR or out.len != windows.len) return error.BadBatch; + for (windows) |w| try b.makeRoom(w.stream, w.rows()); var desc: [fz.MAXR]meta.Window = undefined; var segs: [fz.MAXR]forward.Seg = undefined; for (windows, out, 0..) |w, o, i| { @@ -318,10 +407,12 @@ pub const Backend = struct { fn release(ptr: *anyopaque, stream: *Stream) void { const b = self(ptr); + if (std.mem.indexOfScalar(*Stream, b.paused.items, stream)) |k| _ = b.paused.orderedRemove(k); const kv = b.by.fetchRemove(stream) orelse return; b.sessions[kv.value].used = false; b.sessions[kv.value].round = .{}; b.sessions[kv.value].held_n = 0; + if (b.paused.items.len > 0) b.resumePaused(); } pub fn snapBytes(ptr: *anyopaque, at: u32) u64 { diff --git a/zig/src/families/flashnext/session.zig b/zig/src/families/flashnext/session.zig index 612b37e434..bb17c23772 100644 --- a/zig/src/families/flashnext/session.zig +++ b/zig/src/families/flashnext/session.zig @@ -142,6 +142,59 @@ pub const Session = struct { @memcpy(s.held[0..depth], s.draft_ids.b.slice(u32, depth)); } + /// This session's stream into `dst`, a free session with more room: its cache rows, states, n-gram tail and head + /// copied on the GPU at a round's end (states settled), so the stream goes on there with the same bits. + pub fn moveTo(s: *Session, dst: *Session, e: *Engine) !void { + const m = &s.m; + const d = &dst.m; + if (m.state != 0 or m.state_row != 0 or s.round.pending) return error.StateNotSettled; + const rows = @min(m.cap, m.pos + 2 * fz.MAXR + 2); // the cache's rows and the head's chained ones past them + if (rows > d.cap) return error.ContextFull; + const cb = e.r.queue.commandBuffer(); + const blit = cb.blit(); + for (&m.layers, &d.layers) |*a, *b| if (a.linear) { + blit.copy(a.cs[0].b, a.cs[0].off, b.cs[0].b, b.cs[0].off, fz.CS_ROW); + blit.copy(a.so[0].b, a.so[0].off, b.so[0].b, b.so[0].off, fz.SO_ROW); + } else { + moveRows(blit, a, b, m.cap, d.cap, rows); + }; + moveRows(blit, &m.mtp, &d.mtp, m.cap, d.cap, rows); + blit.copy(m.ple.cin.b, m.ple.cin.off, d.ple.cin.b, d.ple.cin.off, (fz.PLE_TAIL + fz.MAXR) * fz.WIDE * 2); + blit.copy(s.last.b, s.last.off, dst.last.b, dst.last.off, fz.WIDE * 2); + blit.end(); + cb.commit(); + cb.wait(); + if (cb.failure()) |message| { + std.log.err("session move failed: {s}", .{message}); + return error.GpuFailed; + } + d.pos = m.pos; + d.state = 0; + d.state_row = 0; + d.ple.hist = m.ple.hist; + d.mtp.pos = m.mtp.pos; + d.mtp.drafted = m.mtp.drafted; + dst.held = s.held; + dst.held_n = s.held_n; + dst.first = s.first; + dst.round = s.round; + dst.tokens = s.tokens; + dst.row0 = s.row0; + dst.used = true; + } + + /// A sparse-attention layer's first `rows` keys, values and indexer keys and its pooled blocks, head by head. + fn moveRows(blit: mtl.BlitEncoder, a: anytype, b: anytype, from: usize, to: usize, rows: usize) void { + const row = 256 * 2; + for (0..2) |hd| { + blit.copy(a.keys.b, a.keys.off + hd * from * row, b.keys.b, b.keys.off + hd * to * row, rows * row); + blit.copy(a.vals.b, a.vals.off + hd * from * row, b.vals.b, b.vals.off + hd * to * row, rows * row); + } + blit.copy(a.raw.b, a.raw.off, b.raw.b, b.raw.off, rows * 128 * 2); + if (a.pooled_n > 0) blit.copy(a.pooled.b, a.pooled.off, b.pooled.b, b.pooled.off, a.pooled_n * 128 * 2); + b.pooled_n = a.pooled_n; + } + pub fn deinit(s: *Session, gpa: std.mem.Allocator) void { for (s.owned.items) |b| b.deinit(); s.owned.deinit(gpa); diff --git a/zig/src/native/flashnext_batch_host.zig b/zig/src/native/flashnext_batch_host.zig index 92a1da42f5..9c221ae879 100644 --- a/zig/src/native/flashnext_batch_host.zig +++ b/zig/src/native/flashnext_batch_host.zig @@ -34,6 +34,33 @@ fn ceiling(total: u64, working_set: u64) error{BadLoadLimit}!u64 { return c.bytes; } +/// The sessions past the engine's own from FLASHNEXT_SLOT_CONTEXTS: token counts, `CONTEXTxN` for N sessions of one +/// size (as "131072,65536x2,16384x8"), each holding at most `capacity` rows; null when unset. +fn slotContexts(gpa: std.mem.Allocator, capacity: usize) !?[]usize { + const text = std.mem.trim(u8, std.mem.span(std.c.getenv("FLASHNEXT_SLOT_CONTEXTS") orelse return null), " \t"); + if (text.len == 0) return null; + var caps: std.ArrayList(usize) = .empty; + errdefer caps.deinit(gpa); + var it = std.mem.tokenizeScalar(u8, text, ','); + while (it.next()) |entry| { + const e = std.mem.trim(u8, entry, " \t"); + const x = std.mem.indexOfScalar(u8, e, 'x'); + const tokens = std.fmt.parseInt(usize, if (x) |i| e[0..i] else e, 10) catch return error.BadSlotContexts; + const n = if (x) |i| std.fmt.parseInt(usize, e[i + 1 ..], 10) catch return error.BadSlotContexts else 1; + if (tokens < 1024 or n == 0) return error.BadSlotContexts; + for (0..n) |_| try caps.append(gpa, @min(capacity, tokens + @as(usize, @intCast(fx.MARGIN)))); + } + if (caps.items.len == 0 or caps.items.len + 1 > tf.flashnext_replay.MAXR) return error.BadSlotContexts; + return try caps.toOwnedSlice(gpa); +} + +/// Reply tokens a stream's session must hold past its prompt at admission (FLASHNEXT_SLOT_GROW, default 8192): a +/// stream that outgrows it moves to a free larger session at a round's end, or waits for one. +fn slotGrow() usize { + const v = std.c.getenv("FLASHNEXT_SLOT_GROW") orelse return 8192; + return std.fmt.parseInt(usize, std.mem.span(v), 10) catch 8192; +} + pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const u8, window: i64, streams: u32, cache_gib: ?f64, over: bool, a: std.mem.Allocator, why: *[]const u8) !api.Opened { const pool = mtl.objc.Pool.push(); defer pool.pop(); @@ -51,13 +78,34 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const const available = cap -| ready -| fit.MARGIN -| reserved; if (available < fixed) return error.OverMemoryLimit; const room = available - fixed; - const slots = try tf.flashnext_batch_meta.fit(streams, true, room, Session.bytes(capacity)); + const tiers = slotContexts(gpa, capacity) catch |err| { + std.log.err("flash next: FLASHNEXT_SLOT_CONTEXTS is not a list of sessions: token counts of 1024 or more, CONTEXTxN for N alike, at most {d} sessions past the engine's own", .{tf.flashnext_replay.MAXR - 1}); + return err; + }; + defer if (tiers) |t| gpa.free(t); + var caps: std.ArrayList(usize) = .empty; + defer caps.deinit(gpa); + try caps.append(gpa, capacity); // the engine's own session: the whole window + if (tiers) |t| { + var bytes: u64 = 0; + for (t) |c| bytes += Session.bytes(c); + if (bytes > room) { + std.log.err("flash next: the sessions FLASHNEXT_SLOT_CONTEXTS lists take {d:.1} GiB; {d:.1} GiB fits under the limit beside the prompt cache", .{ fit.gibs(bytes), fit.gibs(room) }); + return error.OverMemoryLimit; + } + try caps.appendSlice(gpa, t); + } else { + const n = try tf.flashnext_batch_meta.fit(streams, true, room, Session.bytes(capacity)); + for (1..n) |_| try caps.append(gpa, capacity); + } + const slots: u32 = @intCast(caps.items.len); const h = try gpa.create(Host); errdefer gpa.destroy(h); h.gpa = gpa; h.eng = eng; - h.back = try Backend.init(gpa, eng, slots, capacity); + h.back = try Backend.init(gpa, eng, caps.items); errdefer h.back.deinit(); + if (tiers != null) h.back.grow = slotGrow(); const allocated = fit.footprint() orelse return error.NoMemoryReading; if (allocated +| fit.MARGIN +| reserved > cap) return error.OverMemoryLimit; h.cfg = try lanes.Config.init(gpa, h.back.facts(), tf.flashnext_replay.MAXR, tf.flashnext_replay.MAXR - 1); @@ -67,7 +115,7 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const errdefer h.core.deinit(); const plan = try fit.budgetUnder(cap, cache_gib, over, 0, a, why, ""); // the cache under the same limit as the sessions eng.snap_pool.max = plan.budget_bytes; - h.host = api.LaneHost.init(gpa, io, &h.core, .{ .name = "flashnext-zig", .lanes = slots, .context_window = @intCast(window), .prefill_step = @intCast(eng.pr.step), .prompt_cache_plan = plan }); + h.host = api.LaneHost.init(gpa, io, &h.core, .{ .name = "flashnext-zig", .lanes = slots, .context_window = @intCast(window), .prefill_step = @intCast(eng.pr.step), .prompt_cache_plan = plan, .admit = .{ .ctx = &h.back, .fits = Backend.fits } }); h.cache = null; if (plan.budget_bytes > 0) { h.cache = api.prompt_cache.Store.init(gpa, .{ .ptr = &h.back, .vtable = &.{ .bytes = Backend.snapBytes, .save = Backend.snapSave, .restore = Backend.snapRestore, .drop = Backend.snapDrop, .charged = Backend.snapCharged, .spare = Backend.snapSpare, .trim = Backend.snapTrim, .reuses = Backend.snapReuses } }, .{ .lookahead = 1, .planned = true }, plan.budget_bytes); @@ -79,7 +127,10 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const h.warm = .{ .queue = eng.r.queue }; h.host.keepalive_target = .{ .ctx = &h.warm, .tick = mtl.keepalive.Target.tick }; try h.host.start(); - std.log.info("Flash Next: {d} session slots, shared target forwards, serial MTP drafts", .{slots}); + var listed: [tf.flashnext_replay.MAXR * 8]u8 = undefined; + var w: std.Io.Writer = .fixed(&listed); + for (caps.items, 0..) |c, i| w.print("{s}{d}", .{ if (i == 0) "" else ",", c }) catch break; + std.log.info("Flash Next: {d} session slots ({s} tokens), shared target forwards, serial MTP drafts", .{ slots, w.buffered() }); return .{ .engine = h.host.engine(), .close = close, .ctx = h }; } From a3085be24fae3518be1493da2dcca664eb0cd24b Mon Sep 17 00:00:00 2001 From: Seth For Privacy <40500387+sethforprivacy@users.noreply.github.com> Date: Mon, 12 Oct 2026 14:14:39 -0400 Subject: [PATCH 14/31] Flash Next: a lone request on the engine's GPU-side rounds, handed to the shared rounds when a second arrives One stream through the shared rounds decodes slower than the single-reply driver: a host-driven round each window. The batch host now gives LaneHost a lone driver (api.Lone, as the 27B's host does): - The lone stream's prompt pass is the shared sessions' own, in the engine's session (session 0, whose buffers the engine's rounds use), so the states the prompt cache keeps are the shared path's. - It then decodes on the engine's GPU-side rounds: generateFrom's decode, split out as Engine.decode, two rounds in flight. - When LaneHost's yield hook asks (a request waits, or a cancel), the loop stops after a landed round; the round ahead has already started as rounds do (the kept DeltaNet states, the n-gram tail, the head's absorb of the last window), so the session's host fields are set to match (position, history, head position, pooled counts clamped) and the lane core adopts the stream: its next window starts at the cache's end with nothing held. - FLASHNEXT_NO_LONE=1 serves every request through the shared rounds. The lone driver takes greedy streams only (api.Lone.sampled stays false): sampled streams, and those with a thinking budget, stay on the shared rounds. Co-Authored-By: Claude Opus 5.5 --- zig/src/families/flashnext/backend.zig | 94 ++++++++++++++++++++++++- zig/src/families/flashnext/engine.zig | 36 +++++++++- zig/src/native/flashnext_batch_host.zig | 9 ++- 3 files changed, 133 insertions(+), 6 deletions(-) diff --git a/zig/src/families/flashnext/backend.zig b/zig/src/families/flashnext/backend.zig index e5b2c74b58..e456599811 100644 --- a/zig/src/families/flashnext/backend.zig +++ b/zig/src/families/flashnext/backend.zig @@ -13,6 +13,7 @@ const Buf = fz.Buf; const mtl = @import("metal"); const segments = @import("../../core/segments.zig"); const sample = @import("sample.zig"); +const think = @import("think.zig"); /// A stream's sampling when it samples (a temperature above 0); null: the GPU's argmax stands. fn sampling(stream: *const Stream) ?lanes.Sampling { @@ -39,6 +40,7 @@ pub const Backend = struct { draft_calls: u64 = 0, draft_waits: u64 = 0, draft_seconds: f64 = 0, + lone_handoffs: u64 = 0, // lone streams the engine's rounds handed to the shared ones pub fn init(gpa: std.mem.Allocator, e: *Engine, n: u32, capacity: usize) !Backend { if (n < 2 or n > fz.MAXR or e.r.tp != null) return error.BadBatchSlots; @@ -132,13 +134,18 @@ pub const Backend = struct { fn prefill(ptr: *anyopaque, stream: *Stream) anyerror!void { const pool = mtl.objc.Pool.push(); defer pool.pop(); - const b = self(ptr); + try self(ptr).fill(stream, null); + } + + /// A stream's whole prompt pass, in session `slot` when given (free), else the first free one. + fn fill(b: *Backend, stream: *Stream, slot: ?usize) !void { const prompt = stream.prompt(); if (prompt.len == 0) return error.EmptyPrompt; try b.settle(); - const i = for (b.sessions, 0..) |s, j| { + const i = slot orelse for (b.sessions, 0..) |s, j| { if (!s.used) break j; } else return error.NoFreeSlot; + if (b.sessions[i].used) return error.NoFreeSlot; const s = &b.sessions[i]; if (prompt.len + stream.max_new + fz.MAXR + 1 > s.m.cap) return error.ContextFull; try b.by.put(b.gpa, stream, i); @@ -194,6 +201,89 @@ pub const Backend = struct { s.m.mtp.drafted = 0; } + /// LaneHost's driver for a lone stream: the shared sessions' own prompt pass into the engine's session (the states + /// the prompt cache keeps are theirs), then the engine's GPU-side rounds, two deep, until the stream finishes + /// (false) or `hooks.yield` asks for it: its state then stands as a shared round leaves it and the lane core takes + /// it on (true). `hooks`: LaneHost's LoneHooks. + pub fn lone(b: *Backend, stream: *Stream, hooks: anytype) !bool { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const prompt = stream.prompt(); + const s = &b.sessions[0]; // the GPU-side rounds run on the engine's own buffers + if (s.used) return error.NoFreeSlot; + if (prompt.len + stream.max_new + @import("engine.zig").MARGIN > s.m.cap) return error.ContextFull; + var handed = false; + defer if (!handed) release(b, stream); + try b.fill(stream, 0); + _ = try stream.commit(b.gpa, &.{s.first}, &.{}); + hooks.committed(hooks.ctx); + if (stream.finished) return false; + const Hooks = @TypeOf(hooks); + const Ctx = struct { + gpa: std.mem.Allocator, + stream: *Stream, + hooks: Hooks, + failed: ?anyerror = null, + fn prefilled(_: *anyopaque) void {} + fn tokens(p: *anyopaque, toks: []const u32) bool { + const c: *@This() = @ptrCast(@alignCast(p)); + _ = c.stream.commit(c.gpa, toks, &.{}) catch |err| { + c.failed = err; + return true; + }; + c.hooks.committed(c.hooks.ctx); + return c.stream.finished; + } + fn cancelled(p: *anyopaque) bool { // a request waits, or one is cancelled: the lane core takes over + const c: *@This() = @ptrCast(@alignCast(p)); + return c.hooks.yield(c.hooks.ctx); + } + }; + var c: Ctx = .{ .gpa = b.gpa, .stream = stream, .hooks = hooks }; + var ho: Engine.Handoff = .{}; + { + const old = try b.bind(s); + defer old.restore(b); + outputs(b, s, 0); + var watch = try think.Watch.init(.{}); // no thinking budget: lone_host_lone.fits keeps those in the lane core + const res = try b.e.decode(prompt, s.first, s.last, stream.max_new, stream.eos, null, &watch, .{ .ctx = &c, .prefilled = Ctx.prefilled, .tokens = Ctx.tokens, .cancelled = Ctx.cancelled }, &ho); + if (c.failed) |err| return err; + stream.rounds += res.rounds; + stream.drafted += res.drafted; + stream.accepted += res.accepted; + if (res.rounds > 0) stream.min_rows = if (stream.min_rows == 0) res.min_rows else @min(stream.min_rows, res.min_rows); + } + if (!ho.yielded) { + if (!stream.finished) { // the rounds ended where the stream did not (eos or length by the engine's count) + stream.finished = true; + stream.reason = .stop; + } + return false; + } + const ctx = stream.context.items; + const at = ho.cache; // the cache's rows: every committed token but the last, the next window's first + if (at + 1 != ctx.len or at < 2) return error.HandoffOutOfStep; + s.m.pos = at; + s.m.state = 0; + s.m.state_row = 0; + s.m.ple.hist = .{ ctx[at - 2], ctx[at - 1] }; + s.m.mtp.pos = at; + s.m.mtp.drafted = 0; + s.m.mtp.pooled_n = @min(s.m.mtp.pooled_n, at / 4); // blocks the rounds pooled are pooled again here + for (&s.m.layers) |*l| if (!l.linear) { + l.pooled_n = @min(l.pooled_n, at / 4); + }; + s.held_n = 0; + s.round = .{}; + stream.cache_len = at; + stream.pending = ctx[at]; + stream.next = null; + handed = true; + b.lone_handoffs += 1; + std.log.info("flash next: a lone stream at {d} tokens handed to the shared rounds ({d} handoffs)", .{ at, b.lone_handoffs }); + return true; + } + fn first(ptr: *anyopaque, stream: *Stream, position: u64) anyerror!u64 { const s = try self(ptr).session(stream); if (position != s.m.pos) return error.PositionMismatch; diff --git a/zig/src/families/flashnext/engine.zig b/zig/src/families/flashnext/engine.zig index d588ec1b32..ce6f6aef95 100644 --- a/zig/src/families/flashnext/engine.zig +++ b/zig/src/families/flashnext/engine.zig @@ -617,12 +617,27 @@ pub const Engine = struct { } out.prefilled(out.ctx); pick = watch.first(pick); - var res: Result = .{ .reason = .length }; const first_stop = out.tokens(out.ctx, &.{pick}) or isEos(eos, pick); try dr.flush(e, out); // after the first token: the slots, rows and tail stay put till the rounds if (e.call_log) calls.log(if (r.tp) |tp| tp.rank else 0); if (try e.agree(first_stop)) return .{ .reason = .stop }; - if (max_tokens <= 1) return res; + if (max_tokens <= 1) return .{ .reason = .length }; + return e.decode(prompt, pick, .{ .b = m.last.b, .off = m.last.off + (last_n - 1) * WIDE * 2 }, max_tokens, eos, depth, &watch, out, null); + } + + /// Where `decode` left a stream it handed over (a cancel with a handoff): its state as a host round leaves it + /// (the DeltaNet states kept, the n-gram tail in the model's own buffer, the head absorbed through the last + /// cache row), `cache` rows in the cache. + pub const Handoff = struct { yielded: bool = false, cache: usize = 0 }; + + /// The GPU-side rounds after a prompt pass: `prompt` in the cache, `pick` its first token (out has it), `streams` + /// the last prompt row's (the head's first drafts), `watch` the reply's thinking budget. With `handoff`, a cancel + /// hands the stream over (handOver). + pub fn decode(e: *Engine, prompt: []const u32, pick: u32, streams: Buf, max_tokens: usize, eos: []const u32, depth: ?usize, watch: *think.Watch, out: Out, handoff: ?*Handoff) !Result { + const r = e.r; + const m = e.m; + defer e.hostMode(); + var res: Result = .{ .reason = .length }; var emitted: usize = 1; const ar = r.ar.b.slice(i32, fz.AR_WORDS); if (m.state == 1) { // the state into the rounds' state buffers @@ -644,7 +659,7 @@ pub const Engine = struct { w[0] = pick; if (depth0 > 0) { // the head: the last prompt row with the first token, then its chain m.mtp.pos = prompt.len - 1; - w[1] = try m.mtpRun(&.{pick}, .{ .b = m.last.b, .off = m.last.off + (last_n - 1) * WIDE * 2 }); + w[1] = try m.mtpRun(&.{pick}, streams); for (2..depth0 + 1) |j| w[j] = try m.mtpChain(w[j - 1]); } const W0 = depth0 + 1; @@ -692,6 +707,7 @@ pub const Engine = struct { res.min_rows = @intCast(W0); const rg = e.ring.slice(u32, fz.RING_WORDS * RING); var failed: ?anyerror = null; + var kv_len: usize = m.pos; // the cache's rows after the rounds landed so far while (true) { const wr = widths[round % 4]; const cb = r.queue.commandBuffer(); @@ -779,6 +795,7 @@ pub const Engine = struct { if (learn) res.accepted += keep - 1; res.min_rows = @min(res.min_rows, @as(u32, @intCast(wd))); done += 1; + kv_len += keep; var take: usize = 0; var stop = false; while (take < got.len and emitted + take < max_tokens) { @@ -808,9 +825,22 @@ pub const Engine = struct { }; r.event_value = base + round + 1; if (failed) |err| return err; + if (handoff) |h| if (res.reason == .cancelled) try e.handOver(h, round, kv_len); return res; } + /// The rounds handed over at `cache` rows, `round` the next round's index: the round ahead (greedy rounds run two + /// deep) started as each round does, with the last landed round's kept states, n-gram tail and head absorb; past + /// that nothing a host round reads. Then the tail into the model's own buffer. + fn handOver(e: *Engine, h: *Handoff, round: usize, cache: usize) !void { + if (e.r.gdn_kept != null) return error.HandoffUnsupported; // kept states replay their rows in the next window + if ((round - 1) % 2 == 1) { + const n = PLE_TAIL * WIDE * 2; + @memcpy(e.cins[0].b.contents()[e.cins[0].off..][0..n], e.cins[1].b.contents()[e.cins[1].off..][0..n]); + } + h.* = .{ .yielded = true, .cache = cache }; + } + /// Speed-up mode: whether to stop at a step both ranks take (rank 0 decides, rank 1 follows); alone, `quit`. fn agree(e: *Engine, quit: bool) !bool { return if (e.r.tp) |tp| (try tp.agree(quit, 0)).quit else quit; diff --git a/zig/src/native/flashnext_batch_host.zig b/zig/src/native/flashnext_batch_host.zig index 92a1da42f5..027ebadedb 100644 --- a/zig/src/native/flashnext_batch_host.zig +++ b/zig/src/native/flashnext_batch_host.zig @@ -68,6 +68,8 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const const plan = try fit.budgetUnder(cap, cache_gib, over, 0, a, why, ""); // the cache under the same limit as the sessions eng.snap_pool.max = plan.budget_bytes; h.host = api.LaneHost.init(gpa, io, &h.core, .{ .name = "flashnext-zig", .lanes = slots, .context_window = @intCast(window), .prefill_step = @intCast(eng.pr.step), .prompt_cache_plan = plan }); + // a lone greedy request runs on the engine's GPU-side rounds until a second one arrives (FLASHNEXT_NO_LONE=1: shared rounds) + if (eng.r.gdn_kept == null and std.c.getenv("FLASHNEXT_NO_LONE") == null) h.host.lone = .{ .ctx = h, .run = loneRun }; h.cache = null; if (plan.budget_bytes > 0) { h.cache = api.prompt_cache.Store.init(gpa, .{ .ptr = &h.back, .vtable = &.{ .bytes = Backend.snapBytes, .save = Backend.snapSave, .restore = Backend.snapRestore, .drop = Backend.snapDrop, .charged = Backend.snapCharged, .spare = Backend.snapSpare, .trim = Backend.snapTrim, .reuses = Backend.snapReuses } }, .{ .lookahead = 1, .planned = true }, plan.budget_bytes); @@ -79,10 +81,15 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const h.warm = .{ .queue = eng.r.queue }; h.host.keepalive_target = .{ .ctx = &h.warm, .tick = mtl.keepalive.Target.tick }; try h.host.start(); - std.log.info("Flash Next: {d} session slots, shared target forwards, serial MTP drafts", .{slots}); + std.log.info("Flash Next: {d} session slots, shared target forwards, serial MTP drafts{s}", .{ slots, if (h.host.lone != null) "; a lone request on the engine's own rounds" else "" }); return .{ .engine = h.host.engine(), .close = close, .ctx = h }; } +fn loneRun(ctx: *anyopaque, s: *lanes.Stream, hooks: api.LoneHooks) anyerror!bool { + const h: *Host = @ptrCast(@alignCast(ctx)); + return h.back.lone(s, hooks); +} + fn words(_: ?*anyopaque, err: anyerror) ?[]const u8 { return switch (err) { error.ContextFull => "the prompt and max_tokens do not fit this server's --context", From ba31b91847c79d51d52664ce2a8d20f80cf62e55 Mon Sep 17 00:00:00 2001 From: sethforprivacy Date: Tue, 13 Oct 2026 13:10:17 -0400 Subject: [PATCH 15/31] Flash Next sized sessions: grow on the reply's remaining work, never wait for a session that doesn't exist; old path when unset Review fixes (cwschroeder on cwschroeder/TensorFold#10): - slots.room decides before each round from the reply's end (prompt + max_new + a window): a stream whose session holds it is never moved or paused, so one in the largest session (admission's guarantee) runs to its end however many drafts are rejected near the limit. A stream that does need more room moves while a larger session is free, else pauses on a round that still fits; it only waits when a larger session exists, and the waits go up in size, so a paused stream resumes once that session frees. - slots.parse validates each repetition count against the sessions still allowed before appending, and adds the margin with checked arithmetic (an overflowing count or repeat is refused, nothing allocated for it). - FLASHNEXT_SLOT_CONTEXTS unset keeps the old path: the first free session (ContextFull when it can't hold the reply), no admission hook, no growth checks. - Tests (slots.zig, in the core test root): the list's parsing, overflow and session-count refusals; session choice unsized (first free) and sized (smallest that holds the first grow); near-limit rounds with every draft but one rejected in a session that holds the reply (never pauses); a smaller session moving while a larger one is free and pausing on a round that fits otherwise; and no wait when no session is larger. --- zig/src/families/flashnext/backend.zig | 48 ++++++----- zig/src/families/flashnext/session.zig | 3 +- zig/src/families/flashnext/slots.zig | 103 ++++++++++++++++++++++++ zig/src/native/flashnext_batch_host.zig | 27 ++----- zig/src/tensorfold.zig | 2 + 5 files changed, 143 insertions(+), 40 deletions(-) create mode 100644 zig/src/families/flashnext/slots.zig diff --git a/zig/src/families/flashnext/backend.zig b/zig/src/families/flashnext/backend.zig index 9a169ade92..62ddcd8629 100644 --- a/zig/src/families/flashnext/backend.zig +++ b/zig/src/families/flashnext/backend.zig @@ -12,6 +12,7 @@ const Stream = lanes.Stream; const Buf = fz.Buf; const mtl = @import("metal"); const segments = @import("../../core/segments.zig"); +const slots = @import("slots.zig"); const sample = @import("sample.zig"); /// A stream's sampling when it samples (a temperature above 0); null: the GPU's argmax stands. @@ -39,6 +40,7 @@ pub const Backend = struct { draft_calls: u64 = 0, draft_waits: u64 = 0, draft_seconds: f64 = 0, + sized: bool = false, // FLASHNEXT_SLOT_CONTEXTS set: sessions of several sizes; unset, every stream takes the first free one and none grows grow: usize = std.math.maxInt(usize), // reply tokens a session must hold at admission past the prompt (the rest: move) paused: std.ArrayList(*Stream) = .empty, // streams that outgrew their session and wait for a larger one to free moves: u64 = 0, @@ -134,46 +136,54 @@ pub const Backend = struct { return old; } - /// The free session with the smallest capacity that holds a stream's prompt and its reply's first `grow` tokens - /// (slot 0 holds the longest window: keep it for those that need it; a stream that outgrows its session moves: - /// makeRoom); error.LaneMemory when only a busy one would (the stream waits), ContextFull when none holds it whole. + /// A new stream's session (slots.pick); LaneMemory while only a busy one holds it (it waits), ContextFull when none can. fn slotFor(b: *Backend, prompt: usize, max_new: usize) !usize { - const need = prompt + @min(max_new, b.grow) + fz.MAXR + 1; - var best: ?usize = null; + var caps: [fz.MAXR]usize = undefined; + var used: [fz.MAXR]bool = undefined; var most: usize = 0; for (b.sessions, 0..) |s, j| { + caps[j] = s.m.cap; + used[j] = s.used; most = @max(most, s.m.cap); - if (s.m.cap >= need and !s.used and (best == null or s.m.cap < b.sessions[best.?].m.cap)) best = j; } - if (prompt + max_new + fz.MAXR + 1 > most) return error.ContextFull; - return best orelse error.LaneMemory; + const n = b.sessions.len; + const whole = prompt + max_new + slots.WINDOW; + const i = slots.pick(caps[0..n], used[0..n], b.sized, prompt, max_new, b.grow) orelse + return if (!b.sized) error.NoFreeSlot else if (whole > most) error.ContextFull else error.LaneMemory; + if (whole > (if (b.sized) most else caps[i])) return error.ContextFull; // sized, a smaller one grows into the largest + return i; } - /// Before a round: a stream left too little room for the next one moves to a free larger session; with none free - /// it pauses after this round (its window fits) until one frees (release). + /// Before a round, sized only: a stream its session cannot hold to its reply's end moves to a free larger one, or pauses (slots.room). fn makeRoom(b: *Backend, stream: *Stream, rows: usize) !void { + if (!b.sized) return; const i = b.by.get(stream) orelse return error.UnknownStream; const s = &b.sessions[i]; - if (s.m.pos + rows + 2 * (fz.MAXR + 1) <= s.m.cap) return; - b.move(stream, i) catch |err| switch (err) { - error.LaneMemory => { - if (s.m.pos + rows + fz.MAXR + 1 > s.m.cap) return error.ContextFull; + var most: usize = 0; + var larger = false; + for (b.sessions) |o| { + most = @max(most, o.m.cap); + larger = larger or (!o.used and o.m.cap > s.m.cap); + } + switch (slots.room(s.m.cap, s.m.pos, rows, stream.prompt().len + stream.max_new + slots.WINDOW, most, larger)) { + .ok => {}, + .move => try b.move(stream, i), + .pause => { if (!stream.paused) { try b.paused.append(b.gpa, stream); std.log.info("flash next: a stream at {d} tokens waits for a session larger than {d} tokens to free", .{ s.m.pos, s.m.cap }); } stream.paused = true; }, - else => return err, - }; + .full => return error.ContextFull, + } } - /// The stream in session `i` into the smallest free larger session that holds its reply's next `grow` tokens (the - /// largest free one when none does); LaneMemory when no larger session is free. + /// The stream in session `i` into the smallest free larger session holding its next `grow` tokens, else the largest free larger one. fn move(b: *Backend, stream: *Stream, i: usize) !void { const from = &b.sessions[i]; const left: usize = @intCast(@max(stream.budgetLeft(), 0)); - const need = from.m.pos + @min(left, b.grow) + 2 * (fz.MAXR + 1); + const need = from.m.pos + @min(left, b.grow) + 2 * slots.WINDOW; var best: ?usize = null; // the smallest free larger session that holds `need` var most: ?usize = null; // else the largest free larger one for (b.sessions, 0..) |o, j| if (!o.used and o.m.cap > from.m.cap) { diff --git a/zig/src/families/flashnext/session.zig b/zig/src/families/flashnext/session.zig index bb17c23772..9a815e404c 100644 --- a/zig/src/families/flashnext/session.zig +++ b/zig/src/families/flashnext/session.zig @@ -142,8 +142,7 @@ pub const Session = struct { @memcpy(s.held[0..depth], s.draft_ids.b.slice(u32, depth)); } - /// This session's stream into `dst`, a free session with more room: its cache rows, states, n-gram tail and head - /// copied on the GPU at a round's end (states settled), so the stream goes on there with the same bits. + /// This session's stream into `dst`, a free larger session: cache rows, states, n-gram tail and head copied on the GPU. pub fn moveTo(s: *Session, dst: *Session, e: *Engine) !void { const m = &s.m; const d = &dst.m; diff --git a/zig/src/families/flashnext/slots.zig b/zig/src/families/flashnext/slots.zig new file mode 100644 index 0000000000..d813532142 --- /dev/null +++ b/zig/src/families/flashnext/slots.zig @@ -0,0 +1,103 @@ +//! Sized shared sessions (FLASHNEXT_SLOT_CONTEXTS): the list, which session a stream takes, and when it must grow. +const std = @import("std"); +const fz = @import("replay.zig"); + +/// Rows a window past a stream's last committed token can take. +pub const WINDOW = fz.MAXR + 1; + +/// The sessions past the engine's own from `text` (counts, `CONTEXTxN` for N alike), each at most `capacity`, `most` in all. +pub fn parse(gpa: std.mem.Allocator, text: []const u8, capacity: usize, margin: usize, most: usize) ![]usize { + var caps: std.ArrayList(usize) = .empty; + errdefer caps.deinit(gpa); + var it = std.mem.tokenizeScalar(u8, text, ','); + while (it.next()) |entry| { + const e = std.mem.trim(u8, entry, " \t"); + const x = std.mem.indexOfScalar(u8, e, 'x'); + const tokens = std.fmt.parseInt(usize, if (x) |i| e[0..i] else e, 10) catch return error.BadSlotContexts; + const n = if (x) |i| std.fmt.parseInt(usize, e[i + 1 ..], 10) catch return error.BadSlotContexts else 1; + if (tokens < 1024 or n == 0 or n > most - caps.items.len) return error.BadSlotContexts; + const rows = std.math.add(usize, tokens, margin) catch return error.BadSlotContexts; + try caps.appendNTimes(gpa, @min(capacity, rows), n); + } + if (caps.items.len == 0) return error.BadSlotContexts; + return try caps.toOwnedSlice(gpa); +} + +/// A new stream's session: unsized the first free one (the caller checks it holds the reply), sized the smallest free one holding the prompt and `grow`. +pub fn pick(caps: []const usize, used: []const bool, sized: bool, prompt: usize, max_new: usize, grow: usize) ?usize { + const need = prompt + @min(max_new, grow) + WINDOW; + var best: ?usize = null; + for (caps, used, 0..) |c, u, j| { + if (u) continue; + if (!sized) return j; + if (c >= need and (best == null or c < caps[best.?])) best = j; + } + return best; +} + +/// What a stream needs from its session before a round. +pub const Room = enum { ok, move, pause, full }; + +/// Room in a `cap`-row session at `pos` for a `rows` window, the reply ending by `end`, `most` the largest session, `larger` one free. +pub fn room(cap: usize, pos: usize, rows: usize, end: usize, most: usize, larger: bool) Room { + if (end <= cap) return .ok; // the reply ends inside this session: it never grows + if (end > most) return .full; // no session holds it: never wait for one + if (pos + rows + 2 * WINDOW <= cap) return .ok; // the next round fits after this one + if (larger) return .move; + if (pos + rows + WINDOW <= cap) return .pause; // this round fits; then wait for a larger session to free + return .full; +} + +test "the list parses counts and repeats, caps each at the window, and refuses overflow and too many sessions" { + const gpa = std.testing.allocator; + const caps = try parse(gpa, "131072, 4096x2", 65536, 32, 15); + defer gpa.free(caps); + try std.testing.expectEqualSlices(usize, &.{ 65536, 4128, 4128 }, caps); + for ([_][]const u8{ "", "512", "4096x0", "4096xa", "18446744073709551615", "4096x16", "4096x8,4096x8", "1024x18446744073709551615" }) |bad| + try std.testing.expectError(error.BadSlotContexts, parse(gpa, bad, 65536, 32, 15)); + const most = try parse(gpa, "4096x15", 65536, 32, 15); + defer gpa.free(most); + try std.testing.expectEqual(@as(usize, 15), most.len); +} + +test "unsized sessions go first free, as before sized sessions; sized take the smallest that holds the first grow" { + const caps = [_]usize{ 262144, 65536, 16384 }; + const free = [_]bool{ false, false, false }; + try std.testing.expectEqual(@as(?usize, 0), pick(&caps, &free, false, 1000, 4000, 8192)); + try std.testing.expectEqual(@as(?usize, 2), pick(&caps, &free, true, 1000, 4000, 8192)); + try std.testing.expectEqual(@as(?usize, 1), pick(&caps, &free, true, 10_000, 100_000, 8192)); + const busy = [_]bool{ true, false, false }; + try std.testing.expectEqual(@as(?usize, 1), pick(&caps, &busy, false, 1000, 4000, 8192)); + try std.testing.expectEqual(@as(?usize, 1), pick(&caps, &busy, false, 1000, 100_000, 8192)); // unsized: first free, its fit checked by the caller + const all = [_]bool{ true, true, true }; + try std.testing.expectEqual(@as(?usize, null), pick(&caps, &all, false, 1000, 4000, 8192)); + try std.testing.expectEqual(@as(?usize, null), pick(&caps, &busy, true, 100_000, 300_000, 8192)); // sized: busy is the only one that holds it +} + +/// Rounds of `rows`-row windows landing `landed` tokens each to the reply's end: the first room that is not ok. +fn run(cap: usize, pos0: usize, rows: usize, landed: usize, prompt: usize, max_new: usize, most: usize, larger: bool) Room { + const end = prompt + max_new + WINDOW; + var pos = pos0; + while (pos < prompt + max_new) : (pos += landed) { + const r = room(cap, pos, rows, end, most, larger); + if (r != .ok) return r; + if (pos + rows > cap) return .full; // a round the session cannot take + } + return .ok; +} + +test "near the window's end with drafts rejected, a stream in a session that holds its reply never pauses" { + // the largest session, reply ending at its last rows: every window offers MAXR rows and one lands + const cap = 1024 + 4096 + WINDOW; + try std.testing.expectEqual(Room.ok, run(cap, 1024, fz.MAXR, 1, 1024, 4096, cap, false)); + try std.testing.expectEqual(Room.ok, run(cap, 1024, fz.MAXR, fz.MAXR, 1024, 4096, cap, false)); +} + +test "a stream that outgrows its session moves while a larger one is free, else pauses on a round that still fits" { + const small = 4096; + const big = 1024 + 8192 + WINDOW; + try std.testing.expectEqual(Room.move, run(small, 1024, fz.MAXR, 1, 1024, 8192, big, true)); + try std.testing.expectEqual(Room.pause, run(small, 1024, fz.MAXR, 1, 1024, 8192, big, false)); + try std.testing.expectEqual(Room.pause, run(small, 1024, fz.MAXR, fz.MAXR, 1024, 8192, big, false)); + try std.testing.expectEqual(Room.full, room(small, 1024, fz.MAXR, 1024 + 8192 + WINDOW, small, false)); // no larger session +} diff --git a/zig/src/native/flashnext_batch_host.zig b/zig/src/native/flashnext_batch_host.zig index 9c221ae879..8538c4ed3b 100644 --- a/zig/src/native/flashnext_batch_host.zig +++ b/zig/src/native/flashnext_batch_host.zig @@ -34,28 +34,14 @@ fn ceiling(total: u64, working_set: u64) error{BadLoadLimit}!u64 { return c.bytes; } -/// The sessions past the engine's own from FLASHNEXT_SLOT_CONTEXTS: token counts, `CONTEXTxN` for N sessions of one -/// size (as "131072,65536x2,16384x8"), each holding at most `capacity` rows; null when unset. +/// The sessions past the engine's own from FLASHNEXT_SLOT_CONTEXTS (as "131072,65536x2,16384x8"); null when unset. fn slotContexts(gpa: std.mem.Allocator, capacity: usize) !?[]usize { const text = std.mem.trim(u8, std.mem.span(std.c.getenv("FLASHNEXT_SLOT_CONTEXTS") orelse return null), " \t"); if (text.len == 0) return null; - var caps: std.ArrayList(usize) = .empty; - errdefer caps.deinit(gpa); - var it = std.mem.tokenizeScalar(u8, text, ','); - while (it.next()) |entry| { - const e = std.mem.trim(u8, entry, " \t"); - const x = std.mem.indexOfScalar(u8, e, 'x'); - const tokens = std.fmt.parseInt(usize, if (x) |i| e[0..i] else e, 10) catch return error.BadSlotContexts; - const n = if (x) |i| std.fmt.parseInt(usize, e[i + 1 ..], 10) catch return error.BadSlotContexts else 1; - if (tokens < 1024 or n == 0) return error.BadSlotContexts; - for (0..n) |_| try caps.append(gpa, @min(capacity, tokens + @as(usize, @intCast(fx.MARGIN)))); - } - if (caps.items.len == 0 or caps.items.len + 1 > tf.flashnext_replay.MAXR) return error.BadSlotContexts; - return try caps.toOwnedSlice(gpa); + return try tf.flashnext_slots.parse(gpa, text, capacity, @intCast(fx.MARGIN), tf.flashnext_replay.MAXR - 1); } -/// Reply tokens a stream's session must hold past its prompt at admission (FLASHNEXT_SLOT_GROW, default 8192): a -/// stream that outgrows it moves to a free larger session at a round's end, or waits for one. +/// Reply tokens a session must hold past the prompt at admission (FLASHNEXT_SLOT_GROW, default 8192); past them a stream moves. fn slotGrow() usize { const v = std.c.getenv("FLASHNEXT_SLOT_GROW") orelse return 8192; return std.fmt.parseInt(usize, std.mem.span(v), 10) catch 8192; @@ -105,7 +91,10 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const h.eng = eng; h.back = try Backend.init(gpa, eng, caps.items); errdefer h.back.deinit(); - if (tiers != null) h.back.grow = slotGrow(); + if (tiers != null) { + h.back.sized = true; + h.back.grow = slotGrow(); + } const allocated = fit.footprint() orelse return error.NoMemoryReading; if (allocated +| fit.MARGIN +| reserved > cap) return error.OverMemoryLimit; h.cfg = try lanes.Config.init(gpa, h.back.facts(), tf.flashnext_replay.MAXR, tf.flashnext_replay.MAXR - 1); @@ -115,7 +104,7 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const errdefer h.core.deinit(); const plan = try fit.budgetUnder(cap, cache_gib, over, 0, a, why, ""); // the cache under the same limit as the sessions eng.snap_pool.max = plan.budget_bytes; - h.host = api.LaneHost.init(gpa, io, &h.core, .{ .name = "flashnext-zig", .lanes = slots, .context_window = @intCast(window), .prefill_step = @intCast(eng.pr.step), .prompt_cache_plan = plan, .admit = .{ .ctx = &h.back, .fits = Backend.fits } }); + h.host = api.LaneHost.init(gpa, io, &h.core, .{ .name = "flashnext-zig", .lanes = slots, .context_window = @intCast(window), .prefill_step = @intCast(eng.pr.step), .prompt_cache_plan = plan, .admit = if (tiers != null) .{ .ctx = &h.back, .fits = Backend.fits } else null }); h.cache = null; if (plan.budget_bytes > 0) { h.cache = api.prompt_cache.Store.init(gpa, .{ .ptr = &h.back, .vtable = &.{ .bytes = Backend.snapBytes, .save = Backend.snapSave, .restore = Backend.snapRestore, .drop = Backend.snapDrop, .charged = Backend.snapCharged, .spare = Backend.snapSpare, .trim = Backend.snapTrim, .reuses = Backend.snapReuses } }, .{ .lookahead = 1, .planned = true }, plan.budget_bytes); diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 4ded2adb9c..5d3b40f5a3 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -28,6 +28,7 @@ pub const flashnext_snapshot = @import("families/flashnext/snapshot.zig"); pub const flashnext_session = @import("families/flashnext/session.zig"); pub const flashnext_batch_forward = @import("families/flashnext/batch_forward.zig"); pub const flashnext_backend = @import("families/flashnext/backend.zig"); +pub const flashnext_slots = @import("families/flashnext/slots.zig"); pub const flashnext_batch_meta = @import("families/flashnext/batch_meta.zig"); pub const flashnext_learned = @import("families/flashnext/learned.zig"); pub const flashnext_think_check = @import("families/flashnext/think_check.zig"); @@ -54,6 +55,7 @@ test { _ = @import("families/flashnext/dense.zig"); // The core projection adapter keeps format, cuts and tiles. _ = @import("families/flashnext/marks.zig"); // a call's marks _ = @import("families/flashnext/think.zig"); // a thinking budget's close on the host + _ = @import("families/flashnext/slots.zig"); // sized shared sessions: the list, a stream's session, when it grows _ = @import("families/flashnext/think_check.zig"); // and in the verdict kernels (Metal) _ = @import("families/glm/glm.zig"); _ = @import("families/qwen3_5/config.zig"); From dd86083c26a8ec87d8ff29c30a4f639169ce184d Mon Sep 17 00:00:00 2001 From: sethforprivacy Date: Tue, 13 Oct 2026 13:06:27 -0400 Subject: [PATCH 16/31] Flash Next shared-round prices: time only on a session that holds the run, FZ_BATCH_UNPRICED=1 skips timing, fuller pricing tests Review fixes (cwschroeder on cwschroeder/TensorFold#9): - batch_costs.time prefills a 192-token scratch prompt and runs windows of up to MAXR rows past it, so a session needs batch_costs.ROWS (209) rows. timeCosts now checks fits(s.m.cap) first and serves unpriced (the allocator as before) when the session is smaller, as at --context 128; time itself refuses with SessionTooSmall. Boundary test at ROWS - 1 and ROWS. - FZ_BATCH_UNPRICED=1 skips the timing entirely: no costs, so the allocator, draft budgets and the lone depth rule are the unpriced ones from before this PR. - allocate.zig: tests for the first-draft exemption (a 100 ms step still takes a stream's first draft), asymmetric streams (priced [3, 1] against unpriced [3, 3]) and a zero step (priced equals allocate). - Doc comments in batch_costs.zig, session.zig and backend.zig cut to one line. - batch_costs.zig joins the core test root (tensorfold.zig), so its boundary test runs. --- zig/src/core/lanes/allocate.zig | 36 ++++++++++++++++++++++ zig/src/families/flashnext/backend.zig | 16 +++++++--- zig/src/families/flashnext/batch_costs.zig | 19 ++++++++++-- zig/src/families/flashnext/session.zig | 3 +- zig/src/tensorfold.zig | 1 + 5 files changed, 66 insertions(+), 9 deletions(-) diff --git a/zig/src/core/lanes/allocate.zig b/zig/src/core/lanes/allocate.zig index 8ec9598dca..2adad78930 100644 --- a/zig/src/core/lanes/allocate.zig +++ b/zig/src/core/lanes/allocate.zig @@ -134,6 +134,42 @@ test "a priced draft step keeps a stream's later drafts out when they land too s try std.testing.expectEqualSlices(u32, &.{ 1, 1 }, priced); } +test "a priced draft step leaves each stream's first draft free" { + const gpa = std.testing.allocator; + var flat = try extendCosts(gpa, &.{ .{ .width = 1, .ms = 10.0 }, .{ .width = 8, .ms = 10.0 } }, 8); + defer flat.deinit(gpa); + const probs = [_][]const f64{&.{0.5}}; + // charged, the draft would cost 100 ms against 0.5 tokens: it stays because the first is free + const counts = try allocatePriced(gpa, &.{1}, &probs, &flat, 0.0, 8, 100.0); + defer gpa.free(counts); + try std.testing.expectEqualSlices(u32, &.{1}, counts); +} + +test "a priced draft step cuts the stream whose later drafts land seldom and keeps the other's" { + const gpa = std.testing.allocator; + var flat = try extendCosts(gpa, &.{ .{ .width = 1, .ms = 10.0 }, .{ .width = 8, .ms = 10.0 } }, 8); + defer flat.deinit(gpa); + const probs = [_][]const f64{ &.{ 0.9, 0.85, 0.8 }, &.{ 0.9, 0.2, 0.1 } }; + const priced = try allocatePriced(gpa, &.{ 1, 1 }, &probs, &flat, 0.0, 8, 1.0); + defer gpa.free(priced); + try std.testing.expectEqualSlices(u32, &.{ 3, 1 }, priced); + const free = try allocate(gpa, &.{ 1, 1 }, &probs, &flat, 0.0, 8); + defer gpa.free(free); + try std.testing.expectEqualSlices(u32, &.{ 3, 3 }, free); +} + +test "a zero draft step prices rounds as allocate does" { + const gpa = std.testing.allocator; + var costs = try extendCosts(gpa, &.{ .{ .width = 1, .ms = 10.0 }, .{ .width = 8, .ms = 12.0 } }, 8); + defer costs.deinit(gpa); + const probs = [_][]const f64{ &.{ 0.9, 0.85, 0.8 }, &.{ 0.9, 0.2, 0.1 }, &.{0.4} }; + const priced = try allocatePriced(gpa, &.{ 1, 1, 1 }, &probs, &costs, 2.0, 8, 0.0); + defer gpa.free(priced); + const free = try allocate(gpa, &.{ 1, 1, 1 }, &probs, &costs, 2.0, 8); + defer gpa.free(free); + try std.testing.expectEqualSlices(u32, free, priced); +} + test "extendCosts interpolates and extrapolates" { const gpa = std.testing.allocator; var t = try extendCosts(gpa, &.{ .{ .width = 4, .ms = 8.0 }, .{ .width = 2, .ms = 4.0 } }, 6); diff --git a/zig/src/families/flashnext/backend.zig b/zig/src/families/flashnext/backend.zig index 92162b8bf7..0b93ebb291 100644 --- a/zig/src/families/flashnext/backend.zig +++ b/zig/src/families/flashnext/backend.zig @@ -62,9 +62,18 @@ pub const Backend = struct { return .{ .gpa = gpa, .e = e, .primary = e.m, .sessions = sessions, .hidden = .{ .b = hidden }, .chain_drafts = env.on("FZ_BATCH_MTP_CHAIN"), .serial_drafts = env.on("FZ_BATCH_MTP_SERIAL") }; } - /// The lane allocator's prices, timed on the last session before any stream holds it. + /// The lane allocator's prices, timed on the last session before any stream holds it; none (unpriced) when it can't be. pub fn timeCosts(b: *Backend) !void { - const c = try batch_costs.time(b, &b.sessions[b.sessions.len - 1]); + if (std.c.getenv("FZ_BATCH_UNPRICED") != null) { + std.log.info("flash next: FZ_BATCH_UNPRICED=1: shared rounds unpriced, nothing timed", .{}); + return; + } + const s = &b.sessions[b.sessions.len - 1]; + if (!batch_costs.fits(s.m.cap)) { + std.log.info("flash next: a session holds {d} rows, under the {d} that timing shared rounds takes: shared rounds unpriced", .{ s.m.cap, batch_costs.ROWS }); + return; + } + const c = try batch_costs.time(b, s); b.costs = c; std.log.info("flash next: shared forwards {d:.2} {d:.2} {d:.2} {d:.2} {d:.2} ms at 1, 2, 4, 8, 16 rows; a chained draft {d:.2} ms", .{ c.shared[0].ms, c.shared[1].ms, c.shared[3].ms, c.shared[7].ms, c.shared[15].ms, c.step_ms }); } @@ -336,8 +345,7 @@ pub const Backend = struct { } } - /// Every request's absorb and chained drafts in one command buffer, one wait for the round's drafts (a follow - /// past MAXR rows, the prompt's first, absorbs alone first). + /// Every request's absorb and chained drafts in one command buffer, one wait (a long follow absorbs alone first). fn draftTogether(b: *Backend, requests: []const be.DraftRequest) !void { const pool = mtl.objc.Pool.push(); defer pool.pop(); diff --git a/zig/src/families/flashnext/batch_costs.zig b/zig/src/families/flashnext/batch_costs.zig index fcf148cb29..66f66c41a2 100644 --- a/zig/src/families/flashnext/batch_costs.zig +++ b/zig/src/families/flashnext/batch_costs.zig @@ -1,6 +1,4 @@ -//! The lane allocator's prices for shared rounds, timed at load on a free session: a shared forward's ms by rows -//! (the fastest of a few at each width, the widest first while the GPU's clocks ramp up) and a chained head step's ms -//! (a draft past a stream's first, in the one command buffer that takes a round's drafts). +//! The lane allocator's prices for shared rounds, timed at load on a free session: shared forwards and a chained draft. const std = @import("std"); const lanes = @import("lanes"); const mtl = @import("metal"); @@ -16,6 +14,13 @@ pub const Costs = struct { const PROMPT = 192; // the scratch prompt the windows follow const TRIES = 4; // the fastest of these at each width const WARM = 6; // widest windows first +/// Cache rows a session needs to be timed on: the scratch prompt, then windows of up to MAXR rows past it. +pub const ROWS = PROMPT + fz.MAXR + 1; + +/// Whether a session of `cap` rows can be timed on (the chained drafts stay inside the widest window). +pub fn fits(cap: usize) bool { + return cap >= ROWS; +} fn tokenAt(i: usize) u32 { return @intCast((37 * i + 11) % 50_000 + 1000); @@ -23,6 +28,7 @@ fn tokenAt(i: usize) u32 { /// The costs timed on `s` (a free session, empty again after), `b` a *Backend. pub fn time(b: anytype, s: *Session) !Costs { + if (!fits(s.m.cap)) return error.SessionTooSmall; const e = b.e; var prompt: [PROMPT]u32 = undefined; for (&prompt, 0..) |*t, i| t.* = tokenAt(i); @@ -101,3 +107,10 @@ fn chainMs(b: anytype, s: *Session, depth: usize) !f64 { s.heldDrafts(depth); return (mtl.clock.seconds() - t0) * 1e3; } + +test "timing needs the scratch prompt and the widest window past it" { + try std.testing.expectEqual(@as(usize, PROMPT + fz.MAXR + 1), ROWS); + try std.testing.expect(!fits(ROWS - 1)); + try std.testing.expect(fits(ROWS)); + try std.testing.expect(!fits(160)); // --context 128's sessions +} diff --git a/zig/src/families/flashnext/session.zig b/zig/src/families/flashnext/session.zig index 4b851fddd4..80f01c7a16 100644 --- a/zig/src/families/flashnext/session.zig +++ b/zig/src/families/flashnext/session.zig @@ -142,8 +142,7 @@ pub const Session = struct { @memcpy(s.held[0..depth], s.draft_ids.b.slice(u32, depth)); } - /// Encode the head's absorb of a kept window and `depth` chained drafts into the open encoder, unsubmitted (one - /// command buffer takes every stream's drafts: heldDrafts reads them once it is done); depth 0 absorbs only. + /// The head's absorb of a kept window and `depth` chained drafts, encoded unsubmitted (heldDrafts reads them). pub fn encodeDrafts(s: *Session, follow: []const u32, hidden: Buf, depth: usize) !void { if (follow.len == 0 or follow.len > fz.MAXR or depth >= fz.MAXR) return error.DraftOutOfStep; const m = &s.m; diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 4ded2adb9c..ee5f917619 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -54,6 +54,7 @@ test { _ = @import("families/flashnext/dense.zig"); // The core projection adapter keeps format, cuts and tiles. _ = @import("families/flashnext/marks.zig"); // a call's marks _ = @import("families/flashnext/think.zig"); // a thinking budget's close on the host + _ = @import("families/flashnext/batch_costs.zig"); // the shared rounds' timing needs a session that holds it _ = @import("families/flashnext/think_check.zig"); // and in the verdict kernels (Metal) _ = @import("families/glm/glm.zig"); _ = @import("families/qwen3_5/config.zig"); From 09c726262ffcdeeb4571e18570bf3d571ecb92f9 Mon Sep 17 00:00:00 2001 From: sethforprivacy Date: Tue, 13 Oct 2026 13:04:44 -0400 Subject: [PATCH 17/31] Flash Next lone driver: read the prompt after the first token's commit, which can move the context The prompt slice was taken before the first token's commit and handed to Engine.decode after it; a commit that grows the context frees the old buffer, so decode copied freed memory into the rounds' history. - Backend.commitFirst commits the first token and returns stream.prompt() from the context as it is then. decode copies the prompt to the rounds' history before any round commits, so later commits leave it alone. - A test fills the context to capacity, refuses growth in place (FailingAllocator resize_fail_index 0), and checks the returned prompt is the moved context's, with the prompt's tokens. - backend.zig joins the core test root (tensorfold.zig), so this test runs. --- zig/src/families/flashnext/backend.zig | 26 +++++++++++++++++++++++--- zig/src/tensorfold.zig | 1 + 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/zig/src/families/flashnext/backend.zig b/zig/src/families/flashnext/backend.zig index e456599811..7cfb9462a9 100644 --- a/zig/src/families/flashnext/backend.zig +++ b/zig/src/families/flashnext/backend.zig @@ -205,17 +205,22 @@ pub const Backend = struct { /// the prompt cache keeps are theirs), then the engine's GPU-side rounds, two deep, until the stream finishes /// (false) or `hooks.yield` asks for it: its state then stands as a shared round leaves it and the lane core takes /// it on (true). `hooks`: LaneHost's LoneHooks. + /// Commits a lone stream's first token; its prompt as the context holds it after the commit, which can move it. + fn commitFirst(gpa: std.mem.Allocator, stream: *Stream, token: u32) ![]const u32 { + _ = try stream.commit(gpa, &.{token}, &.{}); + return stream.prompt(); + } + pub fn lone(b: *Backend, stream: *Stream, hooks: anytype) !bool { const pool = mtl.objc.Pool.push(); defer pool.pop(); - const prompt = stream.prompt(); const s = &b.sessions[0]; // the GPU-side rounds run on the engine's own buffers if (s.used) return error.NoFreeSlot; - if (prompt.len + stream.max_new + @import("engine.zig").MARGIN > s.m.cap) return error.ContextFull; + if (stream.prompt().len + stream.max_new + @import("engine.zig").MARGIN > s.m.cap) return error.ContextFull; var handed = false; defer if (!handed) release(b, stream); try b.fill(stream, 0); - _ = try stream.commit(b.gpa, &.{s.first}, &.{}); + const prompt = try commitFirst(b.gpa, stream, s.first); // decode copies it to the rounds' history before any round commits hooks.committed(hooks.ctx); if (stream.finished) return false; const Hooks = @TypeOf(hooks); @@ -444,3 +449,18 @@ pub const Backend = struct { snap.drop(self(ptr).gpa, @ptrCast(@alignCast(saved))); } }; + +test "a lone stream's prompt is read after its first token's commit moves the context" { + const gpa = std.testing.allocator; + const tokens = [_]u32{ 11, 12, 13, 14, 15, 16, 17 }; + var stream = try Stream.init(gpa, .{ .id = "lone", .prompt = &tokens, .max_new = 4, .eos = &.{} }); + defer stream.deinit(gpa); + stream.context.shrinkAndFree(gpa, stream.context.items.len); // full: the first token must grow it + var failing = std.testing.FailingAllocator.init(gpa, .{ .resize_fail_index = 0 }); // and not in place + const before = stream.context.items.ptr; + const prompt = try Backend.commitFirst(failing.allocator(), &stream, 99); + try std.testing.expect(stream.context.items.ptr != before); + try std.testing.expectEqual(stream.context.items.ptr, prompt.ptr); + try std.testing.expectEqualSlices(u32, &tokens, prompt); + try std.testing.expectEqualSlices(u32, &.{99}, stream.emitted()); +} diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 4ded2adb9c..f0b00d3181 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -54,6 +54,7 @@ test { _ = @import("families/flashnext/dense.zig"); // The core projection adapter keeps format, cuts and tiles. _ = @import("families/flashnext/marks.zig"); // a call's marks _ = @import("families/flashnext/think.zig"); // a thinking budget's close on the host + _ = @import("families/flashnext/backend.zig"); // the shared sessions' backend (a lone stream's prompt after its first commit) _ = @import("families/flashnext/think_check.zig"); // and in the verdict kernels (Metal) _ = @import("families/glm/glm.zig"); _ = @import("families/qwen3_5/config.zig"); From 0f679e234829308c5c2f0227e08287f2d86c485b Mon Sep 17 00:00:00 2001 From: sethforprivacy Date: Wed, 14 Oct 2026 06:25:23 -0400 Subject: [PATCH 18/31] Flash Next lone driver: lone's doc comment above lone again (commitFirst had taken it), doc comments on one line --- zig/src/families/flashnext/backend.zig | 5 +---- zig/src/families/flashnext/engine.zig | 12 +++--------- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/zig/src/families/flashnext/backend.zig b/zig/src/families/flashnext/backend.zig index c234ca3190..8925d2f898 100644 --- a/zig/src/families/flashnext/backend.zig +++ b/zig/src/families/flashnext/backend.zig @@ -229,16 +229,13 @@ pub const Backend = struct { s.m.mtp.drafted = 0; } - /// LaneHost's driver for a lone stream: the shared sessions' own prompt pass into the engine's session (the states - /// the prompt cache keeps are theirs), then the engine's GPU-side rounds, two deep, until the stream finishes - /// (false) or `hooks.yield` asks for it: its state then stands as a shared round leaves it and the lane core takes - /// it on (true). `hooks`: LaneHost's LoneHooks. /// Commits a lone stream's first token; its prompt as the context holds it after the commit, which can move it. fn commitFirst(gpa: std.mem.Allocator, stream: *Stream, token: u32) ![]const u32 { _ = try stream.commit(gpa, &.{token}, &.{}); return stream.prompt(); } + /// LaneHost's lone driver: the shared prompt pass into session 0, then the engine's rounds until done (false) or handed over (true). pub fn lone(b: *Backend, stream: *Stream, hooks: anytype) !bool { const pool = mtl.objc.Pool.push(); defer pool.pop(); diff --git a/zig/src/families/flashnext/engine.zig b/zig/src/families/flashnext/engine.zig index ce6f6aef95..22d5959d6c 100644 --- a/zig/src/families/flashnext/engine.zig +++ b/zig/src/families/flashnext/engine.zig @@ -625,14 +625,10 @@ pub const Engine = struct { return e.decode(prompt, pick, .{ .b = m.last.b, .off = m.last.off + (last_n - 1) * WIDE * 2 }, max_tokens, eos, depth, &watch, out, null); } - /// Where `decode` left a stream it handed over (a cancel with a handoff): its state as a host round leaves it - /// (the DeltaNet states kept, the n-gram tail in the model's own buffer, the head absorbed through the last - /// cache row), `cache` rows in the cache. + /// Where `decode` left a stream it handed over: its state as a host round leaves it, `cache` rows in the cache. pub const Handoff = struct { yielded: bool = false, cache: usize = 0 }; - /// The GPU-side rounds after a prompt pass: `prompt` in the cache, `pick` its first token (out has it), `streams` - /// the last prompt row's (the head's first drafts), `watch` the reply's thinking budget. With `handoff`, a cancel - /// hands the stream over (handOver). + /// The GPU-side rounds after a prompt pass from `pick` and the last prompt row's `streams`; with `handoff` a cancel hands the stream over. pub fn decode(e: *Engine, prompt: []const u32, pick: u32, streams: Buf, max_tokens: usize, eos: []const u32, depth: ?usize, watch: *think.Watch, out: Out, handoff: ?*Handoff) !Result { const r = e.r; const m = e.m; @@ -829,9 +825,7 @@ pub const Engine = struct { return res; } - /// The rounds handed over at `cache` rows, `round` the next round's index: the round ahead (greedy rounds run two - /// deep) started as each round does, with the last landed round's kept states, n-gram tail and head absorb; past - /// that nothing a host round reads. Then the tail into the model's own buffer. + /// The rounds handed over at `cache` rows: the round ahead already started as rounds do; the n-gram tail into the model's own buffer. fn handOver(e: *Engine, h: *Handoff, round: usize, cache: usize) !void { if (e.r.gdn_kept != null) return error.HandoffUnsupported; // kept states replay their rows in the next window if ((round - 1) % 2 == 1) { From b1a4204b9210d9b6704c32992bcae75edb7204e4 Mon Sep 17 00:00:00 2001 From: sethforprivacy Date: Wed, 14 Oct 2026 06:35:34 -0400 Subject: [PATCH 19/31] Flash Next: facts' priced-lone flag renamed from `lone`, which the merge left shadowing the lone driver (Backend.lone) --- zig/src/families/flashnext/backend.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zig/src/families/flashnext/backend.zig b/zig/src/families/flashnext/backend.zig index 8925d2f898..108e89ea3d 100644 --- a/zig/src/families/flashnext/backend.zig +++ b/zig/src/families/flashnext/backend.zig @@ -96,8 +96,8 @@ pub const Backend = struct { const c: ?*const batch_costs.Costs = if (b.costs) |*x| x else null; // shared rounds priced from the timed forwards, each draft past a stream's first at a chained step's time; a // lone stream's depth from the same (FZ_LONE_UNPRICED=1: the heuristic) - const lone = c != null and std.c.getenv("FZ_LONE_UNPRICED") == null; - return .{ .exact_width = fz.MAXR, .first_copy_rows = 4, .mtp = true, .speculate = true, .speculate_early = false, .drafts = 8, .streams_exact = true, .hidden_rows = true, .batch_rows = fz.MAXR, .max_streams = @intCast(b.sessions.len), .draft_streams = !b.serial_drafts, .shared_costs = if (c) |x| &x.shared else &.{}, .draft_step_ms = if (c != null and !b.serial_drafts) c.?.step_ms else 0, .window_costs = if (lone) &c.?.shared else &.{}, .mtp_step_ms = if (lone) c.?.step_ms else 0 }; + const lone_priced = c != null and std.c.getenv("FZ_LONE_UNPRICED") == null; // not `lone`: that names the lone driver + return .{ .exact_width = fz.MAXR, .first_copy_rows = 4, .mtp = true, .speculate = true, .speculate_early = false, .drafts = 8, .streams_exact = true, .hidden_rows = true, .batch_rows = fz.MAXR, .max_streams = @intCast(b.sessions.len), .draft_streams = !b.serial_drafts, .shared_costs = if (c) |x| &x.shared else &.{}, .draft_step_ms = if (c != null and !b.serial_drafts) c.?.step_ms else 0, .window_costs = if (lone_priced) &c.?.shared else &.{}, .mtp_step_ms = if (lone_priced) c.?.step_ms else 0 }; } fn self(ptr: *anyopaque) *Backend { From 069ef4de1462f095e6bbe6d73ca216eb6a951239 Mon Sep 17 00:00:00 2001 From: sethforprivacy Date: Wed, 14 Oct 2026 06:29:37 -0400 Subject: [PATCH 20/31] Flash Next sized sessions: a paused stream whose move fails ends with an error; the lane core sweeps streams a release ends; harness on Backend.init's capacities Review fixes (cwschroeder on cwschroeder/TensorFold#10): - slots.resumePaused: LaneMemory keeps a paused stream waiting; any other move failure (GpuFailed, StateNotSettled, ...) ends it through Backend.failed: slots.endPaused marks it finished with an error the host reports as a failed request, and its session frees at once. Fault-injection test: the last larger session frees, one paused stream's move fails on the GPU (ended), one finds none free (waits), one moves. - The lane core: a release can now end another stream, so Engine.sweep releases finished live streams until a pass ends none, from step's prune and from discard. Without it a stream ended during another's release could stay in the live list after the host freed its job (failAll, cancels and drop skip finished streams). Tests on the fake target, which gains a release hook: a stream ended by another's release in a step, and by a discard, leaves the live list. - zig/tests/flashnext_batch.zig: Backend.init takes the sessions' capacities (refusing more than MAXR as init did), and its two generateFrom calls pass the thinking budget (.{}) that 1.0.8 added, which already broke the harness on 04c8571. --- zig/src/core/lanes/engine.zig | 33 ++++++++----- zig/src/core/lanes/engine_test.zig | 57 ++++++++++++++++++++++ zig/src/core/lanes/fake.zig | 3 ++ zig/src/families/flashnext/backend.zig | 33 ++++++------- zig/src/families/flashnext/slots.zig | 65 ++++++++++++++++++++++++++ zig/tests/flashnext_batch.zig | 10 ++-- 6 files changed, 171 insertions(+), 30 deletions(-) diff --git a/zig/src/core/lanes/engine.zig b/zig/src/core/lanes/engine.zig index 368e69b890..fc239323fe 100644 --- a/zig/src/core/lanes/engine.zig +++ b/zig/src/core/lanes/engine.zig @@ -99,6 +99,27 @@ pub const Engine = struct { } } e.release(s); + e.sweep() catch {}; // its release may have ended another (a backend's paused stream that could not move) + } + + /// Releases the live streams that finished, again while a release ends another, so none outlives its job. + fn sweep(e: *Engine) !void { + var again = true; + while (again) { + again = false; + var kept: usize = 0; + for (e.live.items) |s| { + if (s.finished) { + try trail.finish(e, s); + e.release(s); + again = true; + continue; + } + e.live.items[kept] = s; + kept += 1; + } + e.live.shrinkRetainingCapacity(kept); + } } /// One round for every live stream (Python `_family_step`). @@ -139,17 +160,7 @@ pub const Engine = struct { } } try trail.resolve(e); - var kept: usize = 0; - for (e.live.items) |s| { - if (s.finished) { - try trail.finish(e, s); - e.release(s); - continue; - } - e.live.items[kept] = s; - kept += 1; - } - e.live.shrinkRetainingCapacity(kept); + try e.sweep(); try trail.state(e); } diff --git a/zig/src/core/lanes/engine_test.zig b/zig/src/core/lanes/engine_test.zig index 2100cc3762..06c7ba028d 100644 --- a/zig/src/core/lanes/engine_test.zig +++ b/zig/src/core/lanes/engine_test.zig @@ -460,3 +460,60 @@ test "a family that keeps copies to shared rounds proposes none for a lone strea try std.testing.expectEqual(lone_copies, (try win.copyProposal(&engine, &s)).len >= 2); } } + +/// A release hook that ends `ctx`'s stream with an error when another stream releases (a paused stream whose move failed). +fn endWaiting(ctx: *anyopaque, released: *sm.Stream) void { + const waiting: *sm.Stream = @ptrCast(@alignCast(ctx)); + if (released == waiting or waiting.finished) return; + waiting.paused = false; + waiting.finished = true; + waiting.reason = .@"error"; +} + +test "a stream another's release ends leaves the live list in that step, before its job can be freed" { + var cfg = try model(); + defer cfg.deinit(gpa); + var target: fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var clock: fake.FixedClock = .{}; + var engine = Engine.init(gpa, &cfg, target.backend(), clock.clock()); + defer engine.deinit(); + var waiting = try sm.Stream.init(gpa, .{ .id = "waiting", .prompt = &p2, .max_new = 40, .eos = &.{96}, .drafts = false }); + defer waiting.deinit(gpa); + var short = try sm.Stream.init(gpa, .{ .id = "short", .prompt = &p1, .max_new = 3, .eos = &.{96}, .drafts = false }); + defer short.deinit(gpa); + try engine.addStream(&waiting); // first in the live list: the sweep is past it when the short one releases + waiting.paused = true; + target.release_hook = endWaiting; + target.release_hook_ctx = &waiting; + try engine.addStream(&short); + try std.testing.expect(!short.finished); // its reply ends in a step, released by that step's sweep + var steps: usize = 0; + while (!short.finished and steps < 8) : (steps += 1) try engine.step(); + try std.testing.expect(short.finished); + try std.testing.expect(waiting.finished and waiting.reason == .@"error"); + try std.testing.expectEqual(@as(usize, 0), engine.live.items.len); +} + +test "a discard whose release ends another stream leaves neither in the live list" { + var cfg = try model(); + defer cfg.deinit(gpa); + var target: fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var clock: fake.FixedClock = .{}; + var engine = Engine.init(gpa, &cfg, target.backend(), clock.clock()); + defer engine.deinit(); + var waiting = try sm.Stream.init(gpa, .{ .id = "waiting", .prompt = &p2, .max_new = 40, .eos = &.{96}, .drafts = false }); + defer waiting.deinit(gpa); + var other = try sm.Stream.init(gpa, .{ .id = "other", .prompt = &p1, .max_new = 40, .eos = &.{96}, .drafts = false }); + defer other.deinit(gpa); + try engine.addStream(&waiting); + try engine.addStream(&other); + try engine.step(); + waiting.paused = true; + target.release_hook = endWaiting; + target.release_hook_ctx = &waiting; + engine.discard(&other); + try std.testing.expect(waiting.finished and waiting.reason == .@"error"); + try std.testing.expectEqual(@as(usize, 0), engine.live.items.len); +} diff --git a/zig/src/core/lanes/fake.zig b/zig/src/core/lanes/fake.zig index 9434736c46..4695a93a1d 100644 --- a/zig/src/core/lanes/fake.zig +++ b/zig/src/core/lanes/fake.zig @@ -55,6 +55,8 @@ pub const Fake = struct { prefill_count: usize = 0, prefill_hook: ?*const fn (ctx: *anyopaque, s: *Stream, chunk: usize) void = null, prefill_hook_ctx: ?*anyopaque = null, + release_hook: ?*const fn (ctx: *anyopaque, s: *Stream) void = null, // after a stream's release, as a backend ending another + release_hook_ctx: ?*anyopaque = null, refuse_sampled: bool = false, // prefill refuses a sampled stream with error.SamplingRefused ranked: bool = false, // the head gives its held chain back (alternatives) memory_lanes: ?usize = null, // prefill past this many live lanes fails with error.LaneMemory (no room yet) @@ -330,6 +332,7 @@ pub const Fake = struct { var l = kv.value; freeLane(x.gpa, &l); } + if (x.release_hook) |hook| hook(x.release_hook_ctx.?, s); } }; diff --git a/zig/src/families/flashnext/backend.zig b/zig/src/families/flashnext/backend.zig index 1b2aae04bb..70c8148fcb 100644 --- a/zig/src/families/flashnext/backend.zig +++ b/zig/src/families/flashnext/backend.zig @@ -229,23 +229,24 @@ pub const Backend = struct { std.log.info("flash next: a stream at {d} tokens moved from a {d}-token session to a {d}-token one ({d} moves)", .{ from.m.pos, from.m.cap, b.sessions[j].m.cap, b.moves }); } - /// Paused streams into sessions that freed, oldest first; each goes on once moved. + /// Paused streams into sessions that freed (slots.resumePaused): a move that fails but for LaneMemory ends the stream. fn resumePaused(b: *Backend) void { - var k: usize = 0; - while (k < b.paused.items.len) { - const stream = b.paused.items[k]; - const i = b.by.get(stream) orelse { - _ = b.paused.orderedRemove(k); - continue; - }; - b.move(stream, i) catch |err| { - if (err != error.LaneMemory) std.log.err("flash next: a paused stream could not move: {s}", .{@errorName(err)}); - k += 1; - continue; - }; - stream.paused = false; - _ = b.paused.orderedRemove(k); - } + slots.resumePaused(&b.paused, b); + } + + /// slots.resumePaused's move: the stream into a free larger session, LaneMemory while none is. + pub fn moveOut(b: *Backend, stream: *Stream) !void { + try b.move(stream, b.by.get(stream) orelse return error.UnknownStream); + } + + /// slots.resumePaused's failure: the stream ends with an error the host reports, and its session frees now. + pub fn failed(b: *Backend, stream: *Stream, err: anyerror) void { + std.log.err("flash next: a paused stream at {d} tokens could not move to a larger session ({s}); it ends with an error", .{ stream.context.items.len, @errorName(err) }); + slots.endPaused(stream); + const kv = b.by.fetchRemove(stream) orelse return; + b.sessions[kv.value].used = false; + b.sessions[kv.value].round = .{}; + b.sessions[kv.value].held_n = 0; } /// LaneHost's admission check: a free session holds the request now. diff --git a/zig/src/families/flashnext/slots.zig b/zig/src/families/flashnext/slots.zig index d813532142..8ebf99ed07 100644 --- a/zig/src/families/flashnext/slots.zig +++ b/zig/src/families/flashnext/slots.zig @@ -1,5 +1,6 @@ //! Sized shared sessions (FLASHNEXT_SLOT_CONTEXTS): the list, which session a stream takes, and when it must grow. const std = @import("std"); +const lanes = @import("lanes"); const fz = @import("replay.zig"); /// Rows a window past a stream's last committed token can take. @@ -48,6 +49,33 @@ pub fn room(cap: usize, pos: usize, rows: usize, end: usize, most: usize, larger return .full; } +/// Ends a paused stream whose move failed, with an error the host reports as a failed request. +pub fn endPaused(stream: *lanes.Stream) void { + stream.paused = false; + stream.finished = true; + stream.reason = .@"error"; + stream.problem = "the reply outgrew its session and could not move to a larger one"; +} + +/// Paused streams moved, oldest first: `ctx.moveOut` failing LaneMemory keeps one waiting, any other failure ends it (`ctx.failed`). +pub fn resumePaused(paused: *std.ArrayList(*lanes.Stream), ctx: anytype) void { + var k: usize = 0; + while (k < paused.items.len) { + const stream = paused.items[k]; + ctx.moveOut(stream) catch |err| { + if (err == error.LaneMemory) { + k += 1; + continue; + } + _ = paused.orderedRemove(k); + ctx.failed(stream, err); + continue; + }; + stream.paused = false; + _ = paused.orderedRemove(k); + } +} + test "the list parses counts and repeats, caps each at the window, and refuses overflow and too many sessions" { const gpa = std.testing.allocator; const caps = try parse(gpa, "131072, 4096x2", 65536, 32, 15); @@ -101,3 +129,40 @@ test "a stream that outgrows its session moves while a larger one is free, else try std.testing.expectEqual(Room.pause, run(small, 1024, fz.MAXR, fz.MAXR, 1024, 8192, big, false)); try std.testing.expectEqual(Room.full, room(small, 1024, fz.MAXR, 1024 + 8192 + WINDOW, small, false)); // no larger session } + +test "the last larger session frees and a paused stream's move fails: it ends with an error, LaneMemory keeps waiting" { + const gpa = std.testing.allocator; + var a = try lanes.Stream.init(gpa, .{ .id = "a", .prompt = &.{1}, .max_new = 8 }); + defer a.deinit(gpa); + var b = try lanes.Stream.init(gpa, .{ .id = "b", .prompt = &.{1}, .max_new = 8 }); + defer b.deinit(gpa); + var c = try lanes.Stream.init(gpa, .{ .id = "c", .prompt = &.{1}, .max_new = 8 }); + defer c.deinit(gpa); + var paused: std.ArrayList(*lanes.Stream) = .empty; + defer paused.deinit(gpa); + for ([_]*lanes.Stream{ &a, &b, &c }) |s| { + s.paused = true; + try paused.append(gpa, s); + } + const Mover = struct { // a's copy fails on the GPU, b finds no larger session free, c moves + a: *lanes.Stream, + b: *lanes.Stream, + ended: ?anyerror = null, + fn moveOut(m: *@This(), s: *lanes.Stream) !void { + if (s == m.a) return error.GpuFailed; + if (s == m.b) return error.LaneMemory; + } + fn failed(m: *@This(), s: *lanes.Stream, err: anyerror) void { + m.ended = err; + endPaused(s); + } + }; + var mover: Mover = .{ .a = &a, .b = &b }; + resumePaused(&paused, &mover); + try std.testing.expectEqual(@as(?anyerror, error.GpuFailed), mover.ended); + try std.testing.expect(a.finished and !a.paused and a.reason == .@"error" and a.problem.len > 0); + try std.testing.expect(b.paused and !b.finished); + try std.testing.expect(!c.paused and !c.finished); + try std.testing.expectEqual(@as(usize, 1), paused.items.len); + try std.testing.expectEqual(&b, paused.items[0]); +} diff --git a/zig/tests/flashnext_batch.zig b/zig/tests/flashnext_batch.zig index dd3702c2c5..32d6027f58 100644 --- a/zig/tests/flashnext_batch.zig +++ b/zig/tests/flashnext_batch.zig @@ -130,12 +130,12 @@ pub fn main(init: std.process.Init) !void { capacity = @max(capacity, ids.len + n + fx.MARGIN); ref.* = .{ .a = a }; const start = mtl.clock.seconds(); - _ = try eng.generateFrom(ids.*, 0, &.{}, n, &.{}, 0, ref.out()); + _ = try eng.generateFrom(ids.*, 0, &.{}, n, &.{}, 0, .{}, ref.out()); baseline_seconds += mtl.clock.seconds() - start; if (ref.tokens.items.len != n) return error.ShortBaseline; var drafted: Reply = .{ .a = a }; const mtp_start = mtl.clock.seconds(); - _ = try eng.generateFrom(ids.*, 0, &.{}, n, &.{}, null, drafted.out()); + _ = try eng.generateFrom(ids.*, 0, &.{}, n, &.{}, null, .{}, drafted.out()); baseline_mtp_seconds += mtl.clock.seconds() - mtp_start; if (!std.mem.eql(u32, drafted.tokens.items, ref.tokens.items)) return error.LegacyMtpTokenMismatch; } @@ -144,7 +144,11 @@ pub fn main(init: std.process.Init) !void { for (prompts, 0..) |ids, i| if (ids.len > prompts[longest].len) { longest = i; }; - var back = try Backend.init(init.gpa, eng, @intCast(@max(4, count)), capacity); + var capacities: [tf.flashnext_replay.MAXR]usize = undefined; + const sessions = @max(4, count); + if (sessions > capacities.len) return error.BadBatchSlots; + @memset(capacities[0..sessions], capacity); + var back = try Backend.init(init.gpa, eng, capacities[0..sessions]); defer back.deinit(); try draftDepths(init.gpa, &back, prompts); const boundary = 4 * (tf.flashnext_replay.TOP + 1) - 2; From 72d817cd38450b94b06ef05cc4272856cff39540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Fri, 16 Oct 2026 08:58:49 +0200 Subject: [PATCH 21/31] Finish migration-failed peers after cancellation before freeing their jobs --- zig/src/core/lane_host.zig | 22 +++-- zig/src/core/lane_host_end.zig | 1 + zig/src/core/lane_host_fail_test.zig | 135 +++++++++++++++++++++++++++ zig/src/core/lanes/engine.zig | 26 +++--- zig/src/core/lanes/engine_test.zig | 29 +++++- 5 files changed, 195 insertions(+), 18 deletions(-) diff --git a/zig/src/core/lane_host.zig b/zig/src/core/lane_host.zig index 4a8f7d3493..f582379149 100644 --- a/zig/src/core/lane_host.zig +++ b/zig/src/core/lane_host.zig @@ -494,6 +494,16 @@ pub const LaneHost = struct { }; } + /// Sends replies and finishes jobs only after the core has reaped terminal streams. + fn deliverPending(h: *LaneHost) void { + var i: usize = 0; + while (i < h.admitted.items.len) { + const job = h.admitted.items[i]; + if (h.deliver(job)) continue; + i += 1; + } + } + fn run(h: *LaneHost) void { while (true) { ends.takeCancels(h); @@ -501,6 +511,11 @@ pub const LaneHost = struct { learn.refuse(h); while (h.admitOne()) {} h.fillOne(); + h.core.sweep() catch |err| { + ends.failAll(h, @errorName(err)); + continue; + }; + h.deliverPending(); status.noteLive(h); h.lock(); if (h.closing) { @@ -535,12 +550,7 @@ pub const LaneHost = struct { ends.failAll(h, @errorName(e)); continue; }; - var i: usize = 0; - while (i < h.admitted.items.len) { - const job = h.admitted.items[i]; - if (h.deliver(job)) continue; // it left the list: the next job slid into slot i - i += 1; - } + h.deliverPending(); lone.reclaim(h); } } diff --git a/zig/src/core/lane_host_end.zig b/zig/src/core/lane_host_end.zig index 7163197979..15eaa42431 100644 --- a/zig/src/core/lane_host_end.zig +++ b/zig/src/core/lane_host_end.zig @@ -40,6 +40,7 @@ pub fn takeCancels(h: *LaneHost) void { /// A failed round ends every stream it held, with the backend's error. pub fn failAll(h: *LaneHost, message: []const u8) void { + h.core.sweep() catch {}; h.lock(); const jobs = h.gpa.dupe(*Job, h.admitted.items) catch &.{}; h.admitted.clearRetainingCapacity(); diff --git a/zig/src/core/lane_host_fail_test.zig b/zig/src/core/lane_host_fail_test.zig index 7c9601ba49..a9bc950175 100644 --- a/zig/src/core/lane_host_fail_test.zig +++ b/zig/src/core/lane_host_fail_test.zig @@ -10,6 +10,7 @@ const Box = struct { done: ?Reason = null, message: []const u8 = "", tokens: usize = 0, + finishes: usize = 0, fn event(ctx: *anyopaque, _: api.Id, e: *const api.Event) void { const b: *@This() = @ptrCast(@alignCast(ctx)); b.mutex.lockUncancelable(std.testing.io); @@ -17,12 +18,23 @@ const Box = struct { switch (e.*) { .tokens => |t| b.tokens += t.len, .finished => |f| { + b.finishes += 1; b.done = f.reason; b.message = f.message; }, else => {}, } } + fn waitFor(b: *@This(), ms: usize) ?Reason { + for (0..ms) |_| { + b.mutex.lockUncancelable(std.testing.io); + const done = b.done; + b.mutex.unlock(std.testing.io); + if (done != null) return done; + std.Io.sleep(std.testing.io, .fromMilliseconds(1), .awake) catch {}; + } + return null; + } fn wait(b: *@This()) Reason { while (true) { b.mutex.lockUncancelable(std.testing.io); @@ -62,3 +74,126 @@ test "a failed round ends the request it held; the queued request and a later on try std.testing.expectEqual(Reason.length, later.wait()); try std.testing.expectEqual(@as(usize, 48), later.tokens); } + +/// The last running stream frees a session, but moving its paused peer into it fails. +fn cancelledResumeFailure(in_prefill: bool) !void { + const gpa = std.testing.allocator; + var cfg = try lanes.Config.init(gpa, .{ .exact_width = 8, .hidden_rows = true }, 8, 7); + defer cfg.deinit(gpa); + var target: lanes.fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var clock: lanes.fake.FixedClock = .{}; + var core = lanes.Engine.init(gpa, &cfg, target.backend(), clock.clock()); + defer core.deinit(); + var host = LaneHost.init(gpa, std.testing.io, &core, .{ .lanes = 2 }); + const Ctx = struct { + engine: api.Engine, + waiting: ?*lanes.Stream = null, + in_prefill: bool, + fn prefill(ptr: *anyopaque, stream: *lanes.Stream, _: usize) void { + const ctx: *@This() = @ptrCast(@alignCast(ptr)); + if (stream.prompt()[0] == 2) { + ctx.waiting = stream; + stream.paused = true; + } else if (ctx.in_prefill) ctx.engine.cancel(2); + } + fn release(ptr: *anyopaque, stream: *lanes.Stream) void { + const ctx: *@This() = @ptrCast(@alignCast(ptr)); + const waiting = ctx.waiting orelse return; + if (stream == waiting) return; + waiting.paused = false; + waiting.finished = true; + waiting.reason = .@"error"; + waiting.problem = "migration failed"; + } + }; + var ctx: Ctx = .{ .engine = host.engine(), .in_prefill = in_prefill }; + target.prefill_hook = Ctx.prefill; + target.prefill_hook_ctx = &ctx; + target.release_hook = Ctx.release; + target.release_hook_ctx = &ctx; + const CancelBox = struct { + box: Box = .{}, + engine: api.Engine, + cancelled: bool = false, + fn event(ptr: *anyopaque, id: api.Id, event_: *const api.Event) void { + const b: *@This() = @ptrCast(@alignCast(ptr)); + Box.event(&b.box, id, event_); + if (event_.* == .tokens and !b.cancelled) { + b.cancelled = true; + b.engine.cancel(id); + } + } + }; + var waiting: Box = .{}; + var cancelled: CancelBox = .{ .engine = ctx.engine }; + const request: api.Request = .{ .prompt = &.{ 2, 7, 1, 8 }, .max_tokens = 100000, .drafts = false }; + try ctx.engine.submit(1, &request, .{ .ctx = &waiting, .event = Box.event }); + var second = request; + second.prompt = &.{ 8, 1, 2, 3 }; + try ctx.engine.submit(2, &second, .{ .ctx = &cancelled, .event = CancelBox.event }); + try host.start(); + var stopped = false; + defer if (!stopped) host.stop(); + try std.testing.expectEqual(@as(?Reason, .cancelled), cancelled.box.waitFor(2000)); + try std.testing.expectEqual(@as(?Reason, .failed), waiting.waitFor(2000)); + host.stop(); + stopped = true; + try std.testing.expectEqualStrings("migration failed", waiting.message); + try std.testing.expectEqual(@as(usize, 1), waiting.finishes); + try std.testing.expectEqual(@as(usize, 1), cancelled.box.finishes); + try std.testing.expectEqual(@as(usize, 0), core.live.items.len); + try std.testing.expectEqual(@as(usize, 0), target.lanes.count()); +} + +test "cancel of the last running stream delivers its paused peer's migration failure" { + try cancelledResumeFailure(false); +} + +test "cancel during prefill delivers its paused peer's migration failure after reaping it" { + try cancelledResumeFailure(true); +} + +test "a failed shared keep reaps an already finished stream before freeing its job" { + const gpa = std.testing.allocator; + var cfg = try lanes.Config.init(gpa, .{ .exact_width = 8, .hidden_rows = true, .streams_exact = true, .max_streams = 2 }, 8, 7); + defer cfg.deinit(gpa); + var target: lanes.fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + const Fault = struct { + fn keep(ptr: *anyopaque, windows: []const lanes.backend.Window, paths: []const []const u32) !void { + if (windows.len == 2) for (windows) |window| { + if (window.stream.finished) return error.GpuFailed; + }; + const fake: *lanes.fake.Fake = @ptrCast(@alignCast(ptr)); + return fake.backend().vtable.keep(ptr, windows, paths); + } + }; + var vtable = target.backend().vtable.*; + vtable.keep = Fault.keep; + var clock: lanes.fake.FixedClock = .{}; + var core = lanes.Engine.init(gpa, &cfg, .{ .ptr = &target, .vtable = &vtable }, clock.clock()); + defer core.deinit(); + var host = LaneHost.init(gpa, std.testing.io, &core, .{ .lanes = 2 }); + var short: Box = .{}; + var long: Box = .{}; + const a: api.Request = .{ .prompt = &.{ 2, 7, 1, 8 }, .max_tokens = 2, .drafts = false }; + var b = a; + b.max_tokens = 40; + const engine = host.engine(); + try engine.submit(1, &a, .{ .ctx = &short, .event = Box.event }); + try engine.submit(2, &b, .{ .ctx = &long, .event = Box.event }); + try host.start(); + var stopped = false; + defer if (!stopped) host.stop(); + try std.testing.expectEqual(@as(?Reason, .failed), short.waitFor(2000)); + try std.testing.expectEqual(@as(?Reason, .failed), long.waitFor(2000)); + host.stop(); + stopped = true; + try std.testing.expectEqualStrings("GpuFailed", short.message); + try std.testing.expectEqualStrings("GpuFailed", long.message); + try std.testing.expectEqual(@as(usize, 1), short.finishes); + try std.testing.expectEqual(@as(usize, 1), long.finishes); + try std.testing.expectEqual(@as(usize, 0), core.live.items.len); + try std.testing.expectEqual(@as(usize, 0), target.lanes.count()); +} diff --git a/zig/src/core/lanes/engine.zig b/zig/src/core/lanes/engine.zig index 9c329be91c..67d4bd0e55 100644 --- a/zig/src/core/lanes/engine.zig +++ b/zig/src/core/lanes/engine.zig @@ -115,24 +115,28 @@ pub const Engine = struct { e.sweep() catch {}; // its release may have ended another (a backend's paused stream that could not move) } - /// Releases the live streams that finished, again while a release ends another, so none outlives its job. - fn sweep(e: *Engine) !void { + /// Reaps finished streams before the host frees their jobs; tracing failure cannot interrupt cleanup. + pub fn sweep(e: *Engine) !void { + var failure: ?anyerror = null; var again = true; while (again) { again = false; - var kept: usize = 0; - for (e.live.items) |s| { - if (s.finished) { - try trail.finish(e, s); - e.release(s); - again = true; + var i: usize = 0; + while (i < e.live.items.len) { + const s = e.live.items[i]; + if (!s.finished) { + i += 1; continue; } - e.live.items[kept] = s; - kept += 1; + _ = e.live.orderedRemove(i); + trail.finish(e, s) catch |err| { + if (failure == null) failure = err; + }; + e.release(s); + again = true; } - e.live.shrinkRetainingCapacity(kept); } + if (failure) |err| return err; } /// One round for every live stream (Python `_family_step`). diff --git a/zig/src/core/lanes/engine_test.zig b/zig/src/core/lanes/engine_test.zig index 27d5cf63e8..1a70feef87 100644 --- a/zig/src/core/lanes/engine_test.zig +++ b/zig/src/core/lanes/engine_test.zig @@ -461,7 +461,7 @@ test "a family that keeps copies to shared rounds proposes none for a lone strea } } -/// A release hook that ends `ctx`'s stream with an error when another stream releases (a paused stream whose move failed). +/// A release ends a paused peer whose migration fails. fn endWaiting(ctx: *anyopaque, released: *sm.Stream) void { const waiting: *sm.Stream = @ptrCast(@alignCast(ctx)); if (released == waiting or waiting.finished) return; @@ -517,3 +517,30 @@ test "a discard whose release ends another stream leaves neither in the live lis try std.testing.expect(waiting.finished and waiting.reason == .@"error"); try std.testing.expectEqual(@as(usize, 0), engine.live.items.len); } + +test "discard reaps a peer even when trace allocations fail" { + var cfg = try model(); + defer cfg.deinit(gpa); + var target: fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var clock: fake.FixedClock = .{}; + var engine = Engine.init(gpa, &cfg, target.backend(), clock.clock()); + defer engine.deinit(); + var waiting = try sm.Stream.init(gpa, .{ .id = "waiting", .prompt = &p2, .max_new = 40, .drafts = false }); + defer waiting.deinit(gpa); + var other = try sm.Stream.init(gpa, .{ .id = "other", .prompt = &p1, .max_new = 40, .drafts = false }); + defer other.deinit(gpa); + try engine.addStream(&waiting); + try engine.addStream(&other); + waiting.paused = true; + target.release_hook = endWaiting; + target.release_hook_ctx = &waiting; + var failing = std.testing.FailingAllocator.init(gpa, .{ .fail_index = 0 }); + var log = @import("events.zig").Log.init(failing.allocator()); + defer log.deinit(); + engine.log = &log; + engine.discard(&other); + try std.testing.expect(waiting.finished); + try std.testing.expectEqual(@as(usize, 0), engine.live.items.len); + try std.testing.expectEqual(@as(usize, 0), target.lanes.count()); +} From 1d0f0dfb2b69d9d5dcdd2132d57ae4c13e89eed9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Fri, 16 Oct 2026 08:58:49 +0200 Subject: [PATCH 22/31] Port shared Flash Next session bindings to current upstream --- zig/src/families/flashnext/backend.zig | 94 +++++-------------- zig/src/families/flashnext/batch_costs.zig | 4 +- zig/src/families/flashnext/batch_forward.zig | 3 +- zig/src/families/flashnext/engine.zig | 4 +- zig/src/families/flashnext/host_sample.zig | 20 ++-- zig/src/families/flashnext/lone_stream.zig | 24 +++++ zig/src/families/flashnext/session.zig | 4 +- .../families/flashnext/session_binding.zig | 34 +++++++ zig/src/families/flashnext/slots.zig | 8 +- zig/src/tensorfold.zig | 2 +- 10 files changed, 102 insertions(+), 95 deletions(-) create mode 100644 zig/src/families/flashnext/lone_stream.zig create mode 100644 zig/src/families/flashnext/session_binding.zig diff --git a/zig/src/families/flashnext/backend.zig b/zig/src/families/flashnext/backend.zig index 2f86a49078..d54ca05b9f 100644 --- a/zig/src/families/flashnext/backend.zig +++ b/zig/src/families/flashnext/backend.zig @@ -1,4 +1,4 @@ -//! Flash Next's isolated sessions behind the shared lane scheduler; target rounds batch, MTP drafts are serial. +//! Isolated Flash Next sessions sharing target rounds and MTP command buffers. const std = @import("std"); const lanes = @import("lanes"); const fz = @import("replay.zig"); @@ -17,12 +17,6 @@ const sample = @import("host_sample.zig"); const think = @import("think.zig"); const batch_costs = @import("batch_costs.zig"); -/// A stream's sampling when it samples (a temperature above 0); null: the GPU's argmax stands. -fn sampling(stream: *const Stream) ?lanes.Sampling { - const p = stream.sampling orelse return null; - return if (p.temperature > 0) p else null; -} - /// Row `row` of a [rows, VOCAB] bf16 logits buffer, drawn with `p` keyed at `position`. fn drawRow(gpa: std.mem.Allocator, logits: Buf, row: usize, p: lanes.Sampling, position: u64) !u32 { const bytes = logits.b.contents()[logits.off + row * fz.VOCAB * 2 ..][0 .. fz.VOCAB * 2]; @@ -70,7 +64,7 @@ pub const Backend = struct { return .{ .gpa = gpa, .e = e, .primary = e.m, .sessions = sessions, .hidden = .{ .b = hidden }, .chain_drafts = env.on("FZ_BATCH_MTP_CHAIN"), .serial_drafts = env.on("FZ_BATCH_MTP_SERIAL") }; } - /// The lane allocator's prices, timed on the last session before any stream holds it; none (unpriced) when it can't be. + /// Times shared-round costs on the last free session, or leaves them unpriced when it cannot fit. pub fn timeCosts(b: *Backend) !void { if (std.c.getenv("FZ_BATCH_UNPRICED") != null) { std.log.info("flash next: FZ_BATCH_UNPRICED=1: shared rounds unpriced, nothing timed", .{}); @@ -101,8 +95,7 @@ pub const Backend = struct { pub fn facts(b: *Backend) lanes.Model { const c: ?*const batch_costs.Costs = if (b.costs) |*x| x else null; - // shared rounds priced from the timed forwards, each draft past a stream's first at a chained step's time; a - // lone stream's depth from the same (FZ_LONE_UNPRICED=1: the heuristic) + // Timed forwards price shared rounds; each extra draft costs one chained head step. const lone_priced = c != null and std.c.getenv("FZ_LONE_UNPRICED") == null; // not `lone`: that names the lone driver return .{ .exact_width = fz.MAXR, .first_copy_rows = 4, .mtp = true, .speculate = true, .speculate_early = false, .drafts = 8, .streams_exact = true, .hidden_rows = true, .batch_rows = fz.MAXR, .max_streams = @intCast(b.sessions.len), .draft_streams = !b.serial_drafts, .shared_costs = if (c) |x| &x.shared else &.{}, .draft_step_ms = if (c != null and !b.serial_drafts) c.?.step_ms else 0, .window_costs = if (lone_priced) &c.?.shared else &.{}, .mtp_step_ms = if (lone_priced) c.?.step_ms else 0 }; } @@ -136,37 +129,13 @@ pub const Backend = struct { }; } - const Binding = struct { - model: *fz.Model, - host: @FieldType(Engine, "host"), - cins: [2]Buf, - select: ?fz.Select, - shape: @TypeOf(@as(fz.Select, undefined).q_shape), - passed: @FieldType(Engine, "passed"), - pub fn restore(old: Binding, b: *Backend) void { - b.e.m = old.model; - b.e.host = old.host; - b.e.cins = old.cins; - b.e.r.sel = old.select; - b.e.r.shapes.put(b.e.r.arena, "Kc_shape", old.shape) catch unreachable; - b.e.passed = old.passed; - } - }; + const Binding = @import("session_binding.zig").Binding; - pub fn bind(b: *Backend, s: *Session) !Binding { - const e = b.e; - const old: Binding = .{ .model = e.m, .host = e.host, .cins = e.cins, .select = e.r.sel, .shape = e.r.shapes.get("Kc_shape").?, .passed = e.passed }; - try e.r.shapes.put(e.r.arena, "Kc_shape", s.shape); - e.m = &s.m; - e.host = .{ .rows = s.m.t.rows, .mdims = s.m.t.mdims, .pos8 = s.m.t.pos8, .nk8 = s.m.t.nk8, .kvmeta = s.m.t.kvmeta, .slots = s.m.mtp.slots }; - e.cins[0] = s.m.ple.cin; - e.r.sel = s.select[fz.CATCH - 1]; - e.passed = null; - e.hostMode(); - return old; + pub fn bind(b: *Backend, session_: *Session) !Binding { + return @import("session_binding.zig").bind(b.e, session_); } - /// A new stream's session (slots.pick); LaneMemory while only a busy one holds it (it waits), ContextFull when none can. + /// Chooses a slot; LaneMemory means wait, ContextFull means no session can hold the reply. fn slotFor(b: *Backend, prompt: usize, max_new: usize) !usize { var caps: [fz.MAXR]usize = undefined; var used: [fz.MAXR]bool = undefined; @@ -184,7 +153,7 @@ pub const Backend = struct { return i; } - /// Before a round, sized only: a stream its session cannot hold to its reply's end moves to a free larger one, or pauses (slots.room). + /// Moves or pauses a sized stream when its remaining reply needs a larger session. fn makeRoom(b: *Backend, stream: *Stream, rows: usize) !void { if (!b.sized) return; const i = b.by.get(stream) orelse return error.UnknownStream; @@ -209,7 +178,7 @@ pub const Backend = struct { } } - /// The stream in session `i` into the smallest free larger session holding its next `grow` tokens, else the largest free larger one. + /// Moves to the smallest free larger session holding the next grow tokens, or the largest available. fn move(b: *Backend, stream: *Stream, i: usize) !void { const from = &b.sessions[i]; const left: usize = @intCast(@max(stream.budgetLeft(), 0)); @@ -231,7 +200,7 @@ pub const Backend = struct { std.log.info("flash next: a stream at {d} tokens moved from a {d}-token session to a {d}-token one ({d} moves)", .{ from.m.pos, from.m.cap, b.sessions[j].m.cap, b.moves }); } - /// Paused streams into sessions that freed (slots.resumePaused): a move that fails but for LaneMemory ends the stream. + /// Resumes paused streams; errors other than LaneMemory terminate the affected stream. fn resumePaused(b: *Backend) void { slots.resumePaused(&b.paused, b); } @@ -283,7 +252,7 @@ pub const Backend = struct { s.row0 = 0; outputs(b, s, 0); const old = try b.bind(s); - defer old.restore(b); + defer old.restore(b.e); s.m.reset(); stream.cached = 0; if (stream.reuse.saved) |saved| { @@ -309,8 +278,8 @@ pub const Backend = struct { end = at + call.rows; const ps = [2]*fz.Prompt{ b.e.pr, b.e.pr2 }; s.first = try fz.Prompt.chunkN(ps[0..call.parts], &s.m, b.gpa, prompt[at..end]); - // the pass's last row's logits are in the model's head buffer until the next forward: a sampled first token now - if (end == prompt.len) if (sampling(stream)) |p| { + // Sample before the next forward replaces the last prompt row's logits. + if (end == prompt.len) if (sample.active(stream.sampling)) |p| { s.first = try drawRow(b.gpa, s.m.t.logits, 0, p, prompt.len); }; for (ps[0..call.parts], 0..) |p, k| { @@ -329,13 +298,7 @@ pub const Backend = struct { s.m.mtp.drafted = 0; } - /// Commits a lone stream's first token; its prompt as the context holds it after the commit, which can move it. - fn commitFirst(gpa: std.mem.Allocator, stream: *Stream, token: u32) ![]const u32 { - _ = try stream.commit(gpa, &.{token}, &.{}); - return stream.prompt(); - } - - /// LaneHost's lone driver: the shared prompt pass into session 0, then the engine's rounds until done (false) or handed over (true). + /// Runs a lone request in session 0 until it finishes or hands over to shared rounds. pub fn lone(b: *Backend, stream: *Stream, hooks: anytype) !bool { const pool = mtl.objc.Pool.push(); defer pool.pop(); @@ -345,7 +308,7 @@ pub const Backend = struct { var handed = false; defer if (!handed) release(b, stream); try b.fill(stream, 0); - const prompt = try commitFirst(b.gpa, stream, s.first); // decode copies it to the rounds' history before any round commits + const prompt = try @import("lone_stream.zig").commitFirst(b.gpa, stream, s.first); // decode copies it to the rounds' history before any round commits hooks.committed(hooks.ctx); if (stream.finished) return false; const Hooks = @TypeOf(hooks); @@ -373,7 +336,7 @@ pub const Backend = struct { var ho: Engine.Handoff = .{}; { const old = try b.bind(s); - defer old.restore(b); + defer old.restore(b.e); outputs(b, s, 0); var watch = try think.Watch.init(.{}); // no thinking budget: lone_host_lone.fits keeps those in the lane core const res = try b.e.decode(prompt, s.first, s.last, stream.max_new, stream.eos, null, &watch, .{ .ctx = &c, .prefilled = Ctx.prefilled, .tokens = Ctx.tokens, .cancelled = Ctx.cancelled }, &ho); @@ -480,7 +443,7 @@ pub const Backend = struct { try b.primary.finish(cb); Session.copy(b.primary.last, b.hidden, plan.total * fz.WIDE * 2); for (out, 0..) |o, i| @memcpy(o.sampled, t.picks.b.slice(u32, plan.total)[plan.starts[i]..][0..o.sampled.len]); - for (windows, out, 0..) |w, o, i| if (sampling(w.stream)) |p| { + for (windows, out, 0..) |w, o, i| if (sample.active(w.stream.sampling)) |p| { for (o.sampled, w.positions, 0..) |*tok, pos, j| tok.* = try drawRow(b.gpa, t.logits, plan.starts[i] + j, p, pos); }; b.window_count += 1; @@ -518,7 +481,7 @@ pub const Backend = struct { break :blk b.hidden.at(s.row0 * fz.WIDE * 2); } else s.last; const old = try b.bind(s); - defer old.restore(b); + defer old.restore(b.e); s.held_n = r.depth; if (follow.len == 0) return error.DraftOutOfStep; const start = mtl.clock.seconds(); @@ -565,7 +528,7 @@ pub const Backend = struct { if (follows[k].len > fz.MAXR) { // all but the last MAXR rows absorbed first, on their own const cut = follows[k].len - fz.MAXR; const old = try b.bind(s); - defer old.restore(b); + defer old.restore(b.e); _ = try s.m.mtpAbsorb(follows[k][0..cut], hiddens[k]); hiddens[k] = hiddens[k].at(cut * fz.WIDE * 2); follows[k] = follows[k][cut..]; @@ -577,7 +540,7 @@ pub const Backend = struct { for (requests, 0..) |r, k| { const s = try b.session(r.stream); const old = try b.bind(s); - defer old.restore(b); + defer old.restore(b.e); s.encodeDrafts(follows[k], hiddens[k], r.depth) catch |err| { b.e.r.enc.end(); return err; @@ -620,7 +583,7 @@ pub const Backend = struct { const b = self(ptr); const s = try b.session(@ptrCast(@alignCast(owner orelse return error.NoStream))); const old = try b.bind(s); - defer old.restore(b); + defer old.restore(b.e); return try snap.save(b.e, b.gpa, at); } pub fn snapRestore(_: *anyopaque, _: ?*anyopaque, _: *anyopaque) anyerror!void { @@ -630,18 +593,3 @@ pub const Backend = struct { snap.drop(self(ptr).gpa, @ptrCast(@alignCast(saved))); } }; - -test "a lone stream's prompt is read after its first token's commit moves the context" { - const gpa = std.testing.allocator; - const tokens = [_]u32{ 11, 12, 13, 14, 15, 16, 17 }; - var stream = try Stream.init(gpa, .{ .id = "lone", .prompt = &tokens, .max_new = 4, .eos = &.{} }); - defer stream.deinit(gpa); - stream.context.shrinkAndFree(gpa, stream.context.items.len); // full: the first token must grow it - var failing = std.testing.FailingAllocator.init(gpa, .{ .resize_fail_index = 0 }); // and not in place - const before = stream.context.items.ptr; - const prompt = try Backend.commitFirst(failing.allocator(), &stream, 99); - try std.testing.expect(stream.context.items.ptr != before); - try std.testing.expectEqual(stream.context.items.ptr, prompt.ptr); - try std.testing.expectEqualSlices(u32, &tokens, prompt); - try std.testing.expectEqualSlices(u32, &.{99}, stream.emitted()); -} diff --git a/zig/src/families/flashnext/batch_costs.zig b/zig/src/families/flashnext/batch_costs.zig index 66f66c41a2..19800fc692 100644 --- a/zig/src/families/flashnext/batch_costs.zig +++ b/zig/src/families/flashnext/batch_costs.zig @@ -36,7 +36,7 @@ pub fn time(b: anytype, s: *Session) !Costs { const pool = mtl.objc.Pool.push(); defer pool.pop(); const old = try b.bind(s); - defer old.restore(b); + defer old.restore(b.e); s.m.reset(); const ps = [1]*fz.Prompt{e.pr}; _ = try fz.Prompt.chunkN(&ps, &s.m, b.gpa, &prompt); @@ -92,7 +92,7 @@ fn chainMs(b: anytype, s: *Session, depth: usize) !f64 { const pool = mtl.objc.Pool.push(); defer pool.pop(); const old = try b.bind(s); - defer old.restore(b); + defer old.restore(b.e); s.m.mtp.pos = PROMPT - 1; s.m.mtp.drafted = 0; const follow = [1]u32{tokenAt(PROMPT)}; diff --git a/zig/src/families/flashnext/batch_forward.zig b/zig/src/families/flashnext/batch_forward.zig index 1a4e0fbca8..8ce901dc10 100644 --- a/zig/src/families/flashnext/batch_forward.zig +++ b/zig/src/families/flashnext/batch_forward.zig @@ -40,8 +40,7 @@ pub fn encode(e: *Engine, segs: []const Seg, rows: usize, ids: Buf) !void { } try m.hcProject(t.h[cur], l.ahc, "qa_hc_down@ahc", "qa_hc_up@ahc", t.inj_a); try m.lane(t.mixed, fz.D, l.proj, if (l.linear) "lane_qmm_bytes_grouped@gdn.in" else "lane_qmm_bytes_grouped@att.proj", t.p); - r.touch(l.out.wq); - r.touch(l.out.sbt); + @import("lane_w.zig").touch(r, l.out); if (l.linear) { for (segs) |g| { const v = view(t.*, g.row0); diff --git a/zig/src/families/flashnext/engine.zig b/zig/src/families/flashnext/engine.zig index 57aa1a6a49..a9da129915 100644 --- a/zig/src/families/flashnext/engine.zig +++ b/zig/src/families/flashnext/engine.zig @@ -613,7 +613,7 @@ pub const Engine = struct { /// Where `decode` left a stream it handed over: its state as a host round leaves it, `cache` rows in the cache. pub const Handoff = struct { yielded: bool = false, cache: usize = 0 }; - /// The GPU-side rounds after a prompt pass from `pick` and the last prompt row's `streams`; with `handoff` a cancel hands the stream over. + /// GPU rounds after prefill; cancellation yields through handoff when supplied. pub fn decode(e: *Engine, prompt: []const u32, pick: u32, streams: Buf, max_tokens: usize, eos: []const u32, depth: ?usize, watch: *think.Watch, out: Out, handoff: ?*Handoff) !Result { const r = e.r; const m = e.m; @@ -827,7 +827,7 @@ pub const Engine = struct { return res; } - /// The rounds handed over at `cache` rows: the round ahead already started as rounds do; the n-gram tail into the model's own buffer. + /// Hands over at cache rows after the queued round, restoring the n-gram tail. fn handOver(e: *Engine, h: *Handoff, round: usize, cache: usize) !void { if (e.r.gdn_kept != null) return error.HandoffUnsupported; // kept states replay their rows in the next window if ((round - 1) % 2 == 1) { diff --git a/zig/src/families/flashnext/host_sample.zig b/zig/src/families/flashnext/host_sample.zig index c388bb4fc2..566c5cee65 100644 --- a/zig/src/families/flashnext/host_sample.zig +++ b/zig/src/families/flashnext/host_sample.zig @@ -1,9 +1,13 @@ -//! The keyed host sampler over a bf16 logits row, in one pass: the top-k candidates by an order-preserving integer key -//! of each bf16 word, then lanes.sampling.choose. The candidates, their order and the draw equal `reference`, which is -//! qwen3_5/backend.zig's draw without its mask (two passes in f64), kept here for wide top-k and the tests. +//! Keyed bf16 sampling with bounded top-k or sorted wide candidates, matching the reference draw. const std = @import("std"); const lanes = @import("lanes"); +/// Positive-temperature sampling for a stream, or null for the GPU argmax. +pub fn active(params: ?lanes.Sampling) ?lanes.Sampling { + const p = params orelse return null; + return if (p.temperature > 0) p else null; +} + extern "c" fn exp(x: f64) f64; // libm's, as lanes.sampling's extern "c" fn log(x: f64) f64; @@ -87,9 +91,7 @@ pub fn draw(gpa: std.mem.Allocator, logits: []const u16, p: lanes.Sampling, posi return @intCast(try lanes.sampling.choose(gpa, values[0..filled], ids[0..filled], position, p)); } -/// A wide or unlimited top-k as `reference` draws it: the candidates in choose's order by a counting sort of their -/// keys (ids ascending within one), top-p and min-p cut as choose cuts them, then the keyed Gumbel draw in that order, -/// passing over candidates whose noise cannot lift them past the best score so far (most, without their two logs). +/// Wide top-k in reference order, with probability cuts and bounded-noise skips before the keyed draw. fn wide(gpa: std.mem.Allocator, logits: []const u16, p: lanes.Sampling, position: u64) !u32 { const starts = try gpa.alloc(u32, 1 << 16); defer gpa.free(starts); @@ -163,11 +165,11 @@ const below_e = blk: { break :blk e; }; -/// Whether the noise for `id` stays a nat short of `d`: -log(-log u) <= -log(1 - u), so 1 - u >= exp(-(d - 1)) -/// bounds it by d - 1, against exp of a whole nat at or below d - 1. 1 - u is exact for the f64 u the draw uses -/// (Sterbenz past 0.5; below it the bound cannot hold), and a u rounded to 1 gives 0: never passed over. +/// Whether the Gumbel noise stays below d - 1; a uniform value rounded to 1 is never skipped. fn beyondReach(seed: u64, position: u64, id: u64, d: f64) bool { + // Above 0.5 this subtraction is exact; below it the bound cannot hold. const one_minus_u = 1.0 - lanes.sampling.uniform(seed, position, id); + // -log(-log u) <= -log(1-u); compare against the whole-nat bound below d - 1. const j: usize = @intFromFloat(@floor(d) - 1); return one_minus_u >= below_e[j] * (1 + 1e-9); } diff --git a/zig/src/families/flashnext/lone_stream.zig b/zig/src/families/flashnext/lone_stream.zig new file mode 100644 index 0000000000..df567825e5 --- /dev/null +++ b/zig/src/families/flashnext/lone_stream.zig @@ -0,0 +1,24 @@ +//! The lone driver's first commit and the prompt slice it must copy after a possible relocation. +const std = @import("std"); +const Stream = @import("lanes").Stream; + +/// Commits a lone stream's first token; its prompt as the context holds it after the commit, which can move it. +pub fn commitFirst(gpa: std.mem.Allocator, stream: *Stream, token: u32) ![]const u32 { + _ = try stream.commit(gpa, &.{token}, &.{}); + return stream.prompt(); +} + +test "a lone stream's prompt is read after its first token's commit moves the context" { + const gpa = std.testing.allocator; + const tokens = [_]u32{ 11, 12, 13, 14, 15, 16, 17 }; + var stream = try Stream.init(gpa, .{ .id = "lone", .prompt = &tokens, .max_new = 4, .eos = &.{} }); + defer stream.deinit(gpa); + stream.context.shrinkAndFree(gpa, stream.context.items.len); // full: the first token must grow it + var failing = std.testing.FailingAllocator.init(gpa, .{ .resize_fail_index = 0 }); // and not in place + const before = stream.context.items.ptr; + const prompt = try commitFirst(failing.allocator(), &stream, 99); + try std.testing.expect(stream.context.items.ptr != before); + try std.testing.expectEqual(stream.context.items.ptr, prompt.ptr); + try std.testing.expectEqualSlices(u32, &tokens, prompt); + try std.testing.expectEqualSlices(u32, &.{99}, stream.emitted()); +} diff --git a/zig/src/families/flashnext/session.zig b/zig/src/families/flashnext/session.zig index 9d819d40a4..5eb00e653a 100644 --- a/zig/src/families/flashnext/session.zig +++ b/zig/src/families/flashnext/session.zig @@ -142,7 +142,7 @@ pub const Session = struct { @memcpy(s.held[0..depth], s.draft_ids.b.slice(u32, depth)); } - /// This session's stream into `dst`, a free larger session: cache rows, states, n-gram tail and head copied on the GPU. + /// Copies this session's cache, recurrent state, n-gram tail and head to a free larger session on the GPU. pub fn moveTo(s: *Session, dst: *Session, e: *Engine) !void { const m = &s.m; const d = &dst.m; @@ -216,7 +216,7 @@ pub const Session = struct { } } - /// encodeDrafts' drafts once their command buffer is done: held, the head's chained rows trimmed at its next absorb. + /// Reads completed encoded drafts and trims chained rows at the next absorb. pub fn heldDrafts(s: *Session, depth: usize) void { s.m.mtp.drafted = depth -| 1; @memcpy(s.held[0..depth], s.draft_ids.b.slice(u32, depth)); diff --git a/zig/src/families/flashnext/session_binding.zig b/zig/src/families/flashnext/session_binding.zig new file mode 100644 index 0000000000..ce4f6b0df1 --- /dev/null +++ b/zig/src/families/flashnext/session_binding.zig @@ -0,0 +1,34 @@ +//! Saves and restores the engine views while a shared session owns its encoder. +const fz = @import("replay.zig"); +const Engine = @import("engine.zig").Engine; +const Session = @import("session.zig").Session; +const Buf = fz.Buf; + +pub const Binding = struct { + model: *fz.Model, + host: @FieldType(Engine, "host"), + cins: [2]Buf, + select: ?fz.Select, + shape: @TypeOf(@as(fz.Select, undefined).q_shape), + passed: @FieldType(Engine, "passed"), + pub fn restore(old: Binding, e: *Engine) void { + e.m = old.model; + e.host = old.host; + e.cins = old.cins; + e.r.sel = old.select; + e.r.shapes.put(e.r.arena, "Kc_shape", old.shape) catch unreachable; + e.passed = old.passed; + } +}; + +pub fn bind(e: *Engine, s: *Session) !Binding { + const old: Binding = .{ .model = e.m, .host = e.host, .cins = e.cins, .select = e.r.sel, .shape = e.r.shapes.get("Kc_shape").?, .passed = e.passed }; + try e.r.shapes.put(e.r.arena, "Kc_shape", s.shape); + e.m = &s.m; + e.host = .{ .rows = s.m.t.rows, .mdims = s.m.t.mdims, .pos8 = s.m.t.pos8, .nk8 = s.m.t.nk8, .kvmeta = s.m.t.kvmeta, .slots = s.m.mtp.slots }; + e.cins[0] = s.m.ple.cin; + e.r.sel = s.select[fz.CATCH - 1]; + e.passed = null; + e.hostMode(); + return old; +} diff --git a/zig/src/families/flashnext/slots.zig b/zig/src/families/flashnext/slots.zig index 8ebf99ed07..dadb5acd00 100644 --- a/zig/src/families/flashnext/slots.zig +++ b/zig/src/families/flashnext/slots.zig @@ -6,7 +6,7 @@ const fz = @import("replay.zig"); /// Rows a window past a stream's last committed token can take. pub const WINDOW = fz.MAXR + 1; -/// The sessions past the engine's own from `text` (counts, `CONTEXTxN` for N alike), each at most `capacity`, `most` in all. +/// Parses up to most extra sessions; CONTEXTxN repeats a size, each capped at capacity. pub fn parse(gpa: std.mem.Allocator, text: []const u8, capacity: usize, margin: usize, most: usize) ![]usize { var caps: std.ArrayList(usize) = .empty; errdefer caps.deinit(gpa); @@ -24,7 +24,7 @@ pub fn parse(gpa: std.mem.Allocator, text: []const u8, capacity: usize, margin: return try caps.toOwnedSlice(gpa); } -/// A new stream's session: unsized the first free one (the caller checks it holds the reply), sized the smallest free one holding the prompt and `grow`. +/// Chooses first-free when unsized, else the smallest free session holding the prompt and grow tokens. pub fn pick(caps: []const usize, used: []const bool, sized: bool, prompt: usize, max_new: usize, grow: usize) ?usize { const need = prompt + @min(max_new, grow) + WINDOW; var best: ?usize = null; @@ -39,7 +39,7 @@ pub fn pick(caps: []const usize, used: []const bool, sized: bool, prompt: usize, /// What a stream needs from its session before a round. pub const Room = enum { ok, move, pause, full }; -/// Room in a `cap`-row session at `pos` for a `rows` window, the reply ending by `end`, `most` the largest session, `larger` one free. +/// Checks the next window and reply end against this session, the largest session and free larger slots. pub fn room(cap: usize, pos: usize, rows: usize, end: usize, most: usize, larger: bool) Room { if (end <= cap) return .ok; // the reply ends inside this session: it never grows if (end > most) return .full; // no session holds it: never wait for one @@ -57,7 +57,7 @@ pub fn endPaused(stream: *lanes.Stream) void { stream.problem = "the reply outgrew its session and could not move to a larger one"; } -/// Paused streams moved, oldest first: `ctx.moveOut` failing LaneMemory keeps one waiting, any other failure ends it (`ctx.failed`). +/// Resumes oldest first; LaneMemory waits, other moveOut errors call failed. pub fn resumePaused(paused: *std.ArrayList(*lanes.Stream), ctx: anytype) void { var k: usize = 0; while (k < paused.items.len) { diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 2d5281c76a..2aff84feba 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -66,7 +66,7 @@ test { _ = @import("families/flashnext/dense.zig"); // The core projection adapter keeps format, cuts and tiles. _ = @import("families/flashnext/marks.zig"); // a call's marks _ = @import("families/flashnext/think.zig"); // a thinking budget's close on the host - _ = @import("families/flashnext/backend.zig"); // the shared sessions' backend (a lone stream's prompt after its first commit) + _ = @import("families/flashnext/lone_stream.zig"); // the first commit can relocate the prompt _ = @import("families/flashnext/slots.zig"); // sized shared sessions: the list, a stream's session, when it grows _ = @import("families/flashnext/batch_costs.zig"); // the shared rounds' timing needs a session that holds it _ = @import("families/flashnext/think_check.zig"); // and in the verdict kernels (Metal) From dd1b74164c6cb1b7da1a83539825bbe31ea189bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Fri, 16 Oct 2026 08:58:49 +0200 Subject: [PATCH 23/31] Serve default Flash Next requests on memory-sized shared sessions --- docs/recipes/qwen3.8-flash-next.md | 7 ++-- zig/src/families/flashnext/batch_meta.zig | 5 +++ .../families/flashnext/batch_meta_test.zig | 12 +++++++ zig/src/native/flashnext_batch_host.zig | 32 ++++++++++++------- zig/src/native/flashnext_host.zig | 23 +++++++++---- zig/src/native/metal.zig | 21 +++++++----- 6 files changed, 71 insertions(+), 29 deletions(-) diff --git a/docs/recipes/qwen3.8-flash-next.md b/docs/recipes/qwen3.8-flash-next.md index eea6f4620b..6a586b7337 100644 --- a/docs/recipes/qwen3.8-flash-next.md +++ b/docs/recipes/qwen3.8-flash-next.md @@ -73,7 +73,7 @@ to one stream, while successful checks allow the lane engine to combine requests ## Native Metal shared sessions -The native 6-bit MTP engine enables shared target rounds with an explicit `--parallel N` above one: +The native 6-bit MTP engine uses shared target rounds with the default `--parallel auto` when at least two sessions fit. An explicit count fixes the number of sessions: ```bash tensorfold-native serve TensorFold/Qwen3.8-Flash-Next-MLX-6bit-MTP \ @@ -82,11 +82,10 @@ tensorfold-native serve TensorFold/Qwen3.8-Flash-Next-MLX-6bit-MTP \ Each request holds its own recurrent state, attention cache, n-gram history and MTP cache. The scheduler packs up to 16 target rows, including draft rows, into one layer loop. Projections and experts process those rows -together; recurrence, convolution and attention read each request's own state. MTP heads draft serially. +together; recurrence, convolution and attention read each request's own state. Every stream's MTP drafts share one command buffer; `FZ_BATCH_MTP_SERIAL=1` restores serial drafting. New requests join between rounds, and cancelling a request releases its slot for another request. -`auto` and `--parallel 1` retain the existing single-reply driver. Shared sessions currently run on one Mac; -`--parallel N` above one with `--speed-up` is refused. Use `--parallel 1` for the two-Mac link. +`auto` chooses up to 16 sessions from the memory left after loading the model and reserving the prompt cache. If fewer than two fit, it opens the single-reply driver instead. Explicit `--parallel 1`, `--learn`, and the two-Mac path retain their supported driver. Shared sessions run on one Mac; `--parallel N` above one with `--speed-up` is refused. Use `--parallel 1` for the two-Mac link. The ceiling is 70% of RAM, or `FLASHNEXT_LOAD_LIMIT_GB` (GB, up to the GPU's recommended working set). It covers the server's whole footprint: the model, the sessions and the prompt cache. Left out, `--prompt-cache-gib` takes what the diff --git a/zig/src/families/flashnext/batch_meta.zig b/zig/src/families/flashnext/batch_meta.zig index 78feb2c428..576fc0a77d 100644 --- a/zig/src/families/flashnext/batch_meta.zig +++ b/zig/src/families/flashnext/batch_meta.zig @@ -57,3 +57,8 @@ pub fn fit(want: u32, fixed: bool, room: u64, per_slot: u64) !u32 { if (fixed and n < want) return error.OverMemoryLimit; return n; } + +/// Default auto uses shared sessions; explicit one, learning and paired serving keep their own host. +pub fn shared(fixed: bool, streams: u32, paired: bool, learning: bool) bool { + return (!fixed or streams > 1) and !paired and !learning; +} diff --git a/zig/src/families/flashnext/batch_meta_test.zig b/zig/src/families/flashnext/batch_meta_test.zig index 70c03f1715..f1f92ac184 100644 --- a/zig/src/families/flashnext/batch_meta_test.zig +++ b/zig/src/families/flashnext/batch_meta_test.zig @@ -62,3 +62,15 @@ test "shared draft before keep and lone draft after rejection use the same forwa try lone.begin(0, 1); try std.testing.expectError(error.WindowOutOfStep, lone.begin(1, 1)); } + +test "default auto selects shared sessions while explicit one, learning and pairs keep their host" { + try std.testing.expect(meta.shared(false, 8, false, false)); + try std.testing.expect(meta.shared(true, 4, false, false)); + try std.testing.expect(!meta.shared(true, 1, false, false)); + try std.testing.expect(!meta.shared(false, 8, true, false)); + try std.testing.expect(!meta.shared(false, 8, false, true)); + try std.testing.expect(!meta.shared(true, 4, false, true)); + try std.testing.expectEqual(@as(u32, 1), try meta.fit(16, false, 0, 1024)); + try std.testing.expectEqual(@as(u32, 3), try meta.fit(16, false, 2048, 1024)); + try std.testing.expectError(error.OverMemoryLimit, meta.fit(4, true, 2048, 1024)); +} diff --git a/zig/src/native/flashnext_batch_host.zig b/zig/src/native/flashnext_batch_host.zig index 7f5e2ea2a8..d6ea811ea6 100644 --- a/zig/src/native/flashnext_batch_host.zig +++ b/zig/src/native/flashnext_batch_host.zig @@ -21,8 +21,7 @@ const Host = struct { warm: mtl.keepalive.Target, }; -/// What the server may hold in all, sessions and prompt cache: 70% of RAM, or FLASHNEXT_LOAD_LIMIT_GB (GB, as -/// GLM_LOAD_LIMIT_GB) up to the GPU's recommended working set; past the default the log names the risk, as GLM's does. +/// The whole-footprint ceiling: 70% of RAM or FLASHNEXT_LOAD_LIMIT_GB, capped by the GPU working set. fn ceiling(total: u64, working_set: u64) error{BadLoadLimit}!u64 { const text: ?[]const u8 = if (std.c.getenv("FLASHNEXT_LOAD_LIMIT_GB")) |v| std.mem.span(v) else null; const c = fit.ceiling(total, working_set, text) catch |err| { @@ -41,13 +40,13 @@ fn slotContexts(gpa: std.mem.Allocator, capacity: usize) !?[]usize { return try tf.flashnext_slots.parse(gpa, text, capacity, @intCast(fx.MARGIN), tf.flashnext_replay.MAXR - 1); } -/// Reply tokens a session must hold past the prompt at admission (FLASHNEXT_SLOT_GROW, default 8192); past them a stream moves. +/// Admission requires FLASHNEXT_SLOT_GROW reply tokens (default 8192); longer replies can move. fn slotGrow() usize { const v = std.c.getenv("FLASHNEXT_SLOT_GROW") orelse return 8192; return std.fmt.parseInt(usize, std.mem.span(v), 10) catch 8192; } -pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const u8, window: i64, streams: u32, cache_gib: ?f64, over: bool, a: std.mem.Allocator, why: *[]const u8) !api.Opened { +pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const u8, window: i64, streams: u32, streams_fixed: bool, cache_gib: ?f64, over: bool, a: std.mem.Allocator, why: *[]const u8) !api.Opened { const pool = mtl.objc.Pool.push(); defer pool.pop(); const capacity: usize = @intCast(@min(tf.flashnext_replay.CAP, window + fx.MARGIN)); @@ -55,20 +54,22 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const errdefer eng.deinit(); if (eng.r.tp != null) return error.BatchSpeedUpUnsupported; try eng.warm(); - eng.hostMode(); const ready = fit.footprint() orelse return error.NoMemoryReading; const total = fit.ram() orelse return error.NoMemoryReading; const cap = try ceiling(total, eng.r.device.maxWorkingSet()); const reserved = if (!over) (if (cache_gib) |g| (if (g > 0) std.math.lossyCast(u64, g * fit.GiB) else 0) else 0) else 0; - const fixed = Session.overhead(tf.flashnext_replay.CAP) + tf.flashnext_replay.MAXR * tf.flashnext_replay.WIDE * 2; - const available = cap -| ready -| fit.MARGIN -| reserved; - if (available < fixed) return error.OverMemoryLimit; - const room = available - fixed; const tiers = slotContexts(gpa, capacity) catch |err| { std.log.err("flash next: FLASHNEXT_SLOT_CONTEXTS is not a list of sessions: token counts of 1024 or more, CONTEXTxN for N alike, at most {d} sessions past the engine's own", .{tf.flashnext_replay.MAXR - 1}); return err; }; defer if (tiers) |t| gpa.free(t); + const overhead = Session.overhead(tf.flashnext_replay.CAP) + tf.flashnext_replay.MAXR * tf.flashnext_replay.WIDE * 2; + const available = cap -| ready -| fit.MARGIN -| reserved; + if (available < overhead) { + if (!streams_fixed and tiers == null) return single(gpa, io, eng, window, cache_gib, over, cap, a, why); + return error.OverMemoryLimit; + } + const room = available - overhead; var caps: std.ArrayList(usize) = .empty; defer caps.deinit(gpa); try caps.append(gpa, capacity); // the engine's own session: the whole window @@ -81,9 +82,11 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const } try caps.appendSlice(gpa, t); } else { - const n = try tf.flashnext_batch_meta.fit(streams, true, room, Session.bytes(capacity)); + const n = try tf.flashnext_batch_meta.fit(streams, streams_fixed, room, Session.bytes(capacity)); + if (n < 2) return single(gpa, io, eng, window, cache_gib, over, cap, a, why); for (1..n) |_| try caps.append(gpa, capacity); } + eng.hostMode(); const slots: u32 = @intCast(caps.items.len); const h = try gpa.create(Host); errdefer gpa.destroy(h); @@ -106,7 +109,7 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const const plan = try fit.budgetUnder(cap, cache_gib, over, 0, a, why, ""); // the cache under the same limit as the sessions eng.snap_pool.max = plan.budget_bytes; h.host = api.LaneHost.init(gpa, io, &h.core, .{ .name = "flashnext-zig", .lanes = slots, .context_window = @intCast(window), .prefill_step = @intCast(eng.pr.step), .prompt_cache_plan = plan, .admit = if (tiers != null) .{ .ctx = &h.back, .fits = Backend.fits } else null }); - // a lone greedy request runs on the engine's GPU-side rounds until a second one arrives (FLASHNEXT_NO_LONE=1: shared rounds) + // FLASHNEXT_NO_LONE=1 keeps even lone requests on shared rounds. if (eng.r.gdn_kept == null and std.c.getenv("FLASHNEXT_NO_LONE") == null) h.host.lone = .{ .ctx = h, .run = loneRun }; h.cache = null; if (plan.budget_bytes > 0) { @@ -148,3 +151,10 @@ fn close(ctx: *anyopaque) void { h.eng.deinit(); h.gpa.destroy(h); } + +/// Auto with room for only one session transfers the loaded engine, preserving the chosen footprint limit. +fn single(gpa: std.mem.Allocator, io: std.Io, eng: *fx.Engine, window: i64, cache_gib: ?f64, over: bool, cap: u64, a: std.mem.Allocator, why: *[]const u8) !api.Opened { + const host = @import("flashnext_host.zig"); + const h = try host.openLoaded(gpa, io, eng, window, cache_gib, over, null, cap, a, why); + return .{ .engine = h.engine(), .close = host.close, .ctx = h }; +} diff --git a/zig/src/native/flashnext_host.zig b/zig/src/native/flashnext_host.zig index 8281b19df4..a65dd650a1 100644 --- a/zig/src/native/flashnext_host.zig +++ b/zig/src/native/flashnext_host.zig @@ -33,6 +33,7 @@ pub const Host = struct { warm: mtl.keepalive.Target = undefined, // the engine's queue, for the server's idle ticker resident: ?Resident = null, // the engine's buffers, kept wired between requests info_: api.Info, + load_limit: u64 = 0, mutex: std.Io.Mutex = .init, wake: std.Io.Condition = .init, queued: std.ArrayList(*Job) = .empty, @@ -175,8 +176,8 @@ pub const Host = struct { }; } - fn memoryFn(_: *anyopaque, reset_peak: bool) ?api.Memory { - return cache_fit.hostMemory(0, reset_peak); // the process footprint; no configured load limit + fn memoryFn(ctx: *anyopaque, reset_peak: bool) ?api.Memory { + return cache_fit.hostMemory(self(ctx).load_limit, reset_peak); } fn emit(job: *Job, event: api.Event) void { @@ -395,9 +396,14 @@ fn ms(ns: i96) f64 { } /// The prompt cache's plan (cache_fit.zig); without an OS reading, `gib` or the Metal working set's spare. -fn cacheBudget(eng: *fx.Engine, gib: ?f64, over: bool, a: Allocator, why: *[]const u8) !api.PromptCachePlan { +fn cacheBudget(eng: *fx.Engine, gib: ?f64, over: bool, limit: ?u64, a: Allocator, why: *[]const u8) !api.PromptCachePlan { const rank = if (eng.followsPeer()) " (rank 1 keeps its halves of rank 0's)" else if (eng.r.tp != null) " (rank 1 mirrors them)" else ""; const dev = eng.r.device; + if (limit) |cap| { + const ready = cache_fit.footprint() orelse return error.NoMemoryReading; + if (ready +| cache_fit.MARGIN > cap) return error.OverMemoryLimit; + return cache_fit.budgetUnder(cap, gib, over, dev.maxWorkingSet() -| dev.allocated(), a, why, rank); + } return cache_fit.budget(gib, over, dev.maxWorkingSet() -| dev.allocated() -| (8 << 30), a, why, rank); } @@ -417,6 +423,11 @@ fn learnedStates(gpa: Allocator, io: std.Io, eng: *fx.Engine, root: []const u8, pub fn open(gpa: Allocator, io: std.Io, dir: []const u8, dump: ?[]const u8, window: i64, speed_up: ?[]const u8, cache_gib: ?f64, over_cap: bool, learn: ?Learn, a: Allocator, why: *[]const u8) !*Host { const eng = try fx.Engine.loadWith(gpa, io, dir, dump, speed_up); errdefer eng.deinit(); + return openLoaded(gpa, io, eng, window, cache_gib, over_cap, learn, null, a, why); +} + +/// Opens a loaded engine without loading its weights again; the caller retains ownership on error. +pub fn openLoaded(gpa: Allocator, io: std.Io, eng: *fx.Engine, window: i64, cache_gib: ?f64, over_cap: bool, learn: ?Learn, limit: ?u64, a: Allocator, why: *[]const u8) !*Host { if (learn != null and eng.r.tp != null) { why.* = "--learn is not kept across a Flash Next pair yet: leave it out, or serve on one Mac"; return error.LearnOnPair; @@ -425,15 +436,15 @@ pub fn open(gpa: Allocator, io: std.Io, dir: []const u8, dump: ?[]const u8, wind std.log.err("flash next: warm-up failed: {s}", .{@errorName(err)}); return err; }; - const cache_plan = try cacheBudget(eng, cache_gib, over_cap, a, why); + const cache_plan = try cacheBudget(eng, cache_gib, over_cap, limit, a, why); const budget = cache_plan.budget_bytes; eng.peer_budget = budget; // rank 1: its kept states and free buffers stay inside the same budget eng.snap_pool.max = budget; const follower: ?std.Thread = if (eng.followsPeer()) try std.Thread.spawn(.{}, follow, .{eng}) else null; // rank 1 const h = try gpa.create(Host); errdefer gpa.destroy(h); - const limit: i64 = tf.flashnext_replay.CAP - fx.MARGIN; - h.* = .{ .gpa = gpa, .io = io, .eng = eng, .follower = follower, .info_ = .{ .name = "flashnext-zig", .prompt_cache_plan = cache_plan, .lanes = 1, .greedy_only = eng.r.tp != null, .context_window = @intCast(if (window > 0) @min(window, limit) else limit) } }; + const context_limit: i64 = tf.flashnext_replay.CAP - fx.MARGIN; + h.* = .{ .gpa = gpa, .io = io, .eng = eng, .follower = follower, .load_limit = limit orelse 0, .info_ = .{ .name = "flashnext-zig", .prompt_cache_plan = cache_plan, .lanes = 1, .greedy_only = eng.r.tp != null, .context_window = @intCast(if (window > 0) @min(window, context_limit) else context_limit) } }; h.warm = .{ .queue = eng.r.queue }; // macOS unwires idle GPU memory within seconds: the engine's buffers join one set the idle keepalive's ticks use if (std.c.getenv("FLASHNEXT_NO_RESIDENCY") == null) holdResident(h) catch |err| std.log.warn("flash next: no residency set ({s}): the first request after an idle second waits for the model's memory", .{@errorName(err)}); diff --git a/zig/src/native/metal.zig b/zig/src/native/metal.zig index ef44d0e2f7..8d527814ee 100644 --- a/zig/src/native/metal.zig +++ b/zig/src/native/metal.zig @@ -385,16 +385,21 @@ fn openFlashNext(a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: problem.* = try std.fmt.allocPrint(a, "--context {d} exceeds this model's {d}-token window", .{ window, native }); return null; } - if (o.lanes_fixed and o.lanes > 1) { - if (o.speed_up != null or std.c.getenv("TF_FLASHNEXT_TP") != null) { - problem.* = "Flash Next shared sessions are single-Mac only; use --parallel 1 with --speed-up"; - return null; - } + const paired = o.speed_up != null or std.c.getenv("TF_FLASHNEXT_TP") != null; + if (o.lanes_fixed and o.lanes > 1 and paired) { + problem.* = "Flash Next shared sessions are single-Mac only; use --parallel 1 with --speed-up"; + return null; + } + if (tf.flashnext_batch_meta.shared(o.lanes_fixed, o.lanes, paired, o.learn != null)) { var why: []const u8 = ""; - return @import("flashnext_batch_host.zig").open(gpa, io, o.dir, dump, if (window > 0) @min(window, tf.flashnext_replay.CAP - tf.flashnext_engine.MARGIN) else tf.flashnext_replay.CAP - tf.flashnext_engine.MARGIN, o.lanes, o.prompt_cache_gib, o.prompt_cache_over_cap, a, &why) catch |err| { - problem.* = if (err == error.CacheOverCap) why else if (err == error.BadLoadLimit) "FLASHNEXT_LOAD_LIMIT_GB is not a load limit: give a number of GB above 0, or unset it for 70% of RAM" else if (err == error.OverMemoryLimit) "the native Flash Next batching engine cannot open (OverMemoryLimit): the sessions and the given prompt cache do not fit under the load limit (70% of RAM, or FLASHNEXT_LOAD_LIMIT_GB) past the loaded model; fewer streams, a smaller --context or --prompt-cache-gib, or a higher FLASHNEXT_LOAD_LIMIT_GB" else try std.fmt.allocPrint(a, "the native Flash Next batching engine cannot open ({s})", .{@errorName(err)}); - return null; + const shared: ?api.Opened = shared: { + const opened = @import("flashnext_batch_host.zig").open(gpa, io, o.dir, dump, if (window > 0) @min(window, tf.flashnext_replay.CAP - tf.flashnext_engine.MARGIN) else tf.flashnext_replay.CAP - tf.flashnext_engine.MARGIN, if (o.lanes_fixed) o.lanes else tf.flashnext_replay.MAXR, o.lanes_fixed, o.prompt_cache_gib, o.prompt_cache_over_cap, a, &why) catch |err| { + problem.* = tf.flashnext_engine.quantizationProblem(err) orelse if (err == error.CacheOverCap) why else if (err == error.BadLoadLimit) "FLASHNEXT_LOAD_LIMIT_GB is not a load limit: give a number of GB above 0, or unset it for 70% of RAM" else if (err == error.OverMemoryLimit) "the native Flash Next batching engine cannot open (OverMemoryLimit): the sessions and prompt cache do not fit under the load limit" else try std.fmt.allocPrint(a, "the native Flash Next batching engine cannot open ({s})", .{@errorName(err)}); + return null; + }; + break :shared opened; }; + if (shared) |opened| return opened; } const pool = mtl.objc.Pool.push(); defer pool.pop(); From 4e184b2e4e9ae87cc4bcf7661ab6157a3a2e1dce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:04:53 +0200 Subject: [PATCH 24/31] Retain shared-family tests after extracting the first-token lifetime helper --- zig/src/tensorfold.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 2aff84feba..56c6c77be4 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -66,6 +66,7 @@ test { _ = @import("families/flashnext/dense.zig"); // The core projection adapter keeps format, cuts and tiles. _ = @import("families/flashnext/marks.zig"); // a call's marks _ = @import("families/flashnext/think.zig"); // a thinking budget's close on the host + _ = @import("families/flashnext/backend.zig"); // shared-session family tests _ = @import("families/flashnext/lone_stream.zig"); // the first commit can relocate the prompt _ = @import("families/flashnext/slots.zig"); // sized shared sessions: the list, a stream's session, when it grows _ = @import("families/flashnext/batch_costs.zig"); // the shared rounds' timing needs a session that holds it From fac556364d40f276837d4425be3f0933ef366062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:11:14 +0200 Subject: [PATCH 25/31] Describe the default shared route consistently in the README and recipe --- README.md | 9 +++++---- docs/recipes/qwen3.8-flash-next.md | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 31403c8397..51c7001269 100644 --- a/README.md +++ b/README.md @@ -118,10 +118,11 @@ Every accepted draft must equal the token the same native engine would produce w A resumed request must equal fresh execution, and each concurrent stream must equal its solo run. The comparison fixes the checkpoint, backend, settings and runtime. Different quantizations and different backends can produce different outputs. -Flash Next on Metal uses shared target rounds with an explicit `--parallel N` above one on a single Mac. -Its default `auto`, `--parallel 1` and two-Mac speed-up mode serve one active reply; MTP heads remain serial. -In shared mode, `FZ_BATCH_MTP_CHAIN=1` experimentally submits each stream's absorb and draft chain in one Metal command buffer. -The option defaults to off. See the [MTP chain receipt](docs/receipts/flashnext-mtp-chain-2026-10-10/README.md) for qualification and measurements. +Flash Next on Metal uses shared target rounds with default `--parallel auto` when at least two sessions fit. +A lone greedy request uses GPU-side rounds and hands over when a second request arrives. +Explicit `--parallel 1`, learning, two-Mac serving and the one-session fallback use the single-reply driver. +`FZ_BATCH_MTP_SERIAL=1` restores serial MTP drafting on the shared host. +The [earlier MTP chain receipt](docs/receipts/flashnext-mtp-chain-2026-10-10/README.md) records its own pinned commits. Nemotron, GLM, the 2B model and Qwen3.8-27B use shared lane rounds. The 27B's DFlash2 drafter drafts for one stream at a time: a request alone, or the one in the drafter's slot when several share rounds. diff --git a/docs/recipes/qwen3.8-flash-next.md b/docs/recipes/qwen3.8-flash-next.md index 6a586b7337..a527b9434b 100644 --- a/docs/recipes/qwen3.8-flash-next.md +++ b/docs/recipes/qwen3.8-flash-next.md @@ -73,11 +73,11 @@ to one stream, while successful checks allow the lane engine to combine requests ## Native Metal shared sessions -The native 6-bit MTP engine uses shared target rounds with the default `--parallel auto` when at least two sessions fit. An explicit count fixes the number of sessions: +The native 6-bit MTP engine uses shared target rounds with the default `--parallel auto` when at least two sessions fit. Without `FLASHNEXT_SLOT_CONTEXTS`, an explicit count fixes the number of sessions. The default needs no parallel flag: ```bash tensorfold-native serve TensorFold/Qwen3.8-Flash-Next-MLX-6bit-MTP \ - --parallel 4 --context 16384 --temperature 0 + --context 16384 --temperature 0 ``` Each request holds its own recurrent state, attention cache, n-gram history and MTP cache. The scheduler packs From f36482a7713340d43e30f2d9b0b4457f4fcedad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:26:34 +0200 Subject: [PATCH 26/31] Gate default Flash Next serving with observed shared GPU rounds --- .github/workflows/no-gpu.yml | 2 + docs/recipes/qwen3.8-flash-next.md | 16 ++- tools/zig/flashnext_batch_serve.py | 52 +++++--- tools/zig/flashnext_batch_serve_test.py | 151 ++++++++++++++++++++++++ 4 files changed, 203 insertions(+), 18 deletions(-) create mode 100644 tools/zig/flashnext_batch_serve_test.py diff --git a/.github/workflows/no-gpu.yml b/.github/workflows/no-gpu.yml index 597ff6b522..a5ce78f146 100644 --- a/.github/workflows/no-gpu.yml +++ b/.github/workflows/no-gpu.yml @@ -30,6 +30,8 @@ jobs: # Cortheon's readers of untrusted web bytes under the fuzz harness: seeds, 100,000 edits, linear time. - name: zig build fuzz-cortheon run: zig build fuzz-cortheon + - name: Served Flash Next gate regressions + run: python3 -m unittest discover -s tools/zig -p flashnext_batch_serve_test.py # The lean rule (600-line files, one-line comments within 120 columns): no problem the base branch lacks. - name: lean check env: diff --git a/docs/recipes/qwen3.8-flash-next.md b/docs/recipes/qwen3.8-flash-next.md index a527b9434b..4f768938eb 100644 --- a/docs/recipes/qwen3.8-flash-next.md +++ b/docs/recipes/qwen3.8-flash-next.md @@ -113,12 +113,20 @@ Run with one model process on the GPU. For a served comparison with identical re ```bash zig build native python tools/zig/flashnext_batch_serve.py zig-out/native/bin/tensorfold-native \ - MODEL_DIR /tmp/flashnext-prompts /tmp/flashnext-served --parallel 1,4 + MODEL_DIR /tmp/flashnext-prompts /tmp/flashnext-served \ + --parallel 1,default,auto,4 --context 16384 --require-shared-rounds ``` -The served gate starts one server at a time on loopback port 52417, compares response bodies with the serial -reference and records wall time, generated tokens and peak running and waiting requests. End-to-end time includes -the prompts. The checkpoint gate separately establishes exact token equality. +The served gate starts one server at a time on loopback port 52417. `default` omits `--parallel`; `auto` passes +it explicitly. Each block uses the same context and requests, including thinking on and off. It compares response +bodies, token counts and native token fingerprints with the serial reference and records wall time and peak +running and waiting requests. `--require-shared-rounds` also requires multi-stream GPU forwards in the batch log +during those requests. Running requests alone do not prove shared rounds. If auto falls back to one session, +that shared-round gate fails and the receipt retains the observed results. + +End-to-end time includes prompts. The shared-round check enables `FZ_BATCH_LOG`, so its times include logging; +omit that option for a separate timing run. Repeat into a fresh output directory with `--context 65536` or another +context that fits the checkpoint. The checkpoint gate separately checks prompt admission, cancellation and reuse. ## CUDA diff --git a/tools/zig/flashnext_batch_serve.py b/tools/zig/flashnext_batch_serve.py index 672a211a4b..74fa525289 100644 --- a/tools/zig/flashnext_batch_serve.py +++ b/tools/zig/flashnext_batch_serve.py @@ -3,7 +3,9 @@ import argparse from concurrent.futures import ThreadPoolExecutor import json +import os from pathlib import Path +import re import subprocess import threading import time @@ -49,12 +51,16 @@ def request(case, tokens, gate): def run(args, slots, cases, block): log_path = args.output / f"serve-{slots}-block-{block}.log" + command = [str(args.binary), "serve", str(args.model), "--name", "batch-gate", "--host", "127.0.0.1", + "--port", "52417", "--context", str(getattr(args, "context", 16384)), + "--temperature", "0", "--prompt-cache-gib", "0", "--no-update-check"] + if str(slots) != "default": + command += ["--parallel", str(slots)] + env = os.environ.copy() + if getattr(args, "require_shared_rounds", False): + env["FZ_BATCH_LOG"] = "1" with log_path.open("w") as log: - process = subprocess.Popen([ - str(args.binary), "serve", str(args.model), "--name", "batch-gate", "--host", "127.0.0.1", - "--port", "52417", "--context", "16384", "--parallel", str(slots), - "--temperature", "0", "--prompt-cache-gib", "0", "--no-update-check", - ], stdout=log, stderr=subprocess.STDOUT) + process = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, env=env) (args.output / "server.pid").write_text(str(process.pid) + "\n") try: deadline = time.monotonic() + 240 @@ -72,6 +78,7 @@ def run(args, slots, cases, block): raise TimeoutError("native server startup") batches = [] for iteration in range(args.rounds): + log_start = log_path.stat().st_size stop = threading.Event() peaks = [0, 0] @@ -95,13 +102,18 @@ def monitor(): futures = [pool.submit(request, case, args.tokens, gate) for case in cases] results = [future.result() for future in futures] seconds = time.monotonic() - start + log_end = log_path.stat().st_size finally: stop.set() watcher.join(timeout=4) generated = sum(result["tokens"] for result in results) + with log_path.open("rb") as observed: + observed.seek(log_start) + rounds = re.findall(rb"flashnext batch: streams (\d+), rows \d+, shared rounds \d+", + observed.read(log_end - log_start)) batch = {"iteration": iteration, "seconds": seconds, "tokens": generated, "tokens_per_second": generated / seconds, "peak_running": peaks[0], - "peak_waiting": peaks[1], "results": results} + "peak_waiting": peaks[1], "shared_rounds": sum(int(n) > 1 for n in rounds), "results": results} batches.append(batch) print(json.dumps({"parallel": slots, **{key: batch[key] for key in ("iteration", "seconds", "tokens", "tokens_per_second", "peak_running", "peak_waiting")}}), flush=True) @@ -124,15 +136,21 @@ def main(): parser.add_argument("output", type=Path) parser.add_argument("--tokens", type=int, default=128) parser.add_argument("--rounds", type=int, default=3) - parser.add_argument("--parallel", default="1,4") + parser.add_argument("--parallel", default="1,default,auto,4", + help="comma-separated blocks starting with 1; default omits --parallel") + parser.add_argument("--context", type=int, default=16384) + parser.add_argument("--require-shared-rounds", action="store_true", + help="enable batch logging and require multi-stream GPU rounds (timings include logging)") args = parser.parse_args() - if args.tokens < 1 or args.rounds < 1: - parser.error("tokens and rounds must be positive") - slots = [int(value) for value in args.parallel.split(",")] - if not slots or slots[0] != 1 or any(value < 1 or value > 16 for value in slots): - parser.error("parallel must start with 1 and stay within 1..16") + if args.tokens < 1 or args.rounds < 1 or args.context < 1: + parser.error("tokens, rounds and context must be positive") + slots = args.parallel.split(",") + if slots[0] != "1" or any(value not in {"default", "auto", *(str(n) for n in range(1, 17))} for value in slots): + parser.error("parallel must start with 1 and contain default, auto or 1..16") args.output.mkdir(parents=True, exist_ok=True) - receipt = {"reply_token_limit": args.tokens, "rounds": args.rounds, "exact": False, "runs": {}} + receipt = {"reply_token_limit": args.tokens, "rounds": args.rounds, "context": args.context, + "exact": False, "shared_rounds_verified": False, + "batch_logging": args.require_shared_rounds or "FZ_BATCH_LOG" in os.environ, "runs": {}} (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") cases = [json.loads((args.fixtures / f"{name}.json").read_text()) for name in ("short", "thinking", "sparse", "sparse-thinking")] @@ -142,6 +160,7 @@ def main(): for batch in batches: batch["block"] = block receipt["runs"].setdefault(str(count), []).extend(batches) + (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") for batch in batches: replies = [{key: result[key] for key in ("reply", "tokens", "prompt_tokens", "token_sha")} for result in batch["results"]] @@ -151,10 +170,15 @@ def main(): receipt["exact"] = False (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") raise RuntimeError("HTTP replies differ from serial reference") - if count > 1 and not any(batch["peak_running"] > 1 for batch in batches): + if count != "1" and not any(batch["peak_running"] > 1 for batch in batches): raise RuntimeError("no simultaneous HTTP sessions observed") + if args.require_shared_rounds and count != "1" and not any(batch["shared_rounds"] > 0 for batch in batches): + raise RuntimeError("no shared GPU rounds observed; auto may have fallen back to one session") (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") receipt["exact"] = True + receipt["shared_rounds_verified"] = any(key != "1" for key in receipt["runs"]) and all( + any(batch["shared_rounds"] > 0 for batch in batches) + for key, batches in receipt["runs"].items() if key != "1") (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") diff --git a/tools/zig/flashnext_batch_serve_test.py b/tools/zig/flashnext_batch_serve_test.py new file mode 100644 index 0000000000..4c036ad44f --- /dev/null +++ b/tools/zig/flashnext_batch_serve_test.py @@ -0,0 +1,151 @@ +"""CPU regressions for served-gate routing and honest qualification receipts.""" +import contextlib +import argparse +import io +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import flashnext_batch_serve as gate + + +class ServedGateTests(unittest.TestCase): + def exercise(self, modes="1,default,auto,4", shared=True, mismatch=False, startup_only=False, + shared_blocks=None, legacy=False, late_log=False): + folder = tempfile.TemporaryDirectory() + self.addCleanup(folder.cleanup) + root = Path(folder.name) + fixtures = root / "fixtures" + fixtures.mkdir() + for name in ("short", "thinking", "sparse", "sparse-thinking"): + (fixtures / f"{name}.json").write_text(json.dumps({"text": name, "thinking": "thinking" in name})) + output = root / "out" + commands = [] + current = {} + + class Process: + pid = 12345 + returncode = None + + def __init__(self, command, stdout, **kwargs): + commands.append(command) + current.update(command=command, log=stdout) + if startup_only: + stdout.write("info: flashnext batch: streams 4, rows 8, shared rounds 99\n") + stdout.flush() + + def poll(self): + return None + + def terminate(self): + self.returncode = 0 + + def wait(self, **kwargs): + return 0 + + def get(path): + if path == "/health": + return '{"status":"ok"}' + return "tensorfold:requests_running 4\n" + + def request(case, tokens, barrier): + barrier.wait(timeout=2) + multi = "--parallel" not in current["command"] or current["command"][current["command"].index("--parallel") + 1] != "1" + if shared and multi and (shared_blocks is None or len(commands) - 1 in shared_blocks): + current["log"].write("info: flashnext batch: streams 4, rows 8, shared rounds 1\n") + current["log"].flush() + return {"reply": {"message": {"content": case["text"]}, "finish_reason": "length"}, + "tokens": tokens, "prompt_tokens": 12, "cached_tokens": 0, + "token_sha": ("b" if mismatch and multi else "a") * 12, "seconds": 0.01} + + argv = ["gate", "binary", "model", str(fixtures), str(output), "--parallel", modes, + "--context", "32768", "--rounds", "1", "--require-shared-rounds"] + run = gate.run + thread = gate.threading.Thread + + class MonitorThread(thread): + def __init__(self, *args, **kwargs): + self.monitor = getattr(kwargs.get("target"), "__name__", "") == "monitor" + super().__init__(*args, **kwargs) + + def join(self, *args, **kwargs): + if late_log and self.monitor: + current["log"].write("info: flashnext batch: streams 4, rows 8, shared rounds 99\n") + current["log"].flush() + return super().join(*args, **kwargs) + + def invoke(args, slots, cases, block): + if legacy: + args = argparse.Namespace(**{k: v for k, v in vars(args).items() + if k not in {"context", "require_shared_rounds"}}) + slots = int(slots) + return run(args, slots, cases, block) + + try: + with patch("sys.argv", argv), patch.object(gate.subprocess, "Popen", Process), \ + patch.object(gate, "get", get), patch.object(gate, "request", request), \ + patch.object(gate, "run", invoke), \ + patch.object(gate.threading, "Thread", MonitorThread), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + gate.main() + finally: + self.receipt = json.loads((output / "receipt.json").read_text()) if output.exists() else None + self.commands = commands + return self.receipt + + def test_default_auto_and_fixed_commands_use_same_context_and_qualify(self): + report = self.exercise() + self.assertTrue(report["exact"]) + self.assertTrue(report["shared_rounds_verified"]) + self.assertEqual(report["context"], 32768) + self.assertEqual(len(self.commands), 4) + self.assertNotIn("--parallel", self.commands[1]) + self.assertEqual(self.commands[2][self.commands[2].index("--parallel") + 1], "auto") + self.assertTrue(all(c[c.index("--context") + 1] == "32768" for c in self.commands)) + self.assertEqual(list(report["runs"]), ["1", "default", "auto", "4"]) + + def test_running_requests_alone_cannot_prove_shared_rounds(self): + with self.assertRaisesRegex(RuntimeError, "shared GPU rounds"): + self.exercise(shared=False) + self.assertFalse(self.receipt["exact"]) + self.assertFalse(self.receipt["shared_rounds_verified"]) + self.assertIn("default", self.receipt["runs"]) + + def test_mismatch_preserves_failing_block_and_cannot_qualify(self): + with self.assertRaisesRegex(RuntimeError, "differ"): + self.exercise(mismatch=True) + self.assertFalse(self.receipt["exact"]) + self.assertIn("default", self.receipt["runs"]) + + def test_startup_rounds_cannot_qualify_requests(self): + with self.assertRaisesRegex(RuntimeError, "shared GPU rounds"): + self.exercise(shared=False, startup_only=True) + self.assertFalse(self.receipt["shared_rounds_verified"]) + + def test_rounds_after_requests_cannot_qualify_them(self): + with self.assertRaisesRegex(RuntimeError, "shared GPU rounds"): + self.exercise(shared=False, late_log=True) + self.assertFalse(self.receipt["shared_rounds_verified"]) + + def test_every_repeated_auto_block_needs_its_own_shared_rounds(self): + with self.assertRaisesRegex(RuntimeError, "shared GPU rounds"): + self.exercise(modes="1,auto,auto", shared_blocks={1}) + self.assertFalse(self.receipt["exact"]) + self.assertEqual([b["shared_rounds"] > 0 for b in self.receipt["runs"]["auto"]], [True, False]) + + def test_legacy_integer_run_without_new_arguments_still_works(self): + report = self.exercise(modes="1,4", legacy=True) + self.assertTrue(report["exact"]) + self.assertTrue(all(c[c.index("--context") + 1] == "16384" for c in self.commands)) + + def test_invalid_modes_fail_before_loading_a_model(self): + for modes in ("auto,1", "1,17", "1,0", "1,typo", "1,"): + with self.subTest(modes=modes), self.assertRaises(SystemExit): + self.exercise(modes=modes) + self.assertEqual(self.commands, []) + + +if __name__ == "__main__": + unittest.main() From b038a3dba5cfe207de4e332a91b0134f619e2eec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:40:42 +0200 Subject: [PATCH 27/31] Preserve Flash Next residency and lifecycle after the upstream rebase --- zig/src/core/lanes/engine.zig | 18 +++++------ zig/src/native/flashnext_batch_host.zig | 42 ++++++++++++++++++++++++- zig/src/native/flashnext_host.zig | 4 +++ 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/zig/src/core/lanes/engine.zig b/zig/src/core/lanes/engine.zig index 67d4bd0e55..6ae633fe4e 100644 --- a/zig/src/core/lanes/engine.zig +++ b/zig/src/core/lanes/engine.zig @@ -124,16 +124,14 @@ pub const Engine = struct { var i: usize = 0; while (i < e.live.items.len) { const s = e.live.items[i]; - if (!s.finished) { - i += 1; - continue; - } - _ = e.live.orderedRemove(i); - trail.finish(e, s) catch |err| { - if (failure == null) failure = err; - }; - e.release(s); - again = true; + if (s.finished) { + _ = e.live.orderedRemove(i); + trail.finish(e, s) catch |err| { + if (failure == null) failure = err; + }; + e.release(s); + again = true; + } else i += 1; } } if (failure) |err| return err; diff --git a/zig/src/native/flashnext_batch_host.zig b/zig/src/native/flashnext_batch_host.zig index d6ea811ea6..a04e7f0ab5 100644 --- a/zig/src/native/flashnext_batch_host.zig +++ b/zig/src/native/flashnext_batch_host.zig @@ -8,6 +8,7 @@ const Backend = tf.flashnext_backend.Backend; const Session = tf.flashnext_session.Session; const fx = tf.flashnext_engine; const fit = @import("cache_fit.zig"); +const Resident = @import("resident.zig").Resident; const Host = struct { gpa: std.mem.Allocator, @@ -19,6 +20,8 @@ const Host = struct { host: api.LaneHost, cache: ?api.prompt_cache.Store = null, warm: mtl.keepalive.Target, + resident: ?Resident, + load_limit: u64, }; /// The whole-footprint ceiling: 70% of RAM or FLASHNEXT_LOAD_LIMIT_GB, capped by the GPU working set. @@ -113,14 +116,21 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, dump: ?[]const if (eng.r.gdn_kept == null and std.c.getenv("FLASHNEXT_NO_LONE") == null) h.host.lone = .{ .ctx = h, .run = loneRun }; h.cache = null; if (plan.budget_bytes > 0) { - h.cache = api.prompt_cache.Store.init(gpa, .{ .ptr = &h.back, .vtable = &.{ .bytes = Backend.snapBytes, .save = Backend.snapSave, .restore = Backend.snapRestore, .drop = Backend.snapDrop, .charged = Backend.snapCharged, .spare = Backend.snapSpare, .trim = Backend.snapTrim, .reuses = Backend.snapReuses } }, .{ .lookahead = 1, .planned = true }, plan.budget_bytes); + h.cache = api.prompt_cache.Store.init(gpa, .{ .ptr = &h.back, .vtable = &.{ .bytes = Backend.snapBytes, .save = Backend.snapSave, .restore = Backend.snapRestore, .drop = Backend.snapDrop, .charged = Backend.snapCharged, .spare = Backend.snapSpare, .trim = Backend.snapTrim, .reuses = Backend.snapReuses } }, .{ .lookahead = 1, .planned = true, .warm = .on }, plan.budget_bytes); h.host.cache = &h.cache.?; h.host.info_.prompt_cache = true; + h.host.info_.warm_turns = true; } errdefer if (h.cache) |*store| store.deinit(); h.host.explain = .{ .text = words }; h.warm = .{ .queue = eng.r.queue }; + h.resident = null; + h.load_limit = cap; + if (std.c.getenv("FLASHNEXT_NO_RESIDENCY") == null) holdResident(h) catch |err| + std.log.warn("flash next shared: no residency set ({s})", .{@errorName(err)}); + errdefer dropResident(h); h.host.keepalive_target = .{ .ctx = &h.warm, .tick = mtl.keepalive.Target.tick }; + h.host.memory = .{ .ctx = h, .read = memory }; try h.host.start(); var listed: [tf.flashnext_replay.MAXR * 8]u8 = undefined; var w: std.Io.Writer = .fixed(&listed); @@ -145,6 +155,7 @@ fn close(ctx: *anyopaque) void { const h: *Host = @ptrCast(@alignCast(ctx)); h.host.stop(); if (h.cache) |*store| store.deinit(); + dropResident(h); h.core.deinit(); h.cfg.deinit(h.gpa); h.back.deinit(); @@ -152,6 +163,35 @@ fn close(ctx: *anyopaque) void { h.gpa.destroy(h); } +fn memory(ctx: ?*anyopaque, reset_peak: bool) ?api.Memory { + const h: *Host = @ptrCast(@alignCast(ctx.?)); + return fit.hostMemory(h.load_limit, reset_peak); +} + +/// Session-owned buffers stay allocated until close, beside the runner's buffers and shared hidden rows. +fn holdResident(h: *Host) !void { + var buffers: std.ArrayList(mtl.Buffer) = .empty; + defer buffers.deinit(h.gpa); + try buffers.appendSlice(h.gpa, h.eng.r.held.items); + for (h.back.sessions) |s| try buffers.appendSlice(h.gpa, s.owned.items); + try buffers.append(h.gpa, h.back.hidden.b); + h.resident = try Resident.hold(h.gpa, h.host.io, h.eng.r.device, h.eng.r.queue, buffers.items, "flash next shared") orelse return; + const r = &h.resident.?; + h.warm = r.target(); + h.eng.r.follow = r.follow(); + h.eng.snap_pool.follow = r.follow(); +} + +/// Stop reporting new buffers before releasing the set and then the session/backend buffers it held. +fn dropResident(h: *Host) void { + const r = if (h.resident) |*x| x else return; + h.eng.r.follow = null; + h.eng.snap_pool.follow = null; + r.deinit(); + h.resident = null; + h.warm = .{ .queue = h.eng.r.queue }; +} + /// Auto with room for only one session transfers the loaded engine, preserving the chosen footprint limit. fn single(gpa: std.mem.Allocator, io: std.Io, eng: *fx.Engine, window: i64, cache_gib: ?f64, over: bool, cap: u64, a: std.mem.Allocator, why: *[]const u8) !api.Opened { const host = @import("flashnext_host.zig"); diff --git a/zig/src/native/flashnext_host.zig b/zig/src/native/flashnext_host.zig index a65dd650a1..4f9491123c 100644 --- a/zig/src/native/flashnext_host.zig +++ b/zig/src/native/flashnext_host.zig @@ -441,6 +441,10 @@ pub fn openLoaded(gpa: Allocator, io: std.Io, eng: *fx.Engine, window: i64, cach eng.peer_budget = budget; // rank 1: its kept states and free buffers stay inside the same budget eng.snap_pool.max = budget; const follower: ?std.Thread = if (eng.followsPeer()) try std.Thread.spawn(.{}, follow, .{eng}) else null; // rank 1 + errdefer if (follower) |th| { + eng.stopFollowing(); + th.join(); + }; const h = try gpa.create(Host); errdefer gpa.destroy(h); const context_limit: i64 = tf.flashnext_replay.CAP - fx.MARGIN; From 8cee8efddfc08cbf783aecc95a743d143d67010e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:40:42 +0200 Subject: [PATCH 28/31] Draw shared Flash Next picks with the upstream GPU sampler --- build.zig | 2 +- docs/recipes/qwen3.8-flash-next.md | 18 ++++-- zig/src/families/flashnext/backend.zig | 17 ++--- zig/src/families/flashnext/batch_sample.zig | 26 ++++++++ zig/src/families/flashnext/host_sample.zig | 2 +- zig/src/tensorfold.zig | 1 + zig/tests/flashnext_batch.zig | 38 +++++++---- zig/tests/flashnext_batch_sample.zig | 70 +++++++++++++++++++++ 8 files changed, 146 insertions(+), 28 deletions(-) create mode 100644 zig/src/families/flashnext/batch_sample.zig create mode 100644 zig/tests/flashnext_batch_sample.zig diff --git a/build.zig b/build.zig index afbd805645..4846765121 100644 --- a/build.zig +++ b/build.zig @@ -266,7 +266,7 @@ fn nativeServer(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.b .target = target, .optimize = optimize, .link_libc = true, - .imports = &.{ .{ .name = "metal", .module = metal }, .{ .name = "tensorfold", .module = engine } }, + .imports = &.{ .{ .name = "metal", .module = metal }, .{ .name = "tensorfold", .module = engine }, .{ .name = "kernel_sources", .module = engine.import_table.get("kernel_sources").? } }, }) }); b.step("tf-flashnext-batch", "Flash Next shared rounds against fresh legacy replies with prompt admission and cancellation").dependOn(&b.addInstallArtifact(batch, .{}).step); return exe; diff --git a/docs/recipes/qwen3.8-flash-next.md b/docs/recipes/qwen3.8-flash-next.md index 4f768938eb..46defb8b9e 100644 --- a/docs/recipes/qwen3.8-flash-next.md +++ b/docs/recipes/qwen3.8-flash-next.md @@ -95,20 +95,30 @@ pass. Additional slots allocate caches for the selected context. Startup reserves their metadata and the prompt-cache budget, then checks the physical footprint before accepting requests. A fixed slot count that does not fit is refused; lower the count, context or prompt-cache budget. `--prompt-cache-over-cap` overrides only the prompt-cache -budget. Shared sessions sample: a request with a temperature draws each token on the host from the head's logits -with the server's keyed sampler (top-k, top-p, min-p, a `seed` or one derived from the prompt), so a sampled reply is -the same with drafting on and off and alone or beside other streams. `--parallel 1` still decodes greedily only. +budget. Shared sessions draw sampled tokens with the same GPU sampler as the single-Mac `--parallel 1` path. +Each packed window uses its own seed, sampling settings and absolute token positions. The two-Mac path remains +greedy only. The model gate below must verify sampled token parity across routes, drafting and batch composition. + +The shared host's residency set includes the engine, session caches and metadata, shared hidden rows and later +prompt snapshots. Residency stays within 70% of RAM even when the load limit is raised; a refused set leaves the +server running without it. `FLASHNEXT_NO_RESIDENCY=1` disables the set. With a prompt cache, background warm turns +keep the end of a reply as the single-request host does. Idle latency and throughput need fresh measurements. The checkpoint gate compares shared replies with fresh legacy replies, with drafting off and on, then checks prompt admission, cancellation, slot reuse and a restored prefix while another request remains active: ```bash zig build tf-flashnext-batch +zig-out/bin/tf-flashnext-batch --sampling-only python tools/zig/flashnext_batch_prompts.py MODEL_DIR /tmp/flashnext-prompts FZ_N=128 zig-out/bin/tf-flashnext-batch MODEL_DIR /tmp/flashnext-prompts/*.json +FZ_SAMPLE=1 FZ_N=128 zig-out/bin/tf-flashnext-batch MODEL_DIR /tmp/flashnext-prompts/*.json ``` -Run with one model process on the GPU. For a served comparison with identical requests and settings: +These device gates require exclusive GPU use. `--sampling-only` uses synthetic logits and checks distinct seeds, +positions and row offsets against the shared GPU-rule reference, including untouched greedy and spare rows. +`FZ_SAMPLE=1` repeats the model gate with fixed seeds per prompt. Compilation alone does not pass either gate. +For a served comparison with identical requests and settings: ```bash zig build native diff --git a/zig/src/families/flashnext/backend.zig b/zig/src/families/flashnext/backend.zig index d54ca05b9f..f8e23b7860 100644 --- a/zig/src/families/flashnext/backend.zig +++ b/zig/src/families/flashnext/backend.zig @@ -13,16 +13,10 @@ const Buf = fz.Buf; const mtl = @import("metal"); const segments = @import("../../core/segments.zig"); const slots = @import("slots.zig"); -const sample = @import("host_sample.zig"); +const sample = @import("batch_sample.zig"); const think = @import("think.zig"); const batch_costs = @import("batch_costs.zig"); -/// Row `row` of a [rows, VOCAB] bf16 logits buffer, drawn with `p` keyed at `position`. -fn drawRow(gpa: std.mem.Allocator, logits: Buf, row: usize, p: lanes.Sampling, position: u64) !u32 { - const bytes = logits.b.contents()[logits.off + row * fz.VOCAB * 2 ..][0 .. fz.VOCAB * 2]; - return sample.draw(gpa, @as([*]const u16, @ptrCast(@alignCast(bytes.ptr)))[0..fz.VOCAB], p, position); -} - pub const Backend = struct { gpa: std.mem.Allocator, e: *Engine, @@ -280,7 +274,7 @@ pub const Backend = struct { s.first = try fz.Prompt.chunkN(ps[0..call.parts], &s.m, b.gpa, prompt[at..end]); // Sample before the next forward replaces the last prompt row's logits. if (end == prompt.len) if (sample.active(stream.sampling)) |p| { - s.first = try drawRow(b.gpa, s.m.t.logits, 0, p, prompt.len); + s.first = try sample.first(b.e, &s.m, p, @intCast(prompt.len)); }; for (ps[0..call.parts], 0..) |p, k| { const start = at + segments.start(call.rows, call.parts, k); @@ -440,12 +434,13 @@ pub const Backend = struct { const cb = b.e.r.queue.commandBuffer(); b.e.r.enc = cb.compute(.serial); try forward.encode(b.e, segs[0..windows.len], plan.total, t.ids8); + @memcpy(b.e.draw_at.b.slice(u32, fz.MAXR)[0..plan.total], plan.keys[0..plan.total]); + for (windows, 0..) |w, i| if (sample.active(w.stream.sampling)) |p| { + sample.encode(b.e.r.enc, b.e.r.draw_pipe, t.logits, b.e.draw_at, t.picks, plan.starts[i], w.rows(), p); + }; try b.primary.finish(cb); Session.copy(b.primary.last, b.hidden, plan.total * fz.WIDE * 2); for (out, 0..) |o, i| @memcpy(o.sampled, t.picks.b.slice(u32, plan.total)[plan.starts[i]..][0..o.sampled.len]); - for (windows, out, 0..) |w, o, i| if (sample.active(w.stream.sampling)) |p| { - for (o.sampled, w.positions, 0..) |*tok, pos, j| tok.* = try drawRow(b.gpa, t.logits, plan.starts[i] + j, p, pos); - }; b.window_count += 1; if (windows.len > 1) b.shared_count += 1; if (std.c.getenv("FZ_BATCH_LOG") != null) diff --git a/zig/src/families/flashnext/batch_sample.zig b/zig/src/families/flashnext/batch_sample.zig new file mode 100644 index 0000000000..4e7f78f5a9 --- /dev/null +++ b/zig/src/families/flashnext/batch_sample.zig @@ -0,0 +1,26 @@ +//! Shared windows use the legacy engine's GPU sampler with each stream's absolute draw positions. +const lanes = @import("lanes"); +const mtl = @import("metal"); +const fz = @import("replay.zig"); +const Engine = @import("engine.zig").Engine; +const sample = @import("sample.zig"); + +pub fn active(params: ?lanes.Sampling) ?lanes.Sampling { + const p = params orelse return null; + return if (p.temperature > 0) p else null; +} + +/// Buffer offsets isolate each window; its positions must be filled before the command buffer is committed. +pub fn encode(enc: mtl.ComputeEncoder, pipe: mtl.Pipeline, logits: fz.Buf, positions: fz.Buf, picks: fz.Buf, row: usize, rows: usize, p: lanes.Sampling) void { + sample.draw(enc, pipe, logits.at(row * fz.VOCAB * 2), positions.at(row * 4), picks.at(row * 4), rows, fz.VOCAB, p); +} + +/// The last prompt row's pick, before another prompt or draft overwrites its logits. +pub fn first(e: *Engine, model: *fz.Model, p: lanes.Sampling, position: u32) !u32 { + e.draw_at.b.slice(u32, fz.MAXR)[0] = position; + const cb = e.r.queue.commandBuffer(); + e.r.enc = cb.compute(.serial); + encode(e.r.enc, e.r.draw_pipe, model.t.logits, e.draw_at, model.t.picks, 0, 1, p); + try model.finish(cb); + return model.t.picks.b.slice(u32, 1)[0]; +} diff --git a/zig/src/families/flashnext/host_sample.zig b/zig/src/families/flashnext/host_sample.zig index 566c5cee65..28144fed8c 100644 --- a/zig/src/families/flashnext/host_sample.zig +++ b/zig/src/families/flashnext/host_sample.zig @@ -1,4 +1,4 @@ -//! Keyed bf16 sampling with bounded top-k or sorted wide candidates, matching the reference draw. +//! Historical bf16 host sampler; shared serving uses batch_sample.zig and the GPU rule. const std = @import("std"); const lanes = @import("lanes"); diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 56c6c77be4..8872bdfbfb 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -31,6 +31,7 @@ pub const nemotron = @import("families/nemotron/nemotron.zig"); pub const flashnext_replay = @import("families/flashnext/replay.zig"); pub const flashnext_rows = @import("families/flashnext/rows_load.zig"); pub const flashnext_sample = @import("families/flashnext/host_sample.zig"); +pub const flashnext_batch_sample = @import("families/flashnext/batch_sample.zig"); pub const flashnext_engine = @import("families/flashnext/engine.zig"); pub const flashnext_snapshot = @import("families/flashnext/snapshot.zig"); pub const flashnext_session = @import("families/flashnext/session.zig"); diff --git a/zig/tests/flashnext_batch.zig b/zig/tests/flashnext_batch.zig index 32d6027f58..0fafdc24bd 100644 --- a/zig/tests/flashnext_batch.zig +++ b/zig/tests/flashnext_batch.zig @@ -1,4 +1,4 @@ -//! Checkpoint gate: shared target rounds against fresh legacy greedy replies, including admission and cancellation. +//! Checkpoint gate: shared target rounds against legacy replies, including seeded draws, admission and cancellation. const std = @import("std"); const mtl = @import("metal"); const tf = @import("tensorfold"); @@ -32,11 +32,15 @@ const Reply = struct { return false; } fn marked(_: *anyopaque, _: usize) void {} - fn out(r: *Reply) fx.Out { - return .{ .ctx = r, .prefilled = prefilled, .tokens = emitted, .cancelled = cancelled, .marked = marked }; + fn out(r: *Reply, sampling: ?lanes.Sampling) fx.Out { + return .{ .ctx = r, .prefilled = prefilled, .tokens = emitted, .cancelled = cancelled, .marked = marked, .sampling = sampling }; } }; +fn samplingFor(enabled: bool, index: usize) ?lanes.Sampling { + return if (enabled) .{ .seed = 1234 + index, .temperature = 0.7, .top_k = 40, .top_p = 0.9, .min_p = 0.05 } else null; +} + fn prompt(a: std.mem.Allocator, path: []const u8) ![]u32 { const f = try mtl.MappedFile.open(try std.fmt.allocPrintSentinel(a, "{s}", .{path}, 0)); defer f.deinit(); @@ -109,9 +113,18 @@ fn draftDepths(a: std.mem.Allocator, back: *Backend, prompts: []const []u32) !vo pub fn main(init: std.process.Init) !void { const a = init.arena.allocator(); const args = try init.minimal.args.toSlice(a); - if (args.len < 4) return error.ExpectedModelAndTwoPromptFiles; const pool = mtl.objc.Pool.push(); defer pool.pop(); + if (args.len == 2 and std.mem.eql(u8, args[1], "--sampling-only")) + return @import("flashnext_batch_sample.zig").check(init.gpa); + if (args.len < 4) return error.ExpectedModelAndTwoPromptFiles; + const sampled = if (std.c.getenv("FZ_SAMPLE")) |v| blk: { + const text = std.mem.span(v); + if (std.mem.eql(u8, text, "1")) break :blk true; + if (std.mem.eql(u8, text, "0")) break :blk false; + return error.BadSamplingFlag; + } else false; + std.debug.print("GATE sampled {any}: temperature0.7 top_k40 top_p0.9 min_p0.05 seeds1234+prompt_index\n", .{sampled}); const n: u32 = if (std.c.getenv("FZ_N")) |v| try std.fmt.parseInt(u32, std.mem.span(v), 10) else 32; if (n < 8) return error.NeedEightReplyTokens; const count = args.len - 2; @@ -125,17 +138,17 @@ pub fn main(init: std.process.Init) !void { var capacity: usize = 1024; var baseline_seconds: f64 = 0; var baseline_mtp_seconds: f64 = 0; - for (args[2..], prompts, refs) |path, *ids, *ref| { + for (args[2..], prompts, refs, 0..) |path, *ids, *ref, i| { ids.* = try prompt(a, path); capacity = @max(capacity, ids.len + n + fx.MARGIN); ref.* = .{ .a = a }; const start = mtl.clock.seconds(); - _ = try eng.generateFrom(ids.*, 0, &.{}, n, &.{}, 0, .{}, ref.out()); + _ = try eng.generateFrom(ids.*, 0, &.{}, n, &.{}, 0, .{}, ref.out(samplingFor(sampled, i))); baseline_seconds += mtl.clock.seconds() - start; if (ref.tokens.items.len != n) return error.ShortBaseline; var drafted: Reply = .{ .a = a }; const mtp_start = mtl.clock.seconds(); - _ = try eng.generateFrom(ids.*, 0, &.{}, n, &.{}, null, .{}, drafted.out()); + _ = try eng.generateFrom(ids.*, 0, &.{}, n, &.{}, null, .{}, drafted.out(samplingFor(sampled, i))); baseline_mtp_seconds += mtl.clock.seconds() - mtp_start; if (!std.mem.eql(u32, drafted.tokens.items, ref.tokens.items)) return error.LegacyMtpTokenMismatch; } @@ -168,6 +181,7 @@ pub fn main(init: std.process.Init) !void { .prompt = ids, .max_new = n, .drafts = drafts, + .sampling = samplingFor(sampled, i), }); defer for (streams) |*s| s.deinit(init.gpa); const start = mtl.clock.seconds(); @@ -189,11 +203,11 @@ pub fn main(init: std.process.Init) !void { std.debug.print("PASS drafts {any}: {d} streams, {d} exact tokens, {d} shared forwards, {d:.3}s, legacy serial {d:.3}s\n", .{ drafts, count, count * n, back.shared_count - before, seconds, baseline }); std.debug.print("MEASURE chain {any}: draft calls {d}, waits {d}, draft {d:.6}s, total {d:.6}s\n", .{ mode.chain, back.draft_calls - draft_before, back.draft_waits - waits_before, back.draft_seconds - draft_seconds_before, seconds }); // Admit a third prompt after a shared round, cancel its peer, and reuse the released slot. - var first = try lanes.Stream.init(init.gpa, .{ .id = "survivor", .prompt = prompts[0], .max_new = n, .drafts = drafts }); + var first = try lanes.Stream.init(init.gpa, .{ .id = "survivor", .prompt = prompts[0], .max_new = n, .drafts = drafts, .sampling = samplingFor(sampled, 0) }); defer first.deinit(init.gpa); - var cancelled = try lanes.Stream.init(init.gpa, .{ .id = "cancelled", .prompt = prompts[1], .max_new = n, .drafts = drafts }); + var cancelled = try lanes.Stream.init(init.gpa, .{ .id = "cancelled", .prompt = prompts[1], .max_new = n, .drafts = drafts, .sampling = samplingFor(sampled, 1) }); defer cancelled.deinit(init.gpa); - var admitted = try lanes.Stream.init(init.gpa, .{ .id = "admitted", .prompt = prompts[1], .max_new = n, .drafts = drafts }); + var admitted = try lanes.Stream.init(init.gpa, .{ .id = "admitted", .prompt = prompts[1], .max_new = n, .drafts = drafts, .sampling = samplingFor(sampled, 1) }); defer admitted.deinit(init.gpa); try core.addStream(&first); try core.addStream(&cancelled); @@ -212,6 +226,7 @@ pub fn main(init: std.process.Init) !void { .prompt = prompts[longest], .max_new = 1, .drafts = drafts, + .sampling = samplingFor(sampled, longest), .reuse = .{ .marks = &marks, .hook = .{ .ptr = &mark, .at = Mark.at } }, }); defer primer.deinit(init.gpa); @@ -219,13 +234,14 @@ pub fn main(init: std.process.Init) !void { if (mark.err) |err| return err; const saved = mark.saved orelse return error.NoSnapshot; defer Backend.snapDrop(&back, saved); - var peer = try lanes.Stream.init(init.gpa, .{ .id = "cache-peer", .prompt = prompts[0], .max_new = n, .drafts = drafts }); + var peer = try lanes.Stream.init(init.gpa, .{ .id = "cache-peer", .prompt = prompts[0], .max_new = n, .drafts = drafts, .sampling = samplingFor(sampled, 0) }); defer peer.deinit(init.gpa); var resumed = try lanes.Stream.init(init.gpa, .{ .id = "cache-resumed", .prompt = prompts[longest], .max_new = n, .drafts = drafts, + .sampling = samplingFor(sampled, longest), .reuse = .{ .saved = saved, .at = marks[0] }, }); defer resumed.deinit(init.gpa); diff --git a/zig/tests/flashnext_batch_sample.zig b/zig/tests/flashnext_batch_sample.zig new file mode 100644 index 0000000000..fb2f7ee987 --- /dev/null +++ b/zig/tests/flashnext_batch_sample.zig @@ -0,0 +1,70 @@ +//! A device gate for sampled windows with distinct seeds and positions, with untouched greedy and spare rows. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const fz = tf.flashnext_replay; +const sample = tf.flashnext_batch_sample; + +pub fn check(a: std.mem.Allocator) !void { + const dev = try mtl.Device.init(); + defer dev.deinit(); + const lib = try mtl.Library.fromSource(dev, @import("kernel_sources").nemotron_sample, mtl.CompileOptions.mlx()); + defer lib.deinit(); + const pipe = try mtl.Pipeline.init(dev, lib, "tf_sample_full", false); + defer pipe.deinit(); + const q = try dev.queue(); + defer q.deinit(); + const logits = try dev.buffer(fz.MAXR * fz.VOCAB * 2, fz.opts); + defer logits.deinit(); + const positions = try dev.buffer(fz.MAXR * 4, fz.opts); + defer positions.deinit(); + const picks = try dev.buffer(fz.MAXR * 4, fz.opts); + defer picks.deinit(); + const plan = try tf.flashnext_batch_meta.Plan.init(&.{ + .{ .slot = 3, .pos = 19, .rows = 3 }, + .{ .slot = 0, .pos = 2047, .rows = 2 }, + .{ .slot = 5, .pos = 9000, .rows = 1 }, + }, 10000); + const expected_positions = [_]u32{ 20, 21, 22, 2048, 2049, 9001 }; + const settings = [_]?tf.lanes.Sampling{ + .{ .seed = 1, .temperature = 0.7, .top_k = 20, .top_p = 0.95 }, + null, + .{ .seed = 1 << 63 | 4, .temperature = 1.3, .top_k = 0, .top_p = 0.8, .min_p = 0.05 }, + }; + var rng = std.Random.DefaultPrng.init(11); + const random = rng.random(); + for (logits.slice(u16, fz.MAXR * fz.VOCAB)) |*word| { + const x: f32 = random.floatNorm(f32) * 2; + const bits: u32 = @bitCast(x); + word.* = @truncate((bits + 0x7FFF + ((bits >> 16) & 1)) >> 16); + } + const sentinel = 0xDEAD_BEEF; + @memset(picks.slice(u32, fz.MAXR), sentinel); + @memset(positions.slice(u32, fz.MAXR), sentinel); + @memcpy(positions.slice(u32, fz.MAXR)[0..plan.total], plan.keys[0..plan.total]); + const cb = q.commandBuffer(); + const enc = cb.compute(.serial); + for (settings, 0..) |setting, i| if (setting) |p| { + const start = plan.starts[i]; + const end = if (i + 1 < settings.len) plan.starts[i + 1] else plan.total; + sample.encode(enc, pipe, .{ .b = logits }, .{ .b = positions }, .{ .b = picks }, start, end - start, p); + }; + enc.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.SampleCommandFailed; + const row = try a.alloc(f32, fz.VOCAB); + defer a.free(row); + for (0..fz.MAXR) |r| { + const setting: ?tf.lanes.Sampling = if (r < 3) settings[0] else if (r == 5) settings[2] else null; + const expected = if (setting) |p| blk: { + for (row, logits.slice(u16, fz.MAXR * fz.VOCAB)[r * fz.VOCAB ..][0..fz.VOCAB]) |*x, w| + x.* = @bitCast(@as(u32, w) << 16); + break :blk try tf.lanes.gpu_full.sample(a, row, p, expected_positions[r], null); + } else sentinel; + if (picks.slice(u32, fz.MAXR)[r] != expected) return error.PackedSampleMismatch; + if (positions.slice(u32, fz.MAXR)[r] != (if (r < plan.total) expected_positions[r] else sentinel)) + return error.DrawPositionsOverwritten; + } + std.debug.print("PASS GPU sampling: two seeds, unequal positions, packed windows and untouched greedy/spare rows\n", .{}); +} From d0f613857e3affed6b0b8e993ea59a787776de1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:58:47 +0200 Subject: [PATCH 29/31] Avoid a second warmup when auto falls back to one Flash Next session --- zig/src/native/flashnext_batch_host.zig | 2 +- zig/src/native/flashnext_host.zig | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/zig/src/native/flashnext_batch_host.zig b/zig/src/native/flashnext_batch_host.zig index a04e7f0ab5..f438b4002e 100644 --- a/zig/src/native/flashnext_batch_host.zig +++ b/zig/src/native/flashnext_batch_host.zig @@ -195,6 +195,6 @@ fn dropResident(h: *Host) void { /// Auto with room for only one session transfers the loaded engine, preserving the chosen footprint limit. fn single(gpa: std.mem.Allocator, io: std.Io, eng: *fx.Engine, window: i64, cache_gib: ?f64, over: bool, cap: u64, a: std.mem.Allocator, why: *[]const u8) !api.Opened { const host = @import("flashnext_host.zig"); - const h = try host.openLoaded(gpa, io, eng, window, cache_gib, over, null, cap, a, why); + const h = try host.openWarmed(gpa, io, eng, window, cache_gib, over, null, cap, a, why); return .{ .engine = h.engine(), .close = host.close, .ctx = h }; } diff --git a/zig/src/native/flashnext_host.zig b/zig/src/native/flashnext_host.zig index 4f9491123c..d62e568c2b 100644 --- a/zig/src/native/flashnext_host.zig +++ b/zig/src/native/flashnext_host.zig @@ -423,11 +423,6 @@ fn learnedStates(gpa: Allocator, io: std.Io, eng: *fx.Engine, root: []const u8, pub fn open(gpa: Allocator, io: std.Io, dir: []const u8, dump: ?[]const u8, window: i64, speed_up: ?[]const u8, cache_gib: ?f64, over_cap: bool, learn: ?Learn, a: Allocator, why: *[]const u8) !*Host { const eng = try fx.Engine.loadWith(gpa, io, dir, dump, speed_up); errdefer eng.deinit(); - return openLoaded(gpa, io, eng, window, cache_gib, over_cap, learn, null, a, why); -} - -/// Opens a loaded engine without loading its weights again; the caller retains ownership on error. -pub fn openLoaded(gpa: Allocator, io: std.Io, eng: *fx.Engine, window: i64, cache_gib: ?f64, over_cap: bool, learn: ?Learn, limit: ?u64, a: Allocator, why: *[]const u8) !*Host { if (learn != null and eng.r.tp != null) { why.* = "--learn is not kept across a Flash Next pair yet: leave it out, or serve on one Mac"; return error.LearnOnPair; @@ -436,6 +431,11 @@ pub fn openLoaded(gpa: Allocator, io: std.Io, eng: *fx.Engine, window: i64, cach std.log.err("flash next: warm-up failed: {s}", .{@errorName(err)}); return err; }; + return openWarmed(gpa, io, eng, window, cache_gib, over_cap, learn, null, a, why); +} + +/// Takes an already warmed engine without loading or warming again; the caller retains ownership on error. +pub fn openWarmed(gpa: Allocator, io: std.Io, eng: *fx.Engine, window: i64, cache_gib: ?f64, over_cap: bool, learn: ?Learn, limit: ?u64, a: Allocator, why: *[]const u8) !*Host { const cache_plan = try cacheBudget(eng, cache_gib, over_cap, limit, a, why); const budget = cache_plan.budget_bytes; eng.peer_budget = budget; // rank 1: its kept states and free buffers stay inside the same budget From ef85c8b0b8402fc5186a8123fcf2a4ef87042acd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:56:02 +0200 Subject: [PATCH 30/31] Honor validated model load limits when keeping buffers resident --- docs/recipes/qwen3.8-flash-next.md | 6 +- zig/src/families/glm/residency.zig | 3 +- zig/src/metal/residency.zig | 28 ++++++++- zig/src/native/flashnext_batch_host.zig | 2 +- zig/src/native/flashnext_host.zig | 2 +- zig/src/native/resident.zig | 84 ++++++++++++++++++++++--- 6 files changed, 108 insertions(+), 17 deletions(-) diff --git a/docs/recipes/qwen3.8-flash-next.md b/docs/recipes/qwen3.8-flash-next.md index 46defb8b9e..cfa1da035f 100644 --- a/docs/recipes/qwen3.8-flash-next.md +++ b/docs/recipes/qwen3.8-flash-next.md @@ -100,8 +100,10 @@ Each packed window uses its own seed, sampling settings and absolute token posit greedy only. The model gate below must verify sampled token parity across routes, drafting and batch composition. The shared host's residency set includes the engine, session caches and metadata, shared hidden rows and later -prompt snapshots. Residency stays within 70% of RAM even when the load limit is raised; a refused set leaves the -server running without it. `FLASHNEXT_NO_RESIDENCY=1` disables the set. With a prompt cache, background warm turns +prompt snapshots. Residency uses the same validated load limit as the host, with a 2 GiB margin inside it for +growing buffers. The default stays at 70% of RAM; `FLASHNEXT_LOAD_LIMIT_GB` can raise or lower that ceiling, capped +by RAM and the GPU working set. A refused set leaves the server running without it, and `--prompt-cache-over-cap` +does not raise the residency ceiling. `FLASHNEXT_NO_RESIDENCY=1` disables the set. With a prompt cache, warm turns keep the end of a reply as the single-request host does. Idle latency and throughput need fresh measurements. The checkpoint gate compares shared replies with fresh legacy replies, with drafting off and on, then checks diff --git a/zig/src/families/glm/residency.zig b/zig/src/families/glm/residency.zig index 8ebd628639..e5840af691 100644 --- a/zig/src/families/glm/residency.zig +++ b/zig/src/families/glm/residency.zig @@ -6,7 +6,8 @@ const Engine = @import("engine.zig").Engine; /// The set when GLM_RESIDENCY asks for one and its bytes fit the bound, then the idle keepalive's target either way. pub fn wire(e: *Engine) void { // opt-in: wiring 181 GB leaves macOS nothing to reclaim if another model shares the Mac (Flash Next runs without) - const want = std.c.getenv("GLM_RESIDENCY") != null and mtl.ResidencySet.within("glm", 0, bytes(e)); + const cap: ?u64 = if (e.limit.asked) e.limit.bytes else null; + const want = std.c.getenv("GLM_RESIDENCY") != null and mtl.ResidencySet.withinUnder("glm", 0, bytes(e), cap); if (!want) {} else if (e.device.residencySet(e.w.buffers.items.len + e.arena.buffers.items.len + if (e.vision) |v| v.buffers().len else 0)) |set| { for (e.w.buffers.items) |b| set.add(b); for (e.arena.buffers.items) |b| set.add(b); diff --git a/zig/src/metal/residency.zig b/zig/src/metal/residency.zig index 2347c4a6df..726e975de6 100644 --- a/zig/src/metal/residency.zig +++ b/zig/src/metal/residency.zig @@ -15,8 +15,14 @@ pub const ResidencySet = struct { /// Whether a set holding `held` bytes may take `added` more, the margin inside the share of `ram` (null: never). pub fn fits(held: u64, added: u64, ram: ?u64) bool { + return fitsUnder(held, added, ram, null); + } + + /// As fits, under a validated load limit when supplied; the margin stays inside it and RAM must be known. + pub fn fitsUnder(held: u64, added: u64, ram: ?u64, limit: ?u64) bool { const total = ram orelse return false; - return held +| added +| margin <= total / 100 * share_percent; + const cap = if (limit) |n| @min(n, total) else total / 100 * share_percent; + return held <= cap and added <= cap - held and margin <= cap - held - added; } /// Why `fits` said no, logged for `name`: RAM that could not be read, or the bytes past the share. @@ -36,12 +42,28 @@ pub const ResidencySet = struct { /// `fits` on this Mac's RAM, read now; a refusal is logged for `name`. pub fn within(name: []const u8, held: u64, added: u64) bool { + return withinUnder(name, held, added, null); + } + + /// As within, using the family's already validated load limit rather than parsing its environment again. + pub fn withinUnder(name: []const u8, held: u64, added: u64, limit: ?u64) bool { const ram = std.process.totalSystemMemory() catch null; - if (fits(held, added, ram)) return true; - refused(name, held, added, ram); + if (fitsUnder(held, added, ram, limit)) return true; + refusedUnder(name, held, added, ram, limit); return false; } + /// A refusal against the selected bound; the default keeps its existing RAM-share diagnostic. + pub fn refusedUnder(name: []const u8, held: u64, added: u64, ram: ?u64, limit: ?u64) void { + const cap = if (limit) |n| @min(n, ram orelse return refused(name, held, added, ram)) else return refused(name, held, added, ram); + const gib: f64 = 1 << 30; + if (held == 0) { + std.log.warn("{s}: no residency set: its {d:.1} GiB and a {d} GiB margin pass the {d:.1} GiB load limit", .{ name, @as(f64, @floatFromInt(added)) / gib, margin >> 30, @as(f64, @floatFromInt(cap)) / gib }); + } else { + std.log.info("{s}: {d:.1} GiB more would take its residency set past the {d:.1} GiB load limit: it stays out", .{ name, @as(f64, @floatFromInt(added)) / gib, @as(f64, @floatFromInt(cap)) / gib }); + } + } + /// Where buffers a model makes or frees after its set was built report, so the set follows them. pub const Follow = struct { ctx: *anyopaque, diff --git a/zig/src/native/flashnext_batch_host.zig b/zig/src/native/flashnext_batch_host.zig index f438b4002e..cb268a966d 100644 --- a/zig/src/native/flashnext_batch_host.zig +++ b/zig/src/native/flashnext_batch_host.zig @@ -175,7 +175,7 @@ fn holdResident(h: *Host) !void { try buffers.appendSlice(h.gpa, h.eng.r.held.items); for (h.back.sessions) |s| try buffers.appendSlice(h.gpa, s.owned.items); try buffers.append(h.gpa, h.back.hidden.b); - h.resident = try Resident.hold(h.gpa, h.host.io, h.eng.r.device, h.eng.r.queue, buffers.items, "flash next shared") orelse return; + h.resident = try Resident.holdUnder(h.gpa, h.host.io, h.eng.r.device, h.eng.r.queue, buffers.items, "flash next shared", h.load_limit) orelse return; const r = &h.resident.?; h.warm = r.target(); h.eng.r.follow = r.follow(); diff --git a/zig/src/native/flashnext_host.zig b/zig/src/native/flashnext_host.zig index d62e568c2b..7f0d436a7d 100644 --- a/zig/src/native/flashnext_host.zig +++ b/zig/src/native/flashnext_host.zig @@ -472,7 +472,7 @@ pub fn openWarmed(gpa: Allocator, io: std.Io, eng: *fx.Engine, window: i64, cach /// The engine's buffers in one residency set the idle keepalive's ticks use; buffers made later join it or stay out. fn holdResident(h: *Host) !void { - h.resident = try Resident.hold(h.gpa, h.io, h.eng.r.device, h.eng.r.queue, h.eng.r.held.items, "flash next") orelse return; + h.resident = try Resident.holdUnder(h.gpa, h.io, h.eng.r.device, h.eng.r.queue, h.eng.r.held.items, "flash next", if (h.load_limit > 0) h.load_limit else null) orelse return; const r = &h.resident.?; h.warm = r.target(); h.eng.r.follow = r.follow(); diff --git a/zig/src/native/resident.zig b/zig/src/native/resident.zig index ce9dc6396f..e25f4b0c41 100644 --- a/zig/src/native/resident.zig +++ b/zig/src/native/resident.zig @@ -1,4 +1,4 @@ -//! A served model's buffers in one residency set, kept wired by the server's idle keepalive, inside the 70% limit. +//! A served model's buffers in one residency set, kept wired by the server's idle keepalive, inside its load limit. const std = @import("std"); const mtl = @import("metal"); const cache_fit = @import("cache_fit.zig"); @@ -11,11 +11,16 @@ pub const Resident = struct { /// One set of `buffers` that `work` holds, requested resident at once; null (logged for `name`) past the 70% limit. pub fn hold(gpa: std.mem.Allocator, io: std.Io, device: mtl.Device, work: mtl.Queue, buffers: []const mtl.Buffer, name: []const u8) !?Resident { + return holdUnder(gpa, io, device, work, buffers, name, null); + } + + /// As hold, under a family's validated load limit when supplied, keeping the margin inside it. + pub fn holdUnder(gpa: std.mem.Allocator, io: std.Io, device: mtl.Device, work: mtl.Queue, buffers: []const mtl.Buffer, name: []const u8, limit: ?u64) !?Resident { const set = try device.residencySet(buffers.len + 1); errdefer set.deinit(); const scratch = try device.buffer(64, mtl.ResourceOptions.shared); errdefer scratch.deinit(); - var held = try Held(mtl.ResidencySet, mtl.Buffer).init(gpa, io, set, buffers, scratch, name, cache_fit.ram()) orelse { + var held = try Held(mtl.ResidencySet, mtl.Buffer).initUnder(gpa, io, set, buffers, scratch, name, cache_fit.ram(), limit) orelse { scratch.deinit(); set.deinit(); return null; @@ -23,7 +28,9 @@ pub const Resident = struct { errdefer held.deinit(); set.requestResidency(); work.addResidencySet(set); - std.log.info("{s}: {d:.1} GiB kept wired between requests, inside 70% of {d:.0} GiB of RAM", .{ name, cache_fit.gibs(held.bytes), cache_fit.gibs(held.ram.?) }); + if (limit) |n| { + std.log.info("{s}: {d:.1} GiB kept wired between requests, inside the {d:.1} GiB load limit", .{ name, cache_fit.gibs(held.bytes), cache_fit.gibs(@min(n, held.ram.?)) }); + } else std.log.info("{s}: {d:.1} GiB kept wired between requests, inside 70% of {d:.0} GiB of RAM", .{ name, cache_fit.gibs(held.bytes), cache_fit.gibs(held.ram.?) }); return .{ .held = held, .work = work, .scratch = scratch }; } @@ -75,7 +82,7 @@ pub const Resident = struct { } }; -/// What a residency set holds and what waits to join it, inside the 70% limit (`Set` is a counting fake in tests). +/// What a residency set holds and what waits to join it, inside its load limit (`Set` is a counting fake in tests). pub fn Held(comptime Set: type, comptime Buf: type) type { return struct { const Self = @This(); @@ -86,6 +93,7 @@ pub fn Held(comptime Set: type, comptime Buf: type) type { set: Set, name: []const u8, ram: ?u64, + limit: ?u64 = null, bytes: u64 = 0, // the set's buffers and those waiting to join it: what the bound counts members: std.AutoHashMapUnmanaged(usize, Member) = .empty, joining: std.ArrayList(Buf) = .empty, // added and committed at the next flush, never during a request @@ -93,12 +101,17 @@ pub fn Held(comptime Set: type, comptime Buf: type) type { /// `buffers` and `extra` in `set`, committed, if together they fit; else null and nothing is added. pub fn init(gpa: std.mem.Allocator, io: std.Io, set: Set, buffers: []const Buf, extra: Buf, name: []const u8, ram: ?u64) !?Self { + return initUnder(gpa, io, set, buffers, extra, name, ram, null); + } + + /// A family-selected ceiling applies at creation and to every later join. + pub fn initUnder(gpa: std.mem.Allocator, io: std.Io, set: Set, buffers: []const Buf, extra: Buf, name: []const u8, ram: ?u64, limit: ?u64) !?Self { const bytes = sum(buffers) + extra.length(); - if (!cache_fit.residentFits(0, bytes, ram)) { - mtl.ResidencySet.refused(name, 0, bytes, ram); + if (!mtl.ResidencySet.fitsUnder(0, bytes, ram, limit)) { + mtl.ResidencySet.refusedUnder(name, 0, bytes, ram, limit); return null; } - var h: Self = .{ .gpa = gpa, .io = io, .set = set, .name = name, .ram = ram, .bytes = bytes }; + var h: Self = .{ .gpa = gpa, .io = io, .set = set, .name = name, .ram = ram, .limit = limit, .bytes = bytes }; errdefer h.members.deinit(gpa); try h.members.ensureTotalCapacity(gpa, @intCast(buffers.len + 1)); for (buffers) |b| h.place(b); @@ -123,8 +136,8 @@ pub fn Held(comptime Set: type, comptime Buf: type) type { more += b.length(); }; if (more == 0) return true; - if (!cache_fit.residentFits(h.bytes, more, h.ram)) { - mtl.ResidencySet.refused(h.name, h.bytes, more, h.ram); + if (!mtl.ResidencySet.fitsUnder(h.bytes, more, h.ram, h.limit)) { + mtl.ResidencySet.refusedUnder(h.name, h.bytes, more, h.ram, h.limit); return false; } h.members.ensureUnusedCapacity(h.gpa, @intCast(buffers.len)) catch return false; @@ -283,3 +296,56 @@ test "buffers made later wait for a flush inside the limit, and leave at once be if (h.join(&.{slot})) h.flush(); // Resident.addNow: a slot's commit takes what waits with it try std.testing.expectEqual(Fake{ .adds = 2, .commits = 1 }, f); } + +test "an operator load ceiling holds the shared host and stays in force for later buffers" { + const GiB = cache_fit.GiB; + const ids: [4]u8 = @splat(0); + const buffers = [_]Fake.Buffer{.{ .id = &ids[0], .len = 194 * GiB }}; + const scratch: Fake.Buffer = .{ .id = &ids[1], .len = 64 }; + const state: Fake.Buffer = .{ .id = &ids[2], .len = 3 * GiB }; + const more: Fake.Buffer = .{ .id = &ids[3], .len = GiB }; + const cap: u64 = 214_700_000_000; + const H = Held(FakeSet, Fake.Buffer); + var f: Fake = .{}; + try std.testing.expect((try H.init(std.testing.allocator, std.testing.io, .{ .f = &f }, &buffers, scratch, "test", 256 * GiB)) == null); + try std.testing.expect((try H.initUnder(std.testing.allocator, std.testing.io, .{ .f = &f }, &buffers, scratch, "test", null, cap)) == null); + try std.testing.expectEqual(Fake{}, f); + const opened = try H.initUnder(std.testing.allocator, std.testing.io, .{ .f = &f }, &buffers, scratch, "test", 256 * GiB, cap); + try std.testing.expect(opened != null); + var h = opened.?; + defer h.deinit(); + try std.testing.expectEqual(Fake{ .adds = 2, .commits = 1 }, f); + try std.testing.expectEqual(cap, h.limit.?); + f = .{}; + try std.testing.expect(h.join(&.{state})); + try std.testing.expect(!h.join(&.{more})); + try std.testing.expectEqual(197 * GiB + 64, h.bytes); + try std.testing.expectEqual(Fake{}, f); + h.flush(); + try std.testing.expectEqual(Fake{ .adds = 1, .commits = 1 }, f); + h.leave(state); + try std.testing.expectEqual(Fake{ .adds = 1, .removes = 1, .commits = 2 }, f); + try std.testing.expect(h.join(&.{more})); + try std.testing.expectEqual(195 * GiB + 64, h.bytes); +} + +test "an explicit residency ceiling keeps the margin, RAM backstop and overflow checks" { + const gib: u64 = 1 << 30; + const ram = 256 * gib; + const cap: u64 = 214_700_000_000; + for ([_]u64{ 1787, 1798, 1826, 1940 }) |tenths| { + const bytes = tenths * gib / 10; + try std.testing.expect(!mtl.ResidencySet.fits(0, bytes, ram)); + try std.testing.expect(mtl.ResidencySet.fitsUnder(0, bytes, ram, cap)); + } + try std.testing.expect(mtl.ResidencySet.fitsUnder(0, cap - mtl.ResidencySet.margin, ram, cap)); + try std.testing.expect(!mtl.ResidencySet.fitsUnder(0, cap - mtl.ResidencySet.margin + 1, ram, cap)); + try std.testing.expect(!mtl.ResidencySet.fitsUnder(0, 1, null, cap)); + try std.testing.expect(!mtl.ResidencySet.fitsUnder(0, 1, ram, 0)); + try std.testing.expect(mtl.ResidencySet.fitsUnder(0, 8 * gib, ram, 10 * gib)); + try std.testing.expect(!mtl.ResidencySet.fitsUnder(0, 8 * gib + 1, ram, 10 * gib)); + try std.testing.expect(mtl.ResidencySet.fitsUnder(0, 254 * gib, ram, 512 * gib)); + try std.testing.expect(!mtl.ResidencySet.fitsUnder(0, 254 * gib + 1, ram, 512 * gib)); + const most = std.math.maxInt(u64); + try std.testing.expect(!mtl.ResidencySet.fitsUnder(most - 16, 32, most, most)); +} From 5164380ed4e1dc5fa31c67f88e10892c6986afda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:19:56 +0200 Subject: [PATCH 31/31] Refuse --parallel above one with --learn as with --speed-up; restore the memory limit remedies An explicit --parallel N above one with --learn served one reply at a time without a word, while the same request with --speed-up refused. Both refuse now, from one tested predicate. The OverMemoryLimit message names its remedies again. Co-Authored-By: Claude Opus 5.5 --- docs/recipes/qwen3.8-flash-next.md | 2 +- zig/src/families/flashnext/batch_meta.zig | 8 ++++++++ zig/src/families/flashnext/batch_meta_test.zig | 9 +++++++++ zig/src/native/metal.zig | 6 +++--- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/recipes/qwen3.8-flash-next.md b/docs/recipes/qwen3.8-flash-next.md index cfa1da035f..9673f43102 100644 --- a/docs/recipes/qwen3.8-flash-next.md +++ b/docs/recipes/qwen3.8-flash-next.md @@ -85,7 +85,7 @@ up to 16 target rows, including draft rows, into one layer loop. Projections and together; recurrence, convolution and attention read each request's own state. Every stream's MTP drafts share one command buffer; `FZ_BATCH_MTP_SERIAL=1` restores serial drafting. New requests join between rounds, and cancelling a request releases its slot for another request. -`auto` chooses up to 16 sessions from the memory left after loading the model and reserving the prompt cache. If fewer than two fit, it opens the single-reply driver instead. Explicit `--parallel 1`, `--learn`, and the two-Mac path retain their supported driver. Shared sessions run on one Mac; `--parallel N` above one with `--speed-up` is refused. Use `--parallel 1` for the two-Mac link. +`auto` chooses up to 16 sessions from the memory left after loading the model and reserving the prompt cache. If fewer than two fit, it opens the single-reply driver instead. Explicit `--parallel 1`, `--learn`, and the two-Mac path retain their supported driver. Shared sessions run on one Mac; `--parallel N` above one with `--speed-up` or `--learn` is refused. Use `--parallel 1` for the two-Mac link. The ceiling is 70% of RAM, or `FLASHNEXT_LOAD_LIMIT_GB` (GB, up to the GPU's recommended working set). It covers the server's whole footprint: the model, the sessions and the prompt cache. Left out, `--prompt-cache-gib` takes what the diff --git a/zig/src/families/flashnext/batch_meta.zig b/zig/src/families/flashnext/batch_meta.zig index 576fc0a77d..91373f8b03 100644 --- a/zig/src/families/flashnext/batch_meta.zig +++ b/zig/src/families/flashnext/batch_meta.zig @@ -62,3 +62,11 @@ pub fn fit(want: u32, fixed: bool, room: u64, per_slot: u64) !u32 { pub fn shared(fixed: bool, streams: u32, paired: bool, learning: bool) bool { return (!fixed or streams > 1) and !paired and !learning; } + +/// Why an explicit --parallel above one cannot open; null when it can (auto falls back to the single-reply host). +pub fn refusal(fixed: bool, streams: u32, paired: bool, learning: bool) ?[]const u8 { + if (!fixed or streams <= 1) return null; + if (paired) return "Flash Next shared sessions are single-Mac only; use --parallel 1 with --speed-up"; + if (learning) return "Flash Next shared sessions do not keep --learn yet; use --parallel 1 with --learn, or leave --learn out"; + return null; +} diff --git a/zig/src/families/flashnext/batch_meta_test.zig b/zig/src/families/flashnext/batch_meta_test.zig index f1f92ac184..a1860d54ff 100644 --- a/zig/src/families/flashnext/batch_meta_test.zig +++ b/zig/src/families/flashnext/batch_meta_test.zig @@ -74,3 +74,12 @@ test "default auto selects shared sessions while explicit one, learning and pair try std.testing.expectEqual(@as(u32, 3), try meta.fit(16, false, 2048, 1024)); try std.testing.expectError(error.OverMemoryLimit, meta.fit(4, true, 2048, 1024)); } + +test "an explicit --parallel above one refuses pairs and learning alike, auto and one never refuse" { + try std.testing.expect(meta.refusal(false, 16, true, true) == null); + try std.testing.expect(meta.refusal(true, 1, true, true) == null); + try std.testing.expect(meta.refusal(true, 4, false, false) == null); + try std.testing.expect(std.mem.indexOf(u8, meta.refusal(true, 4, true, false).?, "--speed-up") != null); + try std.testing.expect(std.mem.indexOf(u8, meta.refusal(true, 4, true, true).?, "--speed-up") != null); + try std.testing.expect(std.mem.indexOf(u8, meta.refusal(true, 4, false, true).?, "--learn") != null); +} diff --git a/zig/src/native/metal.zig b/zig/src/native/metal.zig index 8d527814ee..0ad2a3d2dd 100644 --- a/zig/src/native/metal.zig +++ b/zig/src/native/metal.zig @@ -386,15 +386,15 @@ fn openFlashNext(a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: return null; } const paired = o.speed_up != null or std.c.getenv("TF_FLASHNEXT_TP") != null; - if (o.lanes_fixed and o.lanes > 1 and paired) { - problem.* = "Flash Next shared sessions are single-Mac only; use --parallel 1 with --speed-up"; + if (tf.flashnext_batch_meta.refusal(o.lanes_fixed, o.lanes, paired, o.learn != null)) |text| { + problem.* = text; return null; } if (tf.flashnext_batch_meta.shared(o.lanes_fixed, o.lanes, paired, o.learn != null)) { var why: []const u8 = ""; const shared: ?api.Opened = shared: { const opened = @import("flashnext_batch_host.zig").open(gpa, io, o.dir, dump, if (window > 0) @min(window, tf.flashnext_replay.CAP - tf.flashnext_engine.MARGIN) else tf.flashnext_replay.CAP - tf.flashnext_engine.MARGIN, if (o.lanes_fixed) o.lanes else tf.flashnext_replay.MAXR, o.lanes_fixed, o.prompt_cache_gib, o.prompt_cache_over_cap, a, &why) catch |err| { - problem.* = tf.flashnext_engine.quantizationProblem(err) orelse if (err == error.CacheOverCap) why else if (err == error.BadLoadLimit) "FLASHNEXT_LOAD_LIMIT_GB is not a load limit: give a number of GB above 0, or unset it for 70% of RAM" else if (err == error.OverMemoryLimit) "the native Flash Next batching engine cannot open (OverMemoryLimit): the sessions and prompt cache do not fit under the load limit" else try std.fmt.allocPrint(a, "the native Flash Next batching engine cannot open ({s})", .{@errorName(err)}); + problem.* = tf.flashnext_engine.quantizationProblem(err) orelse if (err == error.CacheOverCap) why else if (err == error.BadLoadLimit) "FLASHNEXT_LOAD_LIMIT_GB is not a load limit: give a number of GB above 0, or unset it for 70% of RAM" else if (err == error.OverMemoryLimit) "the native Flash Next batching engine cannot open (OverMemoryLimit): the sessions and the given prompt cache do not fit under the load limit (70% of RAM, or FLASHNEXT_LOAD_LIMIT_GB) past the loaded model; fewer streams, a smaller --context or --prompt-cache-gib, or a higher FLASHNEXT_LOAD_LIMIT_GB" else try std.fmt.allocPrint(a, "the native Flash Next batching engine cannot open ({s})", .{@errorName(err)}); return null; }; break :shared opened;