From e0187a04f64e43872574d1910c61d55fa7e38a68 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sat, 14 Nov 2026 23:26:09 -0600 Subject: [PATCH 1/2] Shard DeepSeek V4.1 prefill hyperconnections by rank Full-row hyperconnections repeat the same prefill work on every tensor-parallel rank. Reduce-scatter their branch outputs into contiguous rank-owned rows, normalize there, and gather the branch inputs needed by full-row consumers. Engram layers and decoder boundaries retain complete rows and continuation state. Add the explicit preparation option, fixed-shape shared collective components, row-transition scheduling, and preparation-only rebuild metadata. Regenerate the Python bindings from the combined schema while retaining current main's fields. The intended benefit is less repeated hyperconnection work without changing the serving operation families or the disabled-option behavior. Three focused CLI/planning/metadata round-trip tests pass. The broader Windows CLI selection has 205 passes and 41 failures; the same 41 failures reproduce on unmodified main in this environment, which lacks Heliopause and assumes different configuration paths. Syntax and staged whitespace checks pass. Current-default CUDA preparation and native serving have not been run on this head. Design-Signed-Off-By: Codex (GPT-6); design-review=ed22 Signed-off-by: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> --- docs/TODO.txt | 9 + docs/cli-design.md | 1 + helios/proto/runtime_core.proto | 2 + ignition/cli/__init__.py | 4 + ignition/cli/bench.py | 1 + ignition/cli/preparation.py | 3 +- ignition/cli/serve.py | 7 +- ignition/components/collectives.py | 233 ++++++++++++ ignition/planning.py | 27 +- ignition/preparation/rebuild_manifest.py | 2 + ignition/preparation/runtime_core_pb2.py | 358 +++++++++--------- ignition/recipes/deepseek41/bootstrap.py | 11 +- ignition/recipes/deepseek41/decoder.py | 254 +++++++++++-- ignition/recipes/deepseek41/recipe.py | 2 +- tests/cli/test_commands.py | 7 + tests/cli/test_planning.py | 42 ++ .../components/test_row_shard_collectives.py | 176 +++++++++ .../recipes/test_deepseek41_expert_overlap.py | 4 + tests/recipes/test_deepseek41_row_shard.py | 271 +++++++++++++ .../test_deepseek41_row_shard_release.py | 35 ++ .../recipes/test_deepseek41_shard_schedule.py | 84 ++++ 21 files changed, 1322 insertions(+), 211 deletions(-) create mode 100644 tests/components/test_row_shard_collectives.py create mode 100644 tests/recipes/test_deepseek41_row_shard.py create mode 100644 tests/recipes/test_deepseek41_row_shard_release.py create mode 100644 tests/recipes/test_deepseek41_shard_schedule.py diff --git a/docs/TODO.txt b/docs/TODO.txt index 9f890523b..f8b00a219 100644 --- a/docs/TODO.txt +++ b/docs/TODO.txt @@ -479,3 +479,12 @@ [ ] Publish Debian packages with releases and update the APT repository from them. release.yml builds no .deb, and no tracked tool builds or signs an APT index. See debian.md and release.md. + +[ ] Qualify row-sharded prefill at three ranks. The DeepSeek V4.1 recipe's + 4,096-row chunk has no per-rank block at three ranks, though a 6,144-row + chunk would. Close when an export and native serve with --row-shard-prefill + complete at three ranks. + +[ ] Qualify row-sharded prefill on SM103, where its projections and experts run + a different kernel path from SM121. Close when an export and native serve + with --row-shard-prefill complete on SM103. diff --git a/docs/cli-design.md b/docs/cli-design.md index ac6e9ee83..2070b7304 100644 --- a/docs/cli-design.md +++ b/docs/cli-design.md @@ -461,6 +461,7 @@ explicit flags take precedence. | `--autotune-effort off\|default\|audit` | Kernel search effort. Off uses heuristics without races; default races the candidates that pass each kernel family's performance exclusions; audit lifts those exclusions and races every legal candidate, to check that the default search prunes no winners. Overrides `[preparation] autotune_effort`. | Default | | `--joint-tuning` | Jointly select DeepSeek V4.1 QKV projection, query normalization, and query expansion plans by timing complete prepared layers across every prefill, decode, packed verification, and draft shape. Requires MXFP8 query normalization and autotuning enabled. Command line only. | off | | `--shard-hyperconnections` | Partition Qwen3.8 Flash target and MTP hyperconnection bottlenecks across TP ranks and all-reduce BF16 gate logits before sigmoid. Requires TP2 or TP4. Command line only; changing it requires re-export. | off (replicated) | +| `--row-shard-prefill` | Replace each DeepSeek V4.1 prefill tensor-parallel all-reduce with a reduce-scatter, run the hyperconnection on the rank's contiguous row shard, and gather the normalized branch input back to every rank. The collective pair preserves the pipelined two-shot reduction's arithmetic; whole-decoder numerical equivalence remains unqualified. Preparation requires the runtime's ABI 11 or later row-shard surfaces; the proxy pinned in this tree records ABI 14. Requires TP2 or more; three ranks and SM103 are refused, and a layer whose row count or pipe capacity does not fit keeps the all-reduce path. Command line only; changing it requires re-export. | off | | `--collective-ladder PATH` | JSON file of final joint collective geometries and runtime profiles. Required when the resolved collective tuning policy is `runtime`; rejected with `static`. See [offline ladders](../ignition/preparation/ladders/README.md). | None | | `--collective-tuning static\|runtime` | Static autotunes collectives during preparation; runtime uses the required ladder file for final compiled geometry and captures runtime submode/active-CTA selection for eligible BF16 PCIe reductions. Overrides `[preparation] collective_tuning`, then the recipe declaration. | Static | | `--lock-gpu-clocks` | Lock each tuning GPU at its maximum supported graphics frequency during b12x and Inductor autotuning, then reset driver clock defaults. Frozen warmup, capture, and serving remain unlocked. Requires root or noninteractive sudo access to `nvidia-smi`. Command line only; inactive with `--autotune-effort off`. | off | diff --git a/helios/proto/runtime_core.proto b/helios/proto/runtime_core.proto index ff6466727..603c682d7 100644 --- a/helios/proto/runtime_core.proto +++ b/helios/proto/runtime_core.proto @@ -114,6 +114,8 @@ message RebuildManifest { string collective_ladder_json = 26; // Preparation-only routed W4A16 decode/verification with calibrated A4 prefill. bool force_moe_decode_a16 = 27; + // Preparation-only row-sharded prefill collectives; requires proxy ABI 11 or later. + bool row_shard_prefill = 28; // Preparation-only online conversion group; omitted means all eligible projections. string online_mxfp8_scope = 30; // Preparation context group size; zero/absent retains full-TP context sharding when enabled. diff --git a/ignition/cli/__init__.py b/ignition/cli/__init__.py index 63f60d8f5..a7a27be09 100644 --- a/ignition/cli/__init__.py +++ b/ignition/cli/__init__.py @@ -232,6 +232,10 @@ def _preparation_arguments(running, *, model_default="defaults.model"): advanced.add_argument("--joint-tuning", action="store_true", help="jointly tune DeepSeek V4.1 attention projection plans across the prepared" " prefill, decode, and speculative shapes (default: off)") + advanced.add_argument("--row-shard-prefill", action="store_true", + help="reduce DeepSeek V4.1 prefill into one contiguous row shard per rank, run the" + " hyperconnection on that shard, and gather the branch input every row reads" + " (default: tensor-parallel all-reduce)") advanced.add_argument("--lock-gpu-clocks", action="store_true", help="lock each tuning GPU to its maximum supported graphics clock during" " autotuning; requires root or noninteractive sudo for nvidia-smi") diff --git a/ignition/cli/bench.py b/ignition/cli/bench.py index ae39a76c2..6a218bdcd 100644 --- a/ignition/cli/bench.py +++ b/ignition/cli/bench.py @@ -166,6 +166,7 @@ def run(args): lock_gpu_clocks=args.lock_gpu_clocks, native_dense_activations=args.native_dense_activations, force_moe_decode_a16=args.force_moe_decode_a16, shard_hyperconnections=args.shard_hyperconnections, joint_tuning=args.joint_tuning, + row_shard_prefill=args.row_shard_prefill, image_concurrency=args.image_concurrency, mixed_prefill=args.mixed_prefill, prompt_logprobs=args.prompt_logprobs, expert_scale_compression=args.expert_scale_compression, online_mxfp8=args.online_mxfp8, online_mxfp8_scope=args.online_mxfp8_scope, diff --git a/ignition/cli/preparation.py b/ignition/cli/preparation.py index 5cd907854..fecedb919 100644 --- a/ignition/cli/preparation.py +++ b/ignition/cli/preparation.py @@ -51,7 +51,7 @@ def derive(configuration: Configuration, model: ResolvedModel, host: Host, *, gp compile_workers: int | None = None, tuning_gpus: str | None = None, autotune_effort: str | None = None, collective_tuning: str | None = None, collective_ladder=None, image_concurrency: int | None = None, mla_compute_mode: str | None = None, lock_gpu_clocks: bool = False, native_dense_activations: bool = False, - joint_tuning: bool = False, shard_hyperconnections: bool = False, + joint_tuning: bool = False, shard_hyperconnections: bool = False, row_shard_prefill: bool = False, force_moe_decode_a16: bool | None = None) -> planning.Plan: """Select GPUs and estimate memory for one checkpoint from the configuration and command-line intent.""" from ignition.preparation.collective_ladder import resolve as resolve_ladder @@ -72,6 +72,7 @@ def derive(configuration: Configuration, model: ResolvedModel, host: Host, *, gp collective_tuning=mode, collective_ladder=ladder, force_moe_decode_a16=configuration.get("preparation.force_moe_decode_a16", force_moe_decode_a16).value, joint_tuning=joint_tuning, shard_hyperconnections=shard_hyperconnections, + row_shard_prefill=row_shard_prefill, lock_gpu_clocks=lock_gpu_clocks, native_dense_activations=native_dense_activations) return tuning_devices(result, host, tuning_gpus) diff --git a/ignition/cli/serve.py b/ignition/cli/serve.py index 0ab463359..971275740 100644 --- a/ignition/cli/serve.py +++ b/ignition/cli/serve.py @@ -209,7 +209,8 @@ def _export_hosts(args): collective_ladder=resolve_collective_ladder(collective_mode, args.collective_ladder), lock_gpu_clocks=args.lock_gpu_clocks, native_dense_activations=args.native_dense_activations, force_moe_decode_a16=configuration.get("preparation.force_moe_decode_a16", args.force_moe_decode_a16).value, - shard_hyperconnections=args.shard_hyperconnections, joint_tuning=args.joint_tuning) + shard_hyperconnections=args.shard_hyperconnections, joint_tuning=args.joint_tuning, + row_shard_prefill=args.row_shard_prefill) environment.apply(configuration, local, plan.recipe.b12x_environment) options = _server_options(configuration, model, plan) options.pop("api_key") @@ -277,6 +278,7 @@ def __call__(self, message): lock_gpu_clocks=args.lock_gpu_clocks, native_dense_activations=args.native_dense_activations, force_moe_decode_a16=configuration.get("preparation.force_moe_decode_a16", args.force_moe_decode_a16).value, shard_hyperconnections=args.shard_hyperconnections, joint_tuning=args.joint_tuning, + row_shard_prefill=args.row_shard_prefill, image_concurrency=args.image_concurrency, speculation=speculation(args, model, configuration), prefill_chunk=args.prefill_chunk_size, context_parallel=context_parallel(args, model), context_parallel_size=getattr(args, "dcp_size", None), @@ -475,7 +477,8 @@ def debug_serve(args) -> int: collective_ladder=resolve_collective_ladder(collective_mode, args.collective_ladder), lock_gpu_clocks=args.lock_gpu_clocks, native_dense_activations=args.native_dense_activations, force_moe_decode_a16=configuration.get("preparation.force_moe_decode_a16", args.force_moe_decode_a16).value, - shard_hyperconnections=args.shard_hyperconnections, joint_tuning=args.joint_tuning) + shard_hyperconnections=args.shard_hyperconnections, joint_tuning=args.joint_tuning, + row_shard_prefill=args.row_shard_prefill) if rendezvous is None: plan = tuning_devices(plan, host, args.tuning_gpus) local = plan.gpus diff --git a/ignition/components/collectives.py b/ignition/components/collectives.py index 387043623..90123fce0 100644 --- a/ignition/components/collectives.py +++ b/ignition/components/collectives.py @@ -194,6 +194,239 @@ def __call__(self, source, output=None): return output +def row_shard_surfaces(runtime): + """True for a RoCE runtime whose proxy speaks the ABI 11 or later row-shard surfaces.""" + from b12x.comm.roce._proxy import ABI_VERSION + + return (_roce(runtime) and ABI_VERSION >= 11 + and hasattr(runtime, "should_reduce_scatter") and hasattr(runtime, "should_gather_shards")) + + +@torch.library.custom_op("flare::roce_reduce_scatter", mutates_args=("output",)) +def _reduce_scatter(source: torch.Tensor, output: torch.Tensor, handle: int) -> None: + owner = _OWNERS[handle] + from ignition.preparation.startup_tuning import record_consumer + record_consumer(owner) + owner.runtime.reduce_scatter(source, out=output, plan=owner.plan) + + +@_reduce_scatter.register_fake +def _fake_reduce_scatter(source, output, handle): + pass + + +class PreparedReduceScatter: + """One fixed-shape reduction into this rank's contiguous row shard through a RoCE runtime. + + The reduced rows split into `world_size` equal shards and each rank keeps its own, with the + FP32 rank-order accumulation of the pipelined two-shot all-reduce, so a following + PreparedGatherShards reproduces that all-reduce's bits. The bootstrap owns the runtime, + exchange group and coordinator; every participating rank declares the same operation name + and enters preparation together. + """ + + def __init__(self, source, output, *, runtime, session, coordinator, name): + import torch.distributed as dist + + self.runtime, self.output = runtime, output + # Traced forwards test this flag, never the runtime: compile workers see a runtime that + # owns RDMA resources only as a placeholder. + self.single = runtime is None + if runtime is None: + if source.shape != output.shape or source.dtype != output.dtype: + raise ValueError("a single-rank reduction copies its source into a matching output") + return + if not self.fits(runtime, source, output): + raise ValueError("RoCE row reductions require contiguous BF16 16-byte packs dividing into" + " this rank's shard within the runtime's pipe capacity") + self.plan = _roce_plan(runtime, coordinator, "AllReduce.reduce_scatter") + ranks = tuple(dist.get_process_group_ranks(runtime._group)) + + def prepare(state): + return PreparedCall(run=lambda: state.reduce_scatter(source, out=output), output=output, + owners=(runtime, source, output)) + declare(session, (self.plan.request(name=name, prepare_call=prepare, + collective=CollectiveRequirement(key=name, ranks=ranks)),), coordinator=coordinator) + self.handle = next(_HANDLES) + _OWNERS[self.handle] = self + + @staticmethod + def fits(runtime, source, output): + """True when this runtime can split `source`'s rows into the shard `output` holds.""" + if runtime is None or not row_shard_surfaces(runtime) or source.ndim < 1: + return False + rows, world = source.shape[0], runtime.world_size + row_bytes = source.numel() // max(rows, 1) * source.element_size() + return (rows % world == 0 + and tuple(output.shape) == (rows // world, *source.shape[1:]) + and source.dtype is torch.bfloat16 and output.dtype is source.dtype + and source.is_contiguous() and output.is_contiguous() + and row_bytes % 16 == 0 + and 0 < source.numel() * source.element_size() <= runtime.pipe_bytes + and runtime.should_reduce_scatter(source)) + + def functional(self, *, session): + """Allocate this rank's shard destination inside the traced forward.""" + if self.output is not None: + self.output_layout = (tuple(self.output.shape), self.output.stride(), self.output.dtype) + self.output = None + return self + + def __call__(self, source, output=None): + if output is None: + output = self.output + if output is None: + shape, stride, dtype = self.output_layout + output = torch.empty_strided(shape, stride, dtype=dtype, device=source.device) + if self.single: + output.copy_(source) + else: + _reduce_scatter(source, output, self.handle) + return output + + +@torch.library.custom_op("flare::roce_gather_shards", mutates_args=("output",)) +def _gather_shards(source: torch.Tensor, output: torch.Tensor, handle: int) -> None: + owner = _OWNERS[handle] + from ignition.preparation.startup_tuning import record_consumer + record_consumer(owner) + owner.runtime.gather_shards(source, out=output, plan=owner.plan) + + +@_gather_shards.register_fake +def _fake_gather_shards(source, output, handle): + pass + + +class PreparedGatherShards: + """One fixed-shape gather of every rank's row shard into all rows through a RoCE runtime. + + Rank zero's shard comes first in the output rows, then rank one's, and so on, copied without + arithmetic. The bootstrap owns the runtime, exchange group and coordinator; every + participating rank declares the same operation name and enters preparation together. + """ + + def __init__(self, source, output, *, runtime, session, coordinator, name): + import torch.distributed as dist + + self.runtime, self.output = runtime, output + # Traced forwards test this flag, never the runtime: compile workers see a runtime that + # owns RDMA resources only as a placeholder. + self.single = runtime is None + if runtime is None: + if source.shape != output.shape or source.dtype != output.dtype: + raise ValueError("a single-rank gather copies its shard into a matching output") + return + if not self.fits(runtime, source, output): + raise ValueError("RoCE shard gathers require contiguous BF16 16-byte packs gathering into" + " the runtime's pipe capacity") + self.plan = _roce_plan(runtime, coordinator, "AllReduce.gather_shards") + ranks = tuple(dist.get_process_group_ranks(runtime._group)) + + def prepare(state): + return PreparedCall(run=lambda: state.gather_shards(source, out=output), output=output, + owners=(runtime, source, output)) + declare(session, (self.plan.request(name=name, prepare_call=prepare, + collective=CollectiveRequirement(key=name, ranks=ranks)),), coordinator=coordinator) + self.handle = next(_HANDLES) + _OWNERS[self.handle] = self + + @staticmethod + def fits(runtime, source, output): + """True when this runtime gathers `source`'s shard rows into every rank's `output`.""" + if runtime is None or not row_shard_surfaces(runtime) or source.ndim < 1: + return False + rows, world = source.shape[0], runtime.world_size + row_bytes = source.numel() // max(rows, 1) * source.element_size() + return (tuple(output.shape) == (rows * world, *source.shape[1:]) + and source.dtype is torch.bfloat16 and output.dtype is source.dtype + and source.is_contiguous() and output.is_contiguous() + and row_bytes % 16 == 0 + and 0 < output.numel() * output.element_size() <= runtime.pipe_bytes + and runtime.should_gather_shards(source)) + + def functional(self, *, session): + """Allocate every rank's gathered rows inside the traced forward.""" + if self.output is not None: + self.output_layout = (tuple(self.output.shape), self.output.stride(), self.output.dtype) + self.output = None + return self + + def __call__(self, source, output=None): + if output is None: + output = self.output + if output is None: + shape, stride, dtype = self.output_layout + output = torch.empty_strided(shape, stride, dtype=dtype, device=source.device) + if self.single: + output.copy_(source) + else: + _gather_shards(source, output, self.handle) + return output + + +class PreparedChunkedGatherShards: + """Shard gathers larger than the runtime's pipe capacity, gathered in shard-row chunks. + + Each chunk gathers ``world_size`` slices of `step` shard rows into a contiguous message, which is + copied into the matching rows of every rank's full-size output. A gather publishes rank-major rows, + so a chunk of shard rows occupies one strided window of the output and the chunks stay in order. + """ + + def __init__(self, source, output, *, runtime, session, coordinator, name): + rows, world = source.shape[0], 1 if runtime is None else runtime.world_size + step = rows + if runtime is not None: + # Each rank contributes one shard of the gathered message. + row_bytes = source.numel() // max(rows, 1) * source.element_size() + step = max(1, min(rows, runtime.pipe_bytes // (row_bytes * world))) + self.bounds = [(start, min(start + step, rows)) for start in range(0, rows, step)] + staging = torch.empty((world * step, *source.shape[1:]), dtype=source.dtype, device=source.device) + self.parts = [PreparedGatherShards(source[begin:end], staging[:world * (end - begin)], runtime=runtime, + session=session, coordinator=coordinator, + name=name if len(self.bounds) == 1 else f"{name}.{index}") + for index, (begin, end) in enumerate(self.bounds)] + # A captured graph allocates its staging and output inside the forward; `staging` sizes the + # chunks and the parts keep the shape of every gathered message. + self.staging_layout = (tuple(staging.shape), staging.stride()) + del staging + self.output, self.world = output, world + + @staticmethod + def fits(runtime, source, output): + """True when this runtime gathers `source`'s shards into `output` in whole shard-row chunks.""" + if runtime is None or not row_shard_surfaces(runtime) or source.ndim < 1 or not source.shape[0]: + return False + world = runtime.world_size + # One shard row per rank is the smallest chunk, so its eligibility covers every larger chunk. + return (tuple(output.shape) == (source.shape[0] * world, *source.shape[1:]) + and PreparedGatherShards.fits(runtime, source[:1], output[:world])) + + def functional(self, *, session): + """Allocate the gathered rows and the chunk staging inside the traced forward.""" + if self.output is not None: + self.output_layout = (tuple(self.output.shape), self.output.stride(), self.output.dtype) + self.output = None + for part in self.parts: + part.functional(session=session) + return self + + def __call__(self, source, output=None): + if output is None: + output = self.output + if output is None: + shape, stride, dtype = self.output_layout + output = torch.empty_strided(shape, stride, dtype=dtype, device=source.device) + shape, stride = self.staging_layout + staging = torch.empty_strided(shape, stride, dtype=source.dtype, device=source.device) + window = output.view(self.world, source.shape[0], *output.shape[1:]) + for (begin, end), part in zip(self.bounds, self.parts, strict=True): + chunk = staging[:self.world * (end - begin)] + part(source[begin:end], chunk) + window[:, begin:end].copy_(chunk.view(self.world, end - begin, *window.shape[2:])) + return output + + class PreparedChunkedAllGather: """Column gathers of more rows than the channel's batch capacity, gathered in row chunks it accepts. diff --git a/ignition/planning.py b/ignition/planning.py index d28dcca00..f37c54829 100644 --- a/ignition/planning.py +++ b/ignition/planning.py @@ -192,6 +192,9 @@ class Plan: collective_tuning: str | None = None joint_tuning: bool = False shard_hyperconnections: bool = False + # Prefill replaces each tensor-parallel all-reduce with a reduce-scatter, an mHC on one row + # shard, and a shard gather when the RoCE runtime serves those surfaces. + row_shard_prefill: bool = False lock_gpu_clocks: bool = False native_dense_activations: bool = False force_moe_decode_a16: bool = False @@ -270,6 +273,20 @@ def __post_init__(self): raise UsageError("--shard-hyperconnections requires Qwen3.8 Flash") if self.world_size < 2: raise UsageError("--shard-hyperconnections requires tensor parallelism") + if self.row_shard_prefill: + if self.recipe.name != "deepseek41": + raise UsageError("--row-shard-prefill requires DeepSeek V4.1") + if self.world_size < 2: + raise UsageError("--row-shard-prefill requires tensor parallelism") + # The recipe's 4,096-row chunk cannot be divided evenly among three ranks. + # A 6,144-row chunk would divide evenly. + if self.world_size == 3: + raise UsageError("--row-shard-prefill has no TP3 row partition; select four GPUs") + # The collectives are architecture independent, but the projections and experts on + # SM103 are a different kernel path. + if any(gpu.capability == (10, 3) for gpu in self.gpus): + raise UsageError("--row-shard-prefill is not qualified on SM103;" + " select SM120 or SM121 GPUs") if self.joint_tuning: if self.recipe.name != "deepseek41": raise UsageError("--joint-tuning requires DeepSeek V4.1") @@ -389,6 +406,7 @@ def to_dict(self) -> dict: "collective_ladder": self.collective_ladder, "joint_tuning": self.joint_tuning, "shard_hyperconnections": self.shard_hyperconnections, + "row_shard_prefill": self.row_shard_prefill, "lock_gpu_clocks": self.lock_gpu_clocks, "native_dense_activations": self.native_dense_activations, "force_moe_decode_a16": self.force_moe_decode_a16, @@ -431,7 +449,8 @@ def fit(recipe: Recipe, checkpoint: Checkpoint, gpus: tuple[Gpu, ...], *, concur expert_scale_compression: str | None = None, roce_ring: bool = False, roce_chain: bool = False, image_concurrency: int | None = None, online_mxfp8: bool = False, online_mxfp8_scope: str | None = None, - language_model_only: bool = False, mla_compute_mode: str | None = None) -> Plan: + language_model_only: bool = False, mla_compute_mode: str | None = None, + row_shard_prefill: bool = False) -> Plan: """Size pages and the prefix cache to the supply; the result may not fit. Weights come from the checkpoint's stored tensors and graphs and scratch from @@ -502,7 +521,7 @@ def fit(recipe: Recipe, checkpoint: Checkpoint, gpus: tuple[Gpu, ...], *, concur context_parallel_size=context_parallel_size, stream_overlap=stream_overlap and recipe.stream_overlap, mixed_prefill=mixed_prefill, prompt_logprobs=prompt_logprobs, expert_scale_compression=expert_scale_compression, roce_ring=roce_ring, roce_chain=roce_chain, - mla_compute_mode=mla_compute_mode, + row_shard_prefill=row_shard_prefill, mla_compute_mode=mla_compute_mode, # Language-only plans prepare no vision encoder, so no image feature slots. image_concurrency=0 if language_model_only else (min(4, concurrency) if recipe.max_images else 0) if image_concurrency is None else image_concurrency, online_mxfp8=online_mxfp8, online_mxfp8_scope=online_mxfp8_scope, @@ -561,7 +580,8 @@ def plan(recipe: Recipe, checkpoint: Checkpoint, gpus: tuple[Gpu, ...], *, selec cache_states: int | None = None, mixed_prefill: bool = False, prompt_logprobs: bool = False, expert_scale_compression: str | None = None, roce_ring: bool = False, roce_chain: bool = False, image_concurrency: int | None = None, online_mxfp8: bool = False, online_mxfp8_scope: str | None = None, - language_model_only: bool = False, mla_compute_mode: str | None = None) -> Plan: + language_model_only: bool = False, mla_compute_mode: str | None = None, + row_shard_prefill: bool = False) -> Plan: """Select GPUs and size the plan, or raise a capacity error that lists working settings. Without `selected`, the plan uses the smallest supported tensor-parallel size @@ -627,6 +647,7 @@ def fitted(group, *, concurrency=concurrency, reserve_bytes=reserve_bytes, specu stream_overlap=stream_overlap, cache_states=cache_states, mixed_prefill=mixed_prefill, prompt_logprobs=prompt_logprobs, expert_scale_compression=expert_scale_compression, roce_ring=roce_ring, roce_chain=roce_chain, + row_shard_prefill=row_shard_prefill, image_concurrency=None if image_concurrency is None else min(image_concurrency, concurrency), online_mxfp8=online_mxfp8, online_mxfp8_scope=online_mxfp8_scope, language_model_only=language_model_only, mla_compute_mode=mla_compute_mode) diff --git a/ignition/preparation/rebuild_manifest.py b/ignition/preparation/rebuild_manifest.py index 80f2cda5b..912cd184c 100644 --- a/ignition/preparation/rebuild_manifest.py +++ b/ignition/preparation/rebuild_manifest.py @@ -235,6 +235,8 @@ def from_plan(model, plan, *, compiled_forwards): result['force_moe_decode_a16'] = True if plan.shard_hyperconnections: result['shard_hyperconnections'] = True + if plan.row_shard_prefill: + result['row_shard_prefill'] = True if plan.joint_tuning: result['joint_tuning'] = True if (s := plan.speculation) is not None: diff --git a/ignition/preparation/runtime_core_pb2.py b/ignition/preparation/runtime_core_pb2.py index 68da9eaa8..77059b150 100644 --- a/ignition/preparation/runtime_core_pb2.py +++ b/ignition/preparation/runtime_core_pb2.py @@ -2,7 +2,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: runtime_core.proto -# Protobuf Python Version: 7.35.1 +# Protobuf Python Version: 7.36.2 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -12,8 +12,8 @@ _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, 7, - 35, - 1, + 36, + 2, '', 'runtime_core.proto' ) @@ -24,7 +24,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12runtime_core.proto\x12\rflare.runtime\"\x99\x01\n\x0eProtectedIndex\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x0f\n\x07\x63ore_id\x18\x02 \x01(\x0c\x12\x0e\n\x06signer\x18\x03 \x01(\x0c\x12*\n\x06groups\x18\x04 \x03(\x0b\x32\x1a.flare.runtime.SignedGroup\x12)\n\x05ranks\x18\x05 \x03(\x0b\x32\x1a.flare.runtime.SignedGroup\"<\n\x0bSignedGroup\x12\x0c\n\x04kind\x18\x01 \x01(\r\x12\x0c\n\x04\x62ody\x18\x02 \x01(\x0c\x12\x11\n\tsignature\x18\x03 \x01(\x0c\"\xcc\x01\n\x11PublisherMetadata\x12(\n\x05index\x18\x01 \x01(\x0b\x32\x19.flare.runtime.ModelIndex\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x17\n\x0fofficial_source\x18\x03 \x01(\t\x12\x14\n\x0cgroup_hashes\x18\x04 \x03(\x0c\x12\x13\n\x0b\x64\x65velopment\x18\x05 \x01(\x08\x12\r\n\x05ranks\x18\x06 \x01(\r\x12\'\n\x07license\x18\x07 \x01(\x0b\x32\x16.flare.runtime.License\"%\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04link\x18\x02 \x01(\t\"F\n\x0fResourceCatalog\x12\x33\n\tresources\x18\x01 \x03(\x0b\x32 .flare.runtime.ProtectedResource\"^\n\x11ProtectedResource\x12\x0e\n\x06offset\x18\x01 \x01(\x04\x12\x14\n\x0cstored_bytes\x18\x02 \x01(\x04\x12\x13\n\x0bplain_bytes\x18\x03 \x01(\x04\x12\x0e\n\x06sha256\x18\x04 \x03(\x0c\"`\n\nStoredData\x12\x0e\n\x06nbytes\x18\x01 \x01(\x04\x12\x0e\n\x06sha256\x18\x02 \x01(\x0c\x12\x0e\n\x06offset\x18\x03 \x01(\x04\x12\"\n\x05parts\x18\x04 \x03(\x0b\x32\x13.flare.runtime.Span\"\xd9\x01\n\x0eRankProtection\x12\x0c\n\x04rank\x18\x01 \x01(\r\x12*\n\x06\x62undle\x18\x02 \x01(\x0b\x32\x1a.flare.runtime.BundleIndex\x12\x33\n\x0b\x65xecutables\x18\x03 \x01(\x0b\x32\x1e.flare.runtime.ResourceCatalog\x12/\n\x07support\x18\x04 \x01(\x0b\x32\x1e.flare.runtime.ResourceCatalog\x12\'\n\x04\x64\x61ta\x18\x05 \x01(\x0b\x32\x19.flare.runtime.StoredData\"\xcf\x03\n\nModelIndex\x12\x0f\n\x07version\x18\x01 \x01(\r\x12)\n\x05ranks\x18\x02 \x03(\x0b\x32\x1a.flare.runtime.BundleIndex\x12.\n\x0bpeer_memory\x18\x03 \x03(\x0b\x32\x19.flare.runtime.PeerMemory\x12$\n\x06\x61ssets\x18\x05 \x03(\x0b\x32\x14.flare.runtime.Asset\x12%\n\x06\x65ngram\x18\x06 \x01(\x0b\x32\x15.flare.runtime.Engram\x12\x43\n\x11sampling_defaults\x18\x07 \x01(\x0b\x32(.flare.runtime.DefaultSamplingParameters\x12\x31\n\x08\x66rontend\x18\x08 \x01(\x0b\x32\x1f.flare.runtime.FrontendSettings\x12\'\n\x07\x63luster\x18\t \x01(\x0b\x32\x16.flare.runtime.Cluster\x12\x30\n\x08\x63ontract\x18\n \x01(\x0b\x32\x1e.flare.runtime.ServingContract\x12/\n\x07rebuild\x18\x0b \x01(\x0b\x32\x1e.flare.runtime.RebuildManifestJ\x04\x08\x04\x10\x05\"\xba\x06\n\x0fRebuildManifest\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x0e\n\x06recipe\x18\x02 \x01(\t\x12\x17\n\x0fsource_identity\x18\x03 \x01(\t\x12\x1d\n\x15\x63heckpoint_repository\x18\x04 \x01(\t\x12\x1b\n\x13\x63heckpoint_revision\x18\x05 \x01(\t\x12\x17\n\x0ftensor_parallel\x18\x06 \x01(\r\x12\x17\n\x0fmax_concurrency\x18\x07 \x01(\x04\x12\x15\n\rprefill_chunk\x18\x08 \x01(\x04\x12\x19\n\x11image_concurrency\x18\t \x01(\x04\x12\x18\n\x10\x63ontext_parallel\x18\n \x01(\x08\x12\x16\n\x0estream_overlap\x18\x0b \x01(\x08\x12\x15\n\rmixed_prefill\x18\x0c \x01(\x08\x12%\n\x18\x65xpert_scale_compression\x18\r \x01(\tH\x00\x88\x01\x01\x12\x18\n\x10mla_compute_mode\x18\x0e \x01(\t\x12\x11\n\troce_ring\x18\x0f \x01(\x08\x12\x14\n\x0conline_mxfp8\x18\x10 \x01(\x08\x12\x1b\n\x13language_model_only\x18\x11 \x01(\x08\x12\x36\n\x0bspeculation\x18\x12 \x01(\x0b\x32!.flare.runtime.RebuildSpeculation\x12\x19\n\x11\x63ompiled_forwards\x18\x13 \x01(\x08\x12\x15\n\rreserve_bytes\x18\x14 \x01(\x04\x12\x18\n\x10local_checkpoint\x18\x15 \x01(\x08\x12\x12\n\nroce_chain\x18\x16 \x01(\x08\x12\x14\n\x0cjoint_tuning\x18\x17 \x01(\x08\x12\x1e\n\x16shard_hyperconnections\x18\x18 \x01(\x08\x12\x19\n\x11\x63ollective_tuning\x18\x19 \x01(\t\x12\x1e\n\x16\x63ollective_ladder_json\x18\x1a \x01(\t\x12\x1c\n\x14\x66orce_moe_decode_a16\x18\x1b \x01(\x08\x12\x1a\n\x12online_mxfp8_scope\x18\x1e \x01(\t\x12\x1d\n\x15\x63ontext_parallel_size\x18\x1f \x01(\rB\x1b\n\x19_expert_scale_compression\"\xfe\x03\n\x12RebuildSpeculation\x12\x14\n\x0c\x64raft_tokens\x18\x01 \x01(\r\x12\x10\n\x08\x61\x64\x61ptive\x18\x02 \x01(\x08\x12\x17\n\x0fproposal_source\x18\x03 \x01(\t\x12\x16\n\x0ehistory_drafts\x18\x04 \x01(\x08\x12\x16\n\x0etoken_capacity\x18\x05 \x01(\x04\x12\x1b\n\x13packed_verification\x18\x06 \x01(\x08\x12\x1e\n\x16\x64raft_prefix_selection\x18\x07 \x01(\t\x12$\n\x17\x64raft_head_quantization\x18\x08 \x01(\tH\x00\x88\x01\x01\x12%\n\x18markov_head_quantization\x18\t \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0cgraph_budget\x18\n \x01(\x04H\x02\x88\x01\x01\x12$\n\x1cvocabulary_parallel_sampling\x18\x0b \x01(\x08\x12\x0e\n\x06method\x18\x0c \x01(\t\x12\x10\n\x08mtp_only\x18\r \x01(\x08\x12$\n\x17\x64raft_body_quantization\x18\x0e \x01(\tH\x03\x88\x01\x01\x42\x1a\n\x18_draft_head_quantizationB\x1b\n\x19_markov_head_quantizationB\x0f\n\r_graph_budgetB\x1a\n\x18_draft_body_quantization\"e\n\x0eRebuildWeights\x12\x0f\n\x07version\x18\x01 \x01(\r\x12-\n\x07tensors\x18\x02 \x03(\x0b\x32\x1c.flare.runtime.RebuildTensor\x12\x13\n\x0bunavailable\x18\x03 \x03(\t\"\xa2\x01\n\rRebuildTensor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x64type\x18\x02 \x01(\t\x12\x15\n\relement_bytes\x18\x03 \x01(\r\x12\r\n\x05shape\x18\x04 \x03(\x04\x12\x0e\n\x06stride\x18\x05 \x03(\x04\x12\x13\n\x0blayout_type\x18\x06 \x01(\t\x12)\n\x07storage\x18\x07 \x01(\x0b\x32\x18.flare.runtime.Reference\"\xcb\x06\n\x0fServingContract\x12\x10\n\x08model_id\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x34\n\x0emodel_contract\x18\x03 \x01(\x0b\x32\x1c.flare.runtime.ModelContract\x12\x1b\n\x13max_sequence_tokens\x18\x04 \x01(\x04\x12\x1a\n\x12prefill_chunk_size\x18\x05 \x01(\x04\x12\r\n\x05pages\x18\x06 \x01(\x04\x12\x12\n\nstate_rows\x18\x07 \x01(\x04\x12\x11\n\tsnapshots\x18\x08 \x01(\x04\x12\x13\n\x0bpage_tokens\x18\t \x01(\x04\x12\x1a\n\x12speculative_widths\x18\x0b \x03(\x04\x12$\n\x17speculative_fixed_width\x18\x0c \x01(\x04H\x00\x88\x01\x01\x12\x16\n\x0ehistory_drafts\x18\r \x01(\x08\x12#\n\x1bspeculative_proposal_source\x18\x0e \x01(\t\x12\x1b\n\x0emax_new_tokens\x18\x0f \x01(\x04H\x01\x88\x01\x01\x12\x17\n\nmax_active\x18\x10 \x01(\x04H\x02\x88\x01\x01\x12\x14\n\x0cserved_names\x18\x11 \x03(\t\x12\x1a\n\rreserve_bytes\x18\x12 \x01(\x04H\x03\x88\x01\x01\x12\x1b\n\x0equeue_capacity\x18\x13 \x01(\x04H\x04\x88\x01\x01\x12\x1d\n\x10\x66rontend_workers\x18\x14 \x01(\x04H\x05\x88\x01\x01\x12\x1b\n\x0emax_body_bytes\x18\x15 \x01(\x04H\x06\x88\x01\x01\x12\x14\n\x0c\x63ontext_hint\x18\x16 \x01(\t\x12\x15\n\rcapacity_hint\x18\x17 \x01(\t\x12\"\n\x1aspeculative_token_capacity\x18\x18 \x01(\x04\x12 \n\x13\x64\x65\x63ode_weight_bytes\x18\x19 \x01(\x04H\x07\x88\x01\x01\x42\x1a\n\x18_speculative_fixed_widthB\x11\n\x0f_max_new_tokensB\r\n\x0b_max_activeB\x10\n\x0e_reserve_bytesB\x11\n\x0f_queue_capacityB\x13\n\x11_frontend_workersB\x11\n\x0f_max_body_bytesB\x16\n\x14_decode_weight_bytesJ\x04\x08\n\x10\x0b\"\x8b\x02\n\rModelContract\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x12\n\nvocabulary\x18\x02 \x01(\x04\x12\x18\n\x10input_vocabulary\x18\x03 \x01(\x04\x12-\n\x06vision\x18\x04 \x01(\x0b\x32\x1d.flare.runtime.VisionCapacity\x12+\n\x05\x63\x61\x63he\x18\x05 \x01(\x0b\x32\x1c.flare.runtime.CacheContract\x12,\n\x06graphs\x18\x06 \x03(\x0b\x32\x1c.flare.runtime.GraphContract\x12\x31\n\x08\x66rontend\x18\x07 \x01(\x0b\x32\x1f.flare.runtime.FrontendContract\"\x90\x01\n\x0eVisionCapacity\x12\x15\n\rfeature_slots\x18\x05 \x01(\x04\x12\x1c\n\x14\x66\x65\x61tures_per_request\x18\x01 \x01(\x04\x12\x12\n\nmax_images\x18\x02 \x01(\x04\x12\x1b\n\x13patches_per_feature\x18\x03 \x01(\x04\x12\x18\n\x10patch_capacities\x18\x04 \x03(\x04\"\x86\x01\n\rCacheContract\x12\x13\n\x0bpage_tokens\x18\x01 \x01(\x04\x12-\n\x07regions\x18\x02 \x03(\x0b\x32\x1c.flare.runtime.HistoryRegion\x12\x31\n\x0bpage_tables\x18\x03 \x03(\x0b\x32\x1c.flare.runtime.PageTableView\"\x96\x01\n\rHistoryRegion\x12(\n\x04kind\x18\x01 \x01(\x0e\x32\x1a.flare.runtime.HistoryKind\x12)\n\x07storage\x18\x02 \x01(\x0b\x32\x18.flare.runtime.StateView\x12\x19\n\x11tokens_per_record\x18\x03 \x01(\x04\x12\x15\n\rwindow_tokens\x18\x04 \x01(\x04\"4\n\rPageTableView\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\t\x12\x13\n\x0bpage_offset\x18\x02 \x01(\x04\"\xb1\x01\n\x0cValueBinding\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\t\x12*\n\x06source\x18\x02 \x01(\x0e\x32\x1a.flare.runtime.ValueSource\x12.\n\x08indexing\x18\x03 \x01(\x0e\x32\x1c.flare.runtime.InputIndexing\x12\x14\n\x0cvalue_offset\x18\x04 \x01(\x12\x12\x0f\n\x07padding\x18\x05 \x01(\x12\x12\x0e\n\x06tensor\x18\x06 \x01(\t\"K\n\tStateView\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\t\x12\x0e\n\x06offset\x18\x02 \x01(\x04\x12\x0e\n\x06stride\x18\x03 \x01(\x04\x12\x0e\n\x06nbytes\x18\x04 \x01(\x04\"e\n\x0bStateAccess\x12&\n\x04view\x18\x01 \x01(\x0b\x32\x18.flare.runtime.StateView\x12.\n\x06\x61\x63\x63\x65ss\x18\x02 \x01(\x0e\x32\x1e.flare.runtime.StateAccessKind\"O\n\x10\x43ompletionCursor\x12\'\n\x05state\x18\x01 \x01(\x0b\x32\x18.flare.runtime.StateView\x12\x12\n\nlag_tokens\x18\x02 \x01(\x04\"4\n\x0cTunedBinding\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\t\x12\x14\n\x0ctuning_graph\x18\x02 \x01(\t\"\xfa\x02\n\rGraphContract\x12\r\n\x05graph\x18\x01 \x01(\t\x12+\n\x06inputs\x18\x02 \x03(\x0b\x32\x1b.flare.runtime.ValueBinding\x12)\n\x05state\x18\x03 \x03(\x0b\x32\x1a.flare.runtime.StateAccess\x12\x1e\n\x16state_rows_per_request\x18\x04 \x01(\x04\x12\x35\n\x0c\x64raft_cursor\x18\x05 \x01(\x0b\x32\x1f.flare.runtime.CompletionCursor\x12\x18\n\x10\x65mbedding_tables\x18\x06 \x03(\r\x12/\n\rpending_token\x18\x07 \x01(\x0b\x32\x18.flare.runtime.StateView\x12+\n\x04kind\x18\x08 \x01(\x0e\x32\x1d.flare.runtime.GraphOperation\x12\x33\n\x0etuned_bindings\x18\t \x03(\x0b\x32\x1b.flare.runtime.TunedBinding\"k\n\nToolParser\x12+\n\x04kind\x18\x01 \x01(\x0e\x32\x1d.flare.runtime.ToolParserKind\x12\x19\n\x11wrapping_newlines\x18\x02 \x01(\r\x12\x15\n\rtag_separator\x18\x03 \x01(\t\"\x95\x01\n\x0fReasoningParser\x12\x30\n\x04kind\x18\x01 \x01(\x0e\x32\".flare.runtime.ReasoningParserKind\x12\x0c\n\x04open\x18\x02 \x01(\t\x12\r\n\x05\x63lose\x18\x03 \x01(\t\x12\x19\n\x11wrapping_newlines\x18\x04 \x01(\r\x12\x18\n\x10terminal_markers\x18\x05 \x03(\t\"\xca\x01\n\x0ePrefixBoundary\x12\r\n\x05roles\x18\x01 \x03(\t\x12\x18\n\x0b\x61ppend_role\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x1e\n\x16\x63ontinue_final_message\x18\x03 \x01(\x08\x12\x18\n\x10minimum_messages\x18\x04 \x01(\x04\x12\x16\n\x0erequired_start\x18\x05 \x01(\t\x12\x14\n\x0crequired_end\x18\x06 \x01(\t\x12\x17\n\x0frequired_marker\x18\x07 \x01(\tB\x0e\n\x0c_append_role\"y\n\tImageSpan\x12\x13\n\x0bstart_token\x18\x01 \x01(\r\x12\x11\n\tend_token\x18\x02 \x01(\r\x12\x12\n\nmax_tokens\x18\x03 \x01(\x04\x12\x17\n\x0finterior_tokens\x18\x04 \x03(\r\x12\x17\n\x0funscoped_tokens\x18\x05 \x03(\r\"Q\n\x10ImageTokenLayout\x12\x15\n\rfeature_token\x18\x01 \x01(\r\x12&\n\x04span\x18\x02 \x01(\x0b\x32\x18.flare.runtime.ImageSpan\"\xf2\x03\n\x10\x46rontendContract\x12(\n\x05tools\x18\x01 \x01(\x0b\x32\x19.flare.runtime.ToolParser\x12\x31\n\treasoning\x18\x02 \x01(\x0b\x32\x1e.flare.runtime.ReasoningParser\x12\x43\n\x11image_preparation\x18\x03 \x01(\x0e\x32#.flare.runtime.ImagePreparationKindH\x00\x88\x01\x01\x12\x39\n\x0cimage_layout\x18\x04 \x01(\x0e\x32\x1e.flare.runtime.ImageLayoutKindH\x01\x88\x01\x01\x12\x17\n\x0frejected_tokens\x18\x05 \x03(\r\x12\x1b\n\x13position_dimensions\x18\x06 \x01(\r\x12\x18\n\x10\x61ssistant_marker\x18\x07 \x01(\t\x12\x1b\n\x13message_end_markers\x18\x08 \x03(\t\x12\x36\n\x0fprefix_boundary\x18\t \x01(\x0b\x32\x1d.flare.runtime.PrefixBoundary\x12\x35\n\x0cimage_tokens\x18\n \x01(\x0b\x32\x1f.flare.runtime.ImageTokenLayoutB\x14\n\x12_image_preparationB\x0f\n\r_image_layout\"1\n\x07\x43luster\x12\r\n\x05world\x18\x01 \x01(\r\x12\x11\n\texport_id\x18\x03 \x01(\tJ\x04\x08\x02\x10\x03\"\x8c\x03\n\x19\x44\x65\x66\x61ultSamplingParameters\x12\x11\n\x04seed\x18\x01 \x01(\x04H\x00\x88\x01\x01\x12\x18\n\x0btemperature\x18\x02 \x01(\x02H\x01\x88\x01\x01\x12\x12\n\x05top_k\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05top_p\x18\x04 \x01(\x02H\x03\x88\x01\x01\x12\x12\n\x05min_p\x18\x05 \x01(\x02H\x04\x88\x01\x01\x12\x1f\n\x12repetition_penalty\x18\x06 \x01(\x02H\x05\x88\x01\x01\x12\x1d\n\x10presence_penalty\x18\x07 \x01(\x02H\x06\x88\x01\x01\x12\x1e\n\x11\x66requency_penalty\x18\x08 \x01(\x02H\x07\x88\x01\x01\x12-\n\x06ranges\x18\t \x01(\x0b\x32\x1d.flare.runtime.SamplingRangesB\x07\n\x05_seedB\x0e\n\x0c_temperatureB\x08\n\x06_top_kB\x08\n\x06_top_pB\x08\n\x06_min_pB\x15\n\x13_repetition_penaltyB\x13\n\x11_presence_penaltyB\x14\n\x12_frequency_penalty\"H\n\x12SamplingFloatRange\x12\x10\n\x03min\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12\x10\n\x03max\x18\x02 \x01(\x02H\x01\x88\x01\x01\x42\x06\n\x04_minB\x06\n\x04_max\"F\n\x10SamplingIntRange\x12\x10\n\x03min\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x10\n\x03max\x18\x02 \x01(\x05H\x01\x88\x01\x01\x42\x06\n\x04_minB\x06\n\x04_max\"\x96\x03\n\x0eSamplingRanges\x12\x36\n\x0btemperature\x18\x01 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12.\n\x05top_k\x18\x02 \x01(\x0b\x32\x1f.flare.runtime.SamplingIntRange\x12\x30\n\x05top_p\x18\x03 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12\x30\n\x05min_p\x18\x04 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12=\n\x12repetition_penalty\x18\x05 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12;\n\x10presence_penalty\x18\x06 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12<\n\x11\x66requency_penalty\x18\x07 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\"\xc5\x02\n\x10\x46rontendSettings\x12\x33\n\ttokenizer\x18\x01 \x01(\x0b\x32 .flare.runtime.TokenizerSettings\x12+\n\x05image\x18\x02 \x01(\x0b\x32\x1c.flare.runtime.ImageSettings\x12\x15\n\reos_token_ids\x18\x03 \x03(\r\x12:\n\x11template_encoding\x18\x04 \x01(\x0e\x32\x1f.flare.runtime.TemplateEncoding\x12\x34\n\x0eprompt_encoder\x18\x05 \x01(\x0e\x32\x1c.flare.runtime.PromptEncoder\x12\'\n\x1aimage_placeholder_token_id\x18\x06 \x01(\rH\x00\x88\x01\x01\x42\x1d\n\x1b_image_placeholder_token_id\"\xdc\x02\n\x11TokenizerSettings\x12\x16\n\tbos_token\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\teos_token\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tunk_token\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsep_token\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x16\n\tpad_token\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x16\n\tcls_token\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\nmask_token\x18\x07 \x01(\tH\x06\x88\x01\x01\x12;\n\x19\x61\x64\x64itional_special_tokens\x18\x08 \x01(\x0b\x32\x18.flare.runtime.TokenListB\x0c\n\n_bos_tokenB\x0c\n\n_eos_tokenB\x0c\n\n_unk_tokenB\x0c\n\n_sep_tokenB\x0c\n\n_pad_tokenB\x0c\n\n_cls_tokenB\r\n\x0b_mask_token\"\x1b\n\tTokenList\x12\x0e\n\x06values\x18\x01 \x03(\t\"\xfc\x03\n\rImageSettings\x12\x30\n\x04qwen\x18\x01 \x01(\x0b\x32 .flare.runtime.QwenImageSettingsH\x00\x12\x30\n\x04mimo\x18\x02 \x01(\x0b\x32 .flare.runtime.MimoImageSettingsH\x00\x12)\n\tdeepseek4\x18\x03 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12\'\n\x07moonvit\x18\x04 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12$\n\x04step\x18\x05 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12%\n\x05radio\x18\x06 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12J\n\x12\x64\x65\x65pseek_row_major\x18\x07 \x01(\x0b\x32,.flare.runtime.DeepseekRowMajorImageSettingsH\x00\x12#\n\x03glm\x18\x08 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12=\n\x0bpatch_merge\x18\t \x01(\x0b\x32&.flare.runtime.PatchMergeImageSettingsH\x00\x12+\n\x0bpadded_grid\x18\n \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x42\t\n\x07profile\"\xb9\x01\n\x17PatchMergeImageSettings\x12\x12\n\npatch_size\x18\x01 \x01(\r\x12\x12\n\nmin_pixels\x18\x02 \x01(\x04\x12\x12\n\nmax_pixels\x18\x03 \x01(\x04\x12\x12\n\nimage_mean\x18\x04 \x03(\x02\x12\x11\n\timage_std\x18\x05 \x03(\x02\x12\x13\n\x0bimage_token\x18\x06 \x01(\t\x12\x13\n\x0bstart_token\x18\x07 \x01(\t\x12\x11\n\tend_token\x18\x08 \x01(\t\"\x87\x01\n\x1d\x44\x65\x65pseekRowMajorImageSettings\x12\x12\n\nmin_pixels\x18\x01 \x01(\x04\x12\x12\n\nmax_tokens\x18\x02 \x01(\x04\x12#\n\x16max_width_height_ratio\x18\x03 \x01(\x04H\x00\x88\x01\x01\x42\x19\n\x17_max_width_height_ratio\"\x82\x01\n\x11QwenImageSettings\x12\x12\n\nmin_pixels\x18\x01 \x01(\x04\x12\x12\n\nmax_pixels\x18\x02 \x01(\x04\x12\x15\n\rposition_side\x18\x03 \x01(\r\x12\x16\n\x0eimage_token_id\x18\x04 \x01(\r\x12\x16\n\x0evideo_token_id\x18\x05 \x01(\r\"b\n\x11MimoImageSettings\x12\x12\n\nmin_pixels\x18\x01 \x01(\x04\x12\x12\n\nmax_pixels\x18\x02 \x01(\x04\x12\x12\n\nimage_mean\x18\x03 \x03(\x02\x12\x11\n\timage_std\x18\x04 \x03(\x02\"|\n\x06\x45ngram\x12\x11\n\ttoken_map\x18\x01 \x03(\r\x12\x14\n\x0cpad_token_id\x18\x02 \x01(\r\x12\x1d\n\x15\x63ompressed_vocabulary\x18\x03 \x01(\r\x12*\n\x06tables\x18\x04 \x03(\x0b\x32\x1a.flare.runtime.EngramTable\"\x80\x04\n\x0b\x45ngramTable\x12\r\n\x05layer\x18\x01 \x01(\r\x12\x0c\n\x04rows\x18\x02 \x01(\x04\x12\x17\n\x0f\x65mbedding_width\x18\x03 \x01(\r\x12\x13\n\x0bscale_block\x18\x04 \x01(\r\x12$\n\x07weights\x18\x05 \x01(\x0b\x32\x13.flare.runtime.Span\x12#\n\x06scales\x18\x06 \x01(\x0b\x32\x13.flare.runtime.Span\x12\'\n\x04hash\x18\x07 \x01(\x0b\x32\x19.flare.runtime.EngramHash\x12\x15\n\rweight_buffer\x18\x08 \x01(\t\x12\x14\n\x0cscale_buffer\x18\t \x01(\t\x12\x13\n\x0bhash_buffer\x18\n \x01(\t\x12\x13\n\x0bready_event\x18\x0b \x01(\t\x12\x35\n\x08\x65ncoding\x18\x0c \x01(\x0e\x32#.flare.runtime.EngramTable.Encoding\x12\x16\n\x0erequest_buffer\x18\r \x01(\t\x12\x14\n\x0cglobal_scale\x18\x0e \x01(\x02\x12/\n\x06shards\x18\x0f \x03(\x0b\x32\x1f.flare.runtime.EngramTableShard\"E\n\x08\x45ncoding\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x11\n\rFP8_E4M3_E8M0\x10\x01\x12\x15\n\x11NVFP4_E4M3_GLOBAL\x10\x02\"\x8c\x01\n\x10\x45ngramTableShard\x12\x0c\n\x04rank\x18\x01 \x01(\r\x12\x11\n\tfirst_row\x18\x02 \x01(\x04\x12\x0c\n\x04rows\x18\x03 \x01(\x04\x12$\n\x07weights\x18\x04 \x01(\x0b\x32\x13.flare.runtime.Span\x12#\n\x06scales\x18\x05 \x01(\x0b\x32\x13.flare.runtime.Span\"\xc3\x01\n\nEngramHash\x12\x30\n\x06method\x18\x01 \x01(\x0e\x32 .flare.runtime.EngramHash.Method\x12\x0e\n\x06orders\x18\x02 \x03(\r\x12\x13\n\x0bmultipliers\x18\x03 \x03(\x04\x12\x0e\n\x06primes\x18\x04 \x03(\x04\x12\x0f\n\x07offsets\x18\x05 \x03(\x04\"=\n\x06Method\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nXOR_MODULO\x10\x01\x12\x12\n\x0e\x45OS_XOR_MODULO\x10\x02\"K\n\x05\x41sset\x12\x0c\n\x04name\x18\x01 \x01(\t\x12$\n\x07payload\x18\x02 \x01(\x0b\x32\x13.flare.runtime.Span\x12\x0e\n\x06sha256\x18\x03 \x01(\t\"\xd3\x04\n\x0b\x42undleIndex\x12\x14\n\x0c\x61rchitecture\x18\x01 \x03(\x05\x12.\n\x0b\x61llocations\x18\x02 \x03(\x0b\x32\x19.flare.runtime.Allocation\x12&\n\x07modules\x18\x03 \x03(\x0b\x32\x15.flare.runtime.Module\x12*\n\tfunctions\x18\x04 \x03(\x0b\x32\x17.flare.runtime.Function\x12)\n\x06graphs\x18\x05 \x03(\x0b\x32\x19.flare.runtime.GraphIndex\x12\x30\n\x0creservations\x18\x06 \x03(\x0b\x32\x1a.flare.runtime.Reservation\x12+\n\texecution\x18\x07 \x01(\x0b\x32\x18.flare.runtime.Execution\x12\'\n\x07\x63ontext\x18\x08 \x01(\x0b\x32\x16.flare.runtime.Context\x12<\n\tresources\x18\t \x03(\x0b\x32).flare.runtime.BundleIndex.ResourcesEntry\x12\x0e\n\x06\x65vents\x18\n \x03(\t\x12*\n\x04roce\x18\x0b \x01(\x0b\x32\x1c.flare.runtime.RoceTransport\x12\x36\n\x0frebuild_weights\x18\x0c \x01(\x0b\x32\x1d.flare.runtime.RebuildWeights\x1a\x45\n\x0eResourcesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.flare.runtime.Span:\x02\x38\x01\"\xf8\x01\n\rRoceTransport\x12\x11\n\tproxy_abi\x18\x01 \x01(\r\x12\r\n\x05world\x18\x02 \x01(\r\x12\x0c\n\x04rank\x18\x03 \x01(\r\x12\x16\n\x0eregion_address\x18\x04 \x01(\x04\x12\x14\n\x0cregion_bytes\x18\x05 \x01(\x04\x12\x18\n\x10\x63ounters_address\x18\x06 \x01(\x04\x12\x16\n\x0e\x63ounters_bytes\x18\x07 \x01(\x04\x12\x12\n\nslot_bytes\x18\x08 \x01(\x04\x12\x12\n\npipe_bytes\x18\t \x01(\x04\x12\x13\n\x0b\x63trl_offset\x18\n \x01(\x04\x12\x14\n\x0cvarlen_bytes\x18\x10 \x01(\x04J\x04\x08\x0b\x10\x10\"v\n\nGraphIndex\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x66lags\x18\x02 \x01(\x04\x12\x10\n\x08resource\x18\x03 \x01(\t\x12\x12\n\nnode_count\x18\x04 \x01(\x04\x12\x12\n\nedge_count\x18\x05 \x01(\x04\x12\x11\n\tfunctions\x18\x06 \x03(\x04\"M\n\nPeerMemory\x12\r\n\x05owner\x18\x01 \x01(\r\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\x04\x12\x0e\n\x06nbytes\x18\x03 \x01(\x04\x12\x0f\n\x07readers\x18\x04 \x03(\r\"M\n\x07\x43ontext\x12\r\n\x05\x66lags\x18\x01 \x01(\r\x12\r\n\x05\x63\x61\x63he\x18\x02 \x01(\x05\x12$\n\x06limits\x18\x03 \x03(\x0b\x32\x14.flare.runtime.Limit\"\"\n\x05Limit\x12\n\n\x02id\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\x04\".\n\x0bReservation\x12\x0f\n\x07\x61\x64\x64ress\x18\x01 \x01(\x04\x12\x0e\n\x06nbytes\x18\x02 \x01(\x04\"8\n\x04Span\x12\x0e\n\x06offset\x18\x01 \x01(\x04\x12\x0e\n\x06nbytes\x18\x02 \x01(\x04\x12\x10\n\x08\x66ilename\x18\x03 \x01(\t\"\xa2\x02\n\nAllocation\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06nbytes\x18\x02 \x01(\x04\x12\x11\n\talignment\x18\x03 \x01(\x04\x12\x11\n\x04\x66ile\x18\x04 \x01(\tH\x00\x88\x01\x01\x12\'\n\ndata_spans\x18\x05 \x03(\x0b\x32\x13.flare.runtime.Span\x12.\n\x0brelocations\x18\x06 \x03(\x0b\x32\x19.flare.runtime.Relocation\x12\x14\n\x07\x61\x64\x64ress\x18\x07 \x01(\x04H\x01\x88\x01\x01\x12%\n\x06memory\x18\x08 \x01(\x0e\x32\x15.flare.runtime.Memory\x12%\n\x06sizing\x18\t \x01(\x0b\x32\x15.flare.runtime.SizingB\x07\n\x05_fileB\n\n\x08_address\"X\n\x06Sizing\x12)\n\x08\x63\x61pacity\x18\x01 \x01(\x0e\x32\x17.flare.runtime.Capacity\x12\x12\n\nunit_bytes\x18\x02 \x01(\x04\x12\x0f\n\x07maximum\x18\x03 \x01(\x04\"L\n\x06Module\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x66ile\x18\x02 \x01(\t\x12&\n\x07globals\x18\x03 \x03(\x0b\x32\x15.flare.runtime.Global\"f\n\x06Global\x12\x0e\n\x06symbol\x18\x01 \x01(\t\x12\x0c\n\x04\x66ile\x18\x02 \x01(\t\x12\x0e\n\x06nbytes\x18\x03 \x01(\x04\x12.\n\x0brelocations\x18\x04 \x03(\x0b\x32\x19.flare.runtime.Relocation\"y\n\x08\x46unction\x12\x0e\n\x06module\x18\x01 \x01(\t\x12\x0e\n\x06symbol\x18\x02 \x01(\t\x12\x17\n\x0fparameter_sizes\x18\x03 \x03(\x04\x12\x34\n\nattributes\x18\x04 \x03(\x0b\x32 .flare.runtime.FunctionAttribute\".\n\x11\x46unctionAttribute\x12\n\n\x02id\x18\x01 \x01(\x05\x12\r\n\x05value\x18\x02 \x01(\x05\"/\n\tReference\x12\x12\n\nallocation\x18\x01 \x01(\t\x12\x0e\n\x06offset\x18\x02 \x01(\x04\"s\n\nRelocation\x12\x0e\n\x06offset\x18\x01 \x01(\x04\x12(\n\x06target\x18\x02 \x01(\x0b\x32\x18.flare.runtime.Reference\x12+\n\x04kind\x18\x03 \x01(\x0e\x32\x1d.flare.runtime.RelocationKind\"D\n\x04\x42lob\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12.\n\x0brelocations\x18\x02 \x03(\x0b\x32\x19.flare.runtime.Relocation\";\n\tAttribute\x12\n\n\x02id\x18\x01 \x01(\x05\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.flare.runtime.Blob\"\x9d\x01\n\x06Kernel\x12\x10\n\x08\x66unction\x18\x01 \x01(\x04\x12\x0c\n\x04grid\x18\x02 \x03(\r\x12\r\n\x05\x62lock\x18\x03 \x03(\r\x12\x0e\n\x06shared\x18\x04 \x01(\r\x12&\n\targuments\x18\x05 \x03(\x0b\x32\x13.flare.runtime.Blob\x12,\n\nattributes\x18\x06 \x03(\x0b\x32\x18.flare.runtime.Attribute\"\x8a\x01\n\x06Memset\x12-\n\x0b\x64\x65stination\x18\x01 \x01(\x0b\x32\x18.flare.runtime.Reference\x12\r\n\x05pitch\x18\x02 \x01(\x04\x12\r\n\x05value\x18\x03 \x01(\r\x12\x14\n\x0c\x65lement_size\x18\x04 \x01(\r\x12\r\n\x05width\x18\x05 \x01(\x04\x12\x0e\n\x06height\x18\x06 \x01(\x04\"o\n\x04\x43opy\x12(\n\x06source\x18\x01 \x01(\x0b\x32\x18.flare.runtime.Reference\x12-\n\x0b\x64\x65stination\x18\x02 \x01(\x0b\x32\x18.flare.runtime.Reference\x12\x0e\n\x06nbytes\x18\x03 \x01(\x04\"\x07\n\x05\x45mpty\"\x1a\n\tEventWait\x12\r\n\x05\x65vent\x18\x01 \x01(\t\"\xe1\x01\n\x04Node\x12\'\n\x06kernel\x18\x01 \x01(\x0b\x32\x15.flare.runtime.KernelH\x00\x12\'\n\x06memset\x18\x02 \x01(\x0b\x32\x15.flare.runtime.MemsetH\x00\x12#\n\x04\x63opy\x18\x03 \x01(\x0b\x32\x13.flare.runtime.CopyH\x00\x12%\n\x05\x65mpty\x18\x04 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12.\n\nevent_wait\x18\x05 \x01(\x0b\x32\x18.flare.runtime.EventWaitH\x00\x42\x0b\n\toperation\"9\n\x04\x45\x64ge\x12\x0e\n\x06source\x18\x01 \x01(\x04\x12\x13\n\x0b\x64\x65stination\x18\x02 \x01(\x04\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"\xfc\x01\n\tExecution\x12\x13\n\x0b\x61\x62i_version\x18\x01 \x01(\r\x12\x12\n\nmax_active\x18\x02 \x01(\x04\x12\x19\n\x11input_frame_bytes\x18\x03 \x01(\x04\x12&\n\x07\x62uffers\x18\x04 \x03(\x0b\x32\x15.flare.runtime.Buffer\x12-\n\x06graphs\x18\x05 \x03(\x0b\x32\x1d.flare.runtime.ExecutionGraph\x12\x31\n\rtuning_graphs\x18\x06 \x03(\x0b\x32\x1a.flare.runtime.TuningGraph\x12!\n\x19\x63ompletion_token_capacity\x18\x07 \x01(\r\"C\n\x0bTuningGraph\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07outputs\x18\x02 \x03(\t\x12\x15\n\rstatus_buffer\x18\x03 \x01(\t\"\x82\x01\n\x06\x42uffer\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06nbytes\x18\x02 \x01(\x04\x12\r\n\x05\x64type\x18\x03 \x01(\t\x12\r\n\x05shape\x18\x04 \x03(\x04\x12\x0f\n\x07strides\x18\x05 \x03(\x04\x12+\n\treference\x18\x06 \x01(\x0b\x32\x18.flare.runtime.Reference\"\x95\x04\n\x0e\x45xecutionGraph\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04kind\x18\x02 \x01(\t\x12\x15\n\rsampling_mode\x18\x03 \x01(\t\x12\x16\n\x0e\x62\x61tch_capacity\x18\x04 \x01(\x04\x12\x16\n\x0etoken_capacity\x18\x05 \x01(\x04\x12\x33\n\x0einput_bindings\x18\x06 \x03(\x0b\x32\x1b.flare.runtime.InputBinding\x12\x19\n\x0c\x64raft_tokens\x18\x07 \x01(\rH\x00\x88\x01\x01\x12\x1a\n\rverify_tokens\x18\x08 \x01(\rH\x01\x88\x01\x01\x12\x30\n\x0c\x64raft_source\x18\t \x01(\x0e\x32\x1a.flare.runtime.DraftSource\x12 \n\x13speculation_enabled\x18\n \x01(\x08H\x02\x88\x01\x01\x12\x1d\n\x15\x65stimated_duration_ns\x18\x0b \x01(\x04\x12\x18\n\x10prefill_capacity\x18\x0c \x01(\x04\x12\x1d\n\x15packed_token_capacity\x18\r \x01(\x04\x12\x18\n\x10\x63onfidence_width\x18\x0e \x01(\r\x12\x33\n\x0etuned_bindings\x18\x0f \x03(\x0b\x32\x1b.flare.runtime.TunedBindingB\x0f\n\r_draft_tokensB\x10\n\x0e_verify_tokensB\x16\n\x14_speculation_enabled\"H\n\x0cInputBinding\x12\x13\n\x0b\x62uffer_name\x18\x01 \x01(\t\x12\x13\n\x0bhost_offset\x18\x02 \x01(\x04\x12\x0e\n\x06nbytes\x18\x03 \x01(\x04*\x8e\x01\n\x0bHistoryKind\x12\x17\n\x13HISTORY_UNSPECIFIED\x10\x00\x12\x13\n\x0fPAGED_KEY_VALUE\x10\x01\x12\x10\n\x0cPAGED_LATENT\x10\x02\x12\x12\n\x0eSLIDING_WINDOW\x10\x03\x12\x13\n\x0fRECURRENT_STATE\x10\x04\x12\x16\n\x12\x43ONTINUATION_STATE\x10\x05*\xfa\x06\n\x0bValueSource\x12\x15\n\x11VALUE_UNSPECIFIED\x10\x00\x12\x0c\n\x08\x43ONSTANT\x10\x01\x12\x10\n\x0cREQUEST_SLOT\x10\x02\x12\x16\n\x12REQUEST_GENERATION\x10\x03\x12\r\n\tLIVE_ROWS\x10\x04\x12\x10\n\x0cQUERY_LENGTH\x10\x05\x12\x12\n\x0eSAMPLE_ENABLED\x10\x06\x12\x10\n\x0cPROMPT_TOKEN\x10\x07\x12\x11\n\rPENDING_TOKEN\x10\x08\x12\x1c\n\x18PROMPT_AND_PENDING_TOKEN\x10\t\x12\x13\n\x0fPROMPT_POSITION\x10\n\x12\x17\n\x13IMAGE_FEATURE_INDEX\x10\x0b\x12\x16\n\x12IMAGE_LEFT_CONTEXT\x10\x0c\x12\x17\n\x13IMAGE_RIGHT_CONTEXT\x10\r\x12\x0e\n\nPAGE_TABLE\x10\x0e\x12\x13\n\x0fLIVE_STATE_ROWS\x10\x0f\x12\x12\n\x0eLIVE_STATE_ROW\x10\x10\x12\x14\n\x10PREFIX_STATE_ROW\x10\x11\x12\x16\n\x12SNAPSHOT_STATE_ROW\x10\x12\x12\x13\n\x0f\x43ONSUMED_TOKENS\x10\x13\x12\x1c\n\x18SNAPSHOT_CONSUMED_TOKENS\x10\x14\x12\x0f\n\x0bPREFILL_ROW\x10\x15\x12\x11\n\rSAMPLING_SEED\x10\x16\x12\x17\n\x13SAMPLING_PARAMETERS\x10\x17\x12\x12\n\x0eSAMPLING_TOP_K\x10\x18\x12\x12\n\x0eMAX_NEW_TOKENS\x10\x19\x12\x11\n\rEOS_TOKEN_IDS\x10\x1a\x12\r\n\tEOS_COUNT\x10\x1b\x12\x14\n\x10PROMPT_HISTOGRAM\x10\x1c\x12\x10\n\x0cGRAMMAR_MASK\x10\x1d\x12\x13\n\x0fGRAMMAR_ENABLED\x10\x1e\x12\x12\n\x0e\x43OPY_TOKEN_IDS\x10\x1f\x12\x14\n\x10\x43OPY_TOKEN_COUNT\x10 \x12\x18\n\x14\x43OPY_CONSUMED_TOKENS\x10!\x12\x13\n\x0f\x43OPY_GENERATION\x10\"\x12\x15\n\x11\x43OPY_GRAMMAR_MASK\x10#\x12\x16\n\x12\x43OPY_GRAMMAR_COUNT\x10$\x12\x10\n\x0cIMAGE_TENSOR\x10%\x12\x16\n\x12IMAGE_FEATURE_SLOT\x10&\x12\x17\n\x13PROMPT_TARGET_TOKEN\x10\'\x12\x1c\n\x18SPECULATIVE_PREFIX_LIMIT\x10(*|\n\rInputIndexing\x12\x18\n\x14INDEXING_UNSPECIFIED\x10\x00\x12\n\n\x06SCALAR\x10\x01\x12\x12\n\x0eSUBMISSION_ROW\x10\x02\x12\x0f\n\x0bREQUEST_ROW\x10\x03\x12\x10\n\x0cPACKED_TOKEN\x10\x04\x12\x0e\n\nACTIVE_ROW\x10\x05*T\n\x0fStateAccessKind\x12\x1c\n\x18STATE_ACCESS_UNSPECIFIED\x10\x00\x12\x08\n\x04READ\x10\x01\x12\t\n\x05WRITE\x10\x02\x12\x0e\n\nREAD_WRITE\x10\x03*\xf5\x01\n\x0eGraphOperation\x12\x1f\n\x1bGRAPH_OPERATION_UNSPECIFIED\x10\x00\x12\x08\n\x04INIT\x10\x01\x12\x0b\n\x07RESTORE\x10\x02\x12\x0c\n\x08SNAPSHOT\x10\x03\x12\x0b\n\x07PREFILL\x10\x04\x12\n\n\x06\x44\x45\x43ODE\x10\x05\x12\n\n\x06RESUME\x10\x06\x12\n\n\x06VISION\x10\x07\x12\x0f\n\x0bSPECULATIVE\x10\x08\x12\t\n\x05MIXED\x10\t\x12\x12\n\x0eSCORED_PREFILL\x10\x0c\"\x04\x08\n\x10\n\"\x04\x08\x0b\x10\x0b\"\x04\x08\r\x10\r*\x08PROPOSAL*\x0cVERIFICATION*\x12PACKED_SPECULATIVE*\xa1\x01\n\x0eToolParserKind\x12\x1b\n\x17TOOL_PARSER_UNSPECIFIED\x10\x00\x12\x0e\n\nQWEN_CODER\x10\x01\x12\x15\n\x11GLM_ARGUMENT_TAGS\x10\x02\x12\x08\n\x04\x44SML\x10\x03\x12\x08\n\x04XTML\x10\x04\x12\x0f\n\x0bMINIMAX_XML\x10\x05\x12\x0e\n\nGEMMA_CALL\x10\x06\x12\x08\n\x04\x41TEM\x10\x07\x12\x0c\n\x08TML_JSON\x10\x08*\xa6\x01\n\x13ReasoningParserKind\x12 \n\x1cREASONING_PARSER_UNSPECIFIED\x10\x00\x12\x0e\n\nTHINK_TAGS\x10\x01\x12\x0e\n\nXTML_THINK\x10\x02\x12\x0c\n\x08MM_THINK\x10\x03\x12\x13\n\x0f\x43HANNEL_THOUGHT\x10\x04\x12\x15\n\x11RECIPIENT_HEADERS\x10\x05\x12\x13\n\x0f\x43ONTENT_HEADERS\x10\x06*\xf4\x02\n\x14ImagePreparationKind\x12!\n\x1dIMAGE_PREPARATION_UNSPECIFIED\x10\x00\x12\x1a\n\x16QWEN_IMAGE_PREPARATION\x10\x01\x12\x1a\n\x16MIMO_IMAGE_PREPARATION\x10\x02\x12\x1e\n\x1a\x44\x45\x45PSEEK_IMAGE_PREPARATION\x10\x03\x12\x1d\n\x19MOONVIT_IMAGE_PREPARATION\x10\x04\x12\x1a\n\x16STEP_IMAGE_PREPARATION\x10\x05\x12\x1b\n\x17RADIO_IMAGE_PREPARATION\x10\x06\x12(\n$DEEPSEEK_ROW_MAJOR_IMAGE_PREPARATION\x10\x07\x12\x19\n\x15GLM_IMAGE_PREPARATION\x10\x08\x12!\n\x1dPATCH_MERGE_IMAGE_PREPARATION\x10\t\x12!\n\x1dPADDED_GRID_IMAGE_PREPARATION\x10\n*\xb6\x02\n\x0fImageLayoutKind\x12\x1c\n\x18IMAGE_LAYOUT_UNSPECIFIED\x10\x00\x12\x15\n\x11QWEN_IMAGE_LAYOUT\x10\x01\x12\x15\n\x11MIMO_IMAGE_LAYOUT\x10\x02\x12\x19\n\x15\x44\x45\x45PSEEK_IMAGE_LAYOUT\x10\x03\x12\x18\n\x14MOONVIT_IMAGE_LAYOUT\x10\x04\x12\x15\n\x11STEP_IMAGE_LAYOUT\x10\x05\x12\x16\n\x12RADIO_IMAGE_LAYOUT\x10\x06\x12#\n\x1f\x44\x45\x45PSEEK_ROW_MAJOR_IMAGE_LAYOUT\x10\x07\x12\x14\n\x10GLM_IMAGE_LAYOUT\x10\x08\x12\x1a\n\x16\x42RACKETED_IMAGE_LAYOUT\x10\t\x12\x1c\n\x18\x43LASS_TOKEN_IMAGE_LAYOUT\x10\n*P\n\rPromptEncoder\x12\x10\n\x0cPROMPT_JINJA\x10\x00\x12\x0f\n\x0b\x44\x45\x45PSEEK_V4\x10\x01\x12\x10\n\x0c\x44\x45\x45PSEEK_V41\x10\x02\x12\n\n\x06TML_V0\x10\x03*B\n\x10TemplateEncoding\x12\x11\n\rTEMPLATE_TEXT\x10\x00\x12\x1b\n\x17TEMPLATE_TOKEN_SEGMENTS\x10\x01*/\n\x06Memory\x12\n\n\x06\x44\x45VICE\x10\x00\x12\x0f\n\x0bMAPPED_HOST\x10\x01\x12\x08\n\x04PEER\x10\x02*(\n\x08\x43\x61pacity\x12\x0e\n\nSTATE_ROWS\x10\x00\x12\x0c\n\x08KV_PAGES\x10\x01*-\n\x0eRelocationKind\x12\x0b\n\x07POINTER\x10\x00\x12\x0e\n\nTENSOR_MAP\x10\x01*4\n\x0b\x44raftSource\x12\x0c\n\x08NO_DRAFT\x10\x00\x12\x07\n\x03MTP\x10\x01\x12\x0e\n\nCOPY_DRAFT\x10\x02\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12runtime_core.proto\x12\rflare.runtime\"\x99\x01\n\x0eProtectedIndex\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x0f\n\x07\x63ore_id\x18\x02 \x01(\x0c\x12\x0e\n\x06signer\x18\x03 \x01(\x0c\x12*\n\x06groups\x18\x04 \x03(\x0b\x32\x1a.flare.runtime.SignedGroup\x12)\n\x05ranks\x18\x05 \x03(\x0b\x32\x1a.flare.runtime.SignedGroup\"<\n\x0bSignedGroup\x12\x0c\n\x04kind\x18\x01 \x01(\r\x12\x0c\n\x04\x62ody\x18\x02 \x01(\x0c\x12\x11\n\tsignature\x18\x03 \x01(\x0c\"\xcc\x01\n\x11PublisherMetadata\x12(\n\x05index\x18\x01 \x01(\x0b\x32\x19.flare.runtime.ModelIndex\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x17\n\x0fofficial_source\x18\x03 \x01(\t\x12\x14\n\x0cgroup_hashes\x18\x04 \x03(\x0c\x12\x13\n\x0b\x64\x65velopment\x18\x05 \x01(\x08\x12\r\n\x05ranks\x18\x06 \x01(\r\x12\'\n\x07license\x18\x07 \x01(\x0b\x32\x16.flare.runtime.License\"%\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04link\x18\x02 \x01(\t\"F\n\x0fResourceCatalog\x12\x33\n\tresources\x18\x01 \x03(\x0b\x32 .flare.runtime.ProtectedResource\"^\n\x11ProtectedResource\x12\x0e\n\x06offset\x18\x01 \x01(\x04\x12\x14\n\x0cstored_bytes\x18\x02 \x01(\x04\x12\x13\n\x0bplain_bytes\x18\x03 \x01(\x04\x12\x0e\n\x06sha256\x18\x04 \x03(\x0c\"`\n\nStoredData\x12\x0e\n\x06nbytes\x18\x01 \x01(\x04\x12\x0e\n\x06sha256\x18\x02 \x01(\x0c\x12\x0e\n\x06offset\x18\x03 \x01(\x04\x12\"\n\x05parts\x18\x04 \x03(\x0b\x32\x13.flare.runtime.Span\"\xd9\x01\n\x0eRankProtection\x12\x0c\n\x04rank\x18\x01 \x01(\r\x12*\n\x06\x62undle\x18\x02 \x01(\x0b\x32\x1a.flare.runtime.BundleIndex\x12\x33\n\x0b\x65xecutables\x18\x03 \x01(\x0b\x32\x1e.flare.runtime.ResourceCatalog\x12/\n\x07support\x18\x04 \x01(\x0b\x32\x1e.flare.runtime.ResourceCatalog\x12\'\n\x04\x64\x61ta\x18\x05 \x01(\x0b\x32\x19.flare.runtime.StoredData\"\xcf\x03\n\nModelIndex\x12\x0f\n\x07version\x18\x01 \x01(\r\x12)\n\x05ranks\x18\x02 \x03(\x0b\x32\x1a.flare.runtime.BundleIndex\x12.\n\x0bpeer_memory\x18\x03 \x03(\x0b\x32\x19.flare.runtime.PeerMemory\x12$\n\x06\x61ssets\x18\x05 \x03(\x0b\x32\x14.flare.runtime.Asset\x12%\n\x06\x65ngram\x18\x06 \x01(\x0b\x32\x15.flare.runtime.Engram\x12\x43\n\x11sampling_defaults\x18\x07 \x01(\x0b\x32(.flare.runtime.DefaultSamplingParameters\x12\x31\n\x08\x66rontend\x18\x08 \x01(\x0b\x32\x1f.flare.runtime.FrontendSettings\x12\'\n\x07\x63luster\x18\t \x01(\x0b\x32\x16.flare.runtime.Cluster\x12\x30\n\x08\x63ontract\x18\n \x01(\x0b\x32\x1e.flare.runtime.ServingContract\x12/\n\x07rebuild\x18\x0b \x01(\x0b\x32\x1e.flare.runtime.RebuildManifestJ\x04\x08\x04\x10\x05\"\xd5\x06\n\x0fRebuildManifest\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x0e\n\x06recipe\x18\x02 \x01(\t\x12\x17\n\x0fsource_identity\x18\x03 \x01(\t\x12\x1d\n\x15\x63heckpoint_repository\x18\x04 \x01(\t\x12\x1b\n\x13\x63heckpoint_revision\x18\x05 \x01(\t\x12\x17\n\x0ftensor_parallel\x18\x06 \x01(\r\x12\x17\n\x0fmax_concurrency\x18\x07 \x01(\x04\x12\x15\n\rprefill_chunk\x18\x08 \x01(\x04\x12\x19\n\x11image_concurrency\x18\t \x01(\x04\x12\x18\n\x10\x63ontext_parallel\x18\n \x01(\x08\x12\x16\n\x0estream_overlap\x18\x0b \x01(\x08\x12\x15\n\rmixed_prefill\x18\x0c \x01(\x08\x12%\n\x18\x65xpert_scale_compression\x18\r \x01(\tH\x00\x88\x01\x01\x12\x18\n\x10mla_compute_mode\x18\x0e \x01(\t\x12\x11\n\troce_ring\x18\x0f \x01(\x08\x12\x14\n\x0conline_mxfp8\x18\x10 \x01(\x08\x12\x1b\n\x13language_model_only\x18\x11 \x01(\x08\x12\x36\n\x0bspeculation\x18\x12 \x01(\x0b\x32!.flare.runtime.RebuildSpeculation\x12\x19\n\x11\x63ompiled_forwards\x18\x13 \x01(\x08\x12\x15\n\rreserve_bytes\x18\x14 \x01(\x04\x12\x18\n\x10local_checkpoint\x18\x15 \x01(\x08\x12\x12\n\nroce_chain\x18\x16 \x01(\x08\x12\x14\n\x0cjoint_tuning\x18\x17 \x01(\x08\x12\x1e\n\x16shard_hyperconnections\x18\x18 \x01(\x08\x12\x19\n\x11\x63ollective_tuning\x18\x19 \x01(\t\x12\x1e\n\x16\x63ollective_ladder_json\x18\x1a \x01(\t\x12\x1c\n\x14\x66orce_moe_decode_a16\x18\x1b \x01(\x08\x12\x19\n\x11row_shard_prefill\x18\x1c \x01(\x08\x12\x1a\n\x12online_mxfp8_scope\x18\x1e \x01(\t\x12\x1d\n\x15\x63ontext_parallel_size\x18\x1f \x01(\rB\x1b\n\x19_expert_scale_compression\"\xfe\x03\n\x12RebuildSpeculation\x12\x14\n\x0c\x64raft_tokens\x18\x01 \x01(\r\x12\x10\n\x08\x61\x64\x61ptive\x18\x02 \x01(\x08\x12\x17\n\x0fproposal_source\x18\x03 \x01(\t\x12\x16\n\x0ehistory_drafts\x18\x04 \x01(\x08\x12\x16\n\x0etoken_capacity\x18\x05 \x01(\x04\x12\x1b\n\x13packed_verification\x18\x06 \x01(\x08\x12\x1e\n\x16\x64raft_prefix_selection\x18\x07 \x01(\t\x12$\n\x17\x64raft_head_quantization\x18\x08 \x01(\tH\x00\x88\x01\x01\x12%\n\x18markov_head_quantization\x18\t \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0cgraph_budget\x18\n \x01(\x04H\x02\x88\x01\x01\x12$\n\x1cvocabulary_parallel_sampling\x18\x0b \x01(\x08\x12\x0e\n\x06method\x18\x0c \x01(\t\x12\x10\n\x08mtp_only\x18\r \x01(\x08\x12$\n\x17\x64raft_body_quantization\x18\x0e \x01(\tH\x03\x88\x01\x01\x42\x1a\n\x18_draft_head_quantizationB\x1b\n\x19_markov_head_quantizationB\x0f\n\r_graph_budgetB\x1a\n\x18_draft_body_quantization\"e\n\x0eRebuildWeights\x12\x0f\n\x07version\x18\x01 \x01(\r\x12-\n\x07tensors\x18\x02 \x03(\x0b\x32\x1c.flare.runtime.RebuildTensor\x12\x13\n\x0bunavailable\x18\x03 \x03(\t\"\xa2\x01\n\rRebuildTensor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x64type\x18\x02 \x01(\t\x12\x15\n\relement_bytes\x18\x03 \x01(\r\x12\r\n\x05shape\x18\x04 \x03(\x04\x12\x0e\n\x06stride\x18\x05 \x03(\x04\x12\x13\n\x0blayout_type\x18\x06 \x01(\t\x12)\n\x07storage\x18\x07 \x01(\x0b\x32\x18.flare.runtime.Reference\"\xcb\x06\n\x0fServingContract\x12\x10\n\x08model_id\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x34\n\x0emodel_contract\x18\x03 \x01(\x0b\x32\x1c.flare.runtime.ModelContract\x12\x1b\n\x13max_sequence_tokens\x18\x04 \x01(\x04\x12\x1a\n\x12prefill_chunk_size\x18\x05 \x01(\x04\x12\r\n\x05pages\x18\x06 \x01(\x04\x12\x12\n\nstate_rows\x18\x07 \x01(\x04\x12\x11\n\tsnapshots\x18\x08 \x01(\x04\x12\x13\n\x0bpage_tokens\x18\t \x01(\x04\x12\x1a\n\x12speculative_widths\x18\x0b \x03(\x04\x12$\n\x17speculative_fixed_width\x18\x0c \x01(\x04H\x00\x88\x01\x01\x12\x16\n\x0ehistory_drafts\x18\r \x01(\x08\x12#\n\x1bspeculative_proposal_source\x18\x0e \x01(\t\x12\x1b\n\x0emax_new_tokens\x18\x0f \x01(\x04H\x01\x88\x01\x01\x12\x17\n\nmax_active\x18\x10 \x01(\x04H\x02\x88\x01\x01\x12\x14\n\x0cserved_names\x18\x11 \x03(\t\x12\x1a\n\rreserve_bytes\x18\x12 \x01(\x04H\x03\x88\x01\x01\x12\x1b\n\x0equeue_capacity\x18\x13 \x01(\x04H\x04\x88\x01\x01\x12\x1d\n\x10\x66rontend_workers\x18\x14 \x01(\x04H\x05\x88\x01\x01\x12\x1b\n\x0emax_body_bytes\x18\x15 \x01(\x04H\x06\x88\x01\x01\x12\x14\n\x0c\x63ontext_hint\x18\x16 \x01(\t\x12\x15\n\rcapacity_hint\x18\x17 \x01(\t\x12\"\n\x1aspeculative_token_capacity\x18\x18 \x01(\x04\x12 \n\x13\x64\x65\x63ode_weight_bytes\x18\x19 \x01(\x04H\x07\x88\x01\x01\x42\x1a\n\x18_speculative_fixed_widthB\x11\n\x0f_max_new_tokensB\r\n\x0b_max_activeB\x10\n\x0e_reserve_bytesB\x11\n\x0f_queue_capacityB\x13\n\x11_frontend_workersB\x11\n\x0f_max_body_bytesB\x16\n\x14_decode_weight_bytesJ\x04\x08\n\x10\x0b\"\x8b\x02\n\rModelContract\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x12\n\nvocabulary\x18\x02 \x01(\x04\x12\x18\n\x10input_vocabulary\x18\x03 \x01(\x04\x12-\n\x06vision\x18\x04 \x01(\x0b\x32\x1d.flare.runtime.VisionCapacity\x12+\n\x05\x63\x61\x63he\x18\x05 \x01(\x0b\x32\x1c.flare.runtime.CacheContract\x12,\n\x06graphs\x18\x06 \x03(\x0b\x32\x1c.flare.runtime.GraphContract\x12\x31\n\x08\x66rontend\x18\x07 \x01(\x0b\x32\x1f.flare.runtime.FrontendContract\"\x90\x01\n\x0eVisionCapacity\x12\x15\n\rfeature_slots\x18\x05 \x01(\x04\x12\x1c\n\x14\x66\x65\x61tures_per_request\x18\x01 \x01(\x04\x12\x12\n\nmax_images\x18\x02 \x01(\x04\x12\x1b\n\x13patches_per_feature\x18\x03 \x01(\x04\x12\x18\n\x10patch_capacities\x18\x04 \x03(\x04\"\x86\x01\n\rCacheContract\x12\x13\n\x0bpage_tokens\x18\x01 \x01(\x04\x12-\n\x07regions\x18\x02 \x03(\x0b\x32\x1c.flare.runtime.HistoryRegion\x12\x31\n\x0bpage_tables\x18\x03 \x03(\x0b\x32\x1c.flare.runtime.PageTableView\"\x96\x01\n\rHistoryRegion\x12(\n\x04kind\x18\x01 \x01(\x0e\x32\x1a.flare.runtime.HistoryKind\x12)\n\x07storage\x18\x02 \x01(\x0b\x32\x18.flare.runtime.StateView\x12\x19\n\x11tokens_per_record\x18\x03 \x01(\x04\x12\x15\n\rwindow_tokens\x18\x04 \x01(\x04\"4\n\rPageTableView\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\t\x12\x13\n\x0bpage_offset\x18\x02 \x01(\x04\"\xb1\x01\n\x0cValueBinding\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\t\x12*\n\x06source\x18\x02 \x01(\x0e\x32\x1a.flare.runtime.ValueSource\x12.\n\x08indexing\x18\x03 \x01(\x0e\x32\x1c.flare.runtime.InputIndexing\x12\x14\n\x0cvalue_offset\x18\x04 \x01(\x12\x12\x0f\n\x07padding\x18\x05 \x01(\x12\x12\x0e\n\x06tensor\x18\x06 \x01(\t\"K\n\tStateView\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\t\x12\x0e\n\x06offset\x18\x02 \x01(\x04\x12\x0e\n\x06stride\x18\x03 \x01(\x04\x12\x0e\n\x06nbytes\x18\x04 \x01(\x04\"e\n\x0bStateAccess\x12&\n\x04view\x18\x01 \x01(\x0b\x32\x18.flare.runtime.StateView\x12.\n\x06\x61\x63\x63\x65ss\x18\x02 \x01(\x0e\x32\x1e.flare.runtime.StateAccessKind\"O\n\x10\x43ompletionCursor\x12\'\n\x05state\x18\x01 \x01(\x0b\x32\x18.flare.runtime.StateView\x12\x12\n\nlag_tokens\x18\x02 \x01(\x04\"4\n\x0cTunedBinding\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\t\x12\x14\n\x0ctuning_graph\x18\x02 \x01(\t\"\xfa\x02\n\rGraphContract\x12\r\n\x05graph\x18\x01 \x01(\t\x12+\n\x06inputs\x18\x02 \x03(\x0b\x32\x1b.flare.runtime.ValueBinding\x12)\n\x05state\x18\x03 \x03(\x0b\x32\x1a.flare.runtime.StateAccess\x12\x1e\n\x16state_rows_per_request\x18\x04 \x01(\x04\x12\x35\n\x0c\x64raft_cursor\x18\x05 \x01(\x0b\x32\x1f.flare.runtime.CompletionCursor\x12\x18\n\x10\x65mbedding_tables\x18\x06 \x03(\r\x12/\n\rpending_token\x18\x07 \x01(\x0b\x32\x18.flare.runtime.StateView\x12+\n\x04kind\x18\x08 \x01(\x0e\x32\x1d.flare.runtime.GraphOperation\x12\x33\n\x0etuned_bindings\x18\t \x03(\x0b\x32\x1b.flare.runtime.TunedBinding\"k\n\nToolParser\x12+\n\x04kind\x18\x01 \x01(\x0e\x32\x1d.flare.runtime.ToolParserKind\x12\x19\n\x11wrapping_newlines\x18\x02 \x01(\r\x12\x15\n\rtag_separator\x18\x03 \x01(\t\"\x95\x01\n\x0fReasoningParser\x12\x30\n\x04kind\x18\x01 \x01(\x0e\x32\".flare.runtime.ReasoningParserKind\x12\x0c\n\x04open\x18\x02 \x01(\t\x12\r\n\x05\x63lose\x18\x03 \x01(\t\x12\x19\n\x11wrapping_newlines\x18\x04 \x01(\r\x12\x18\n\x10terminal_markers\x18\x05 \x03(\t\"\xca\x01\n\x0ePrefixBoundary\x12\r\n\x05roles\x18\x01 \x03(\t\x12\x18\n\x0b\x61ppend_role\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x1e\n\x16\x63ontinue_final_message\x18\x03 \x01(\x08\x12\x18\n\x10minimum_messages\x18\x04 \x01(\x04\x12\x16\n\x0erequired_start\x18\x05 \x01(\t\x12\x14\n\x0crequired_end\x18\x06 \x01(\t\x12\x17\n\x0frequired_marker\x18\x07 \x01(\tB\x0e\n\x0c_append_role\"y\n\tImageSpan\x12\x13\n\x0bstart_token\x18\x01 \x01(\r\x12\x11\n\tend_token\x18\x02 \x01(\r\x12\x12\n\nmax_tokens\x18\x03 \x01(\x04\x12\x17\n\x0finterior_tokens\x18\x04 \x03(\r\x12\x17\n\x0funscoped_tokens\x18\x05 \x03(\r\"Q\n\x10ImageTokenLayout\x12\x15\n\rfeature_token\x18\x01 \x01(\r\x12&\n\x04span\x18\x02 \x01(\x0b\x32\x18.flare.runtime.ImageSpan\"\xf2\x03\n\x10\x46rontendContract\x12(\n\x05tools\x18\x01 \x01(\x0b\x32\x19.flare.runtime.ToolParser\x12\x31\n\treasoning\x18\x02 \x01(\x0b\x32\x1e.flare.runtime.ReasoningParser\x12\x43\n\x11image_preparation\x18\x03 \x01(\x0e\x32#.flare.runtime.ImagePreparationKindH\x00\x88\x01\x01\x12\x39\n\x0cimage_layout\x18\x04 \x01(\x0e\x32\x1e.flare.runtime.ImageLayoutKindH\x01\x88\x01\x01\x12\x17\n\x0frejected_tokens\x18\x05 \x03(\r\x12\x1b\n\x13position_dimensions\x18\x06 \x01(\r\x12\x18\n\x10\x61ssistant_marker\x18\x07 \x01(\t\x12\x1b\n\x13message_end_markers\x18\x08 \x03(\t\x12\x36\n\x0fprefix_boundary\x18\t \x01(\x0b\x32\x1d.flare.runtime.PrefixBoundary\x12\x35\n\x0cimage_tokens\x18\n \x01(\x0b\x32\x1f.flare.runtime.ImageTokenLayoutB\x14\n\x12_image_preparationB\x0f\n\r_image_layout\"1\n\x07\x43luster\x12\r\n\x05world\x18\x01 \x01(\r\x12\x11\n\texport_id\x18\x03 \x01(\tJ\x04\x08\x02\x10\x03\"\x8c\x03\n\x19\x44\x65\x66\x61ultSamplingParameters\x12\x11\n\x04seed\x18\x01 \x01(\x04H\x00\x88\x01\x01\x12\x18\n\x0btemperature\x18\x02 \x01(\x02H\x01\x88\x01\x01\x12\x12\n\x05top_k\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05top_p\x18\x04 \x01(\x02H\x03\x88\x01\x01\x12\x12\n\x05min_p\x18\x05 \x01(\x02H\x04\x88\x01\x01\x12\x1f\n\x12repetition_penalty\x18\x06 \x01(\x02H\x05\x88\x01\x01\x12\x1d\n\x10presence_penalty\x18\x07 \x01(\x02H\x06\x88\x01\x01\x12\x1e\n\x11\x66requency_penalty\x18\x08 \x01(\x02H\x07\x88\x01\x01\x12-\n\x06ranges\x18\t \x01(\x0b\x32\x1d.flare.runtime.SamplingRangesB\x07\n\x05_seedB\x0e\n\x0c_temperatureB\x08\n\x06_top_kB\x08\n\x06_top_pB\x08\n\x06_min_pB\x15\n\x13_repetition_penaltyB\x13\n\x11_presence_penaltyB\x14\n\x12_frequency_penalty\"H\n\x12SamplingFloatRange\x12\x10\n\x03min\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12\x10\n\x03max\x18\x02 \x01(\x02H\x01\x88\x01\x01\x42\x06\n\x04_minB\x06\n\x04_max\"F\n\x10SamplingIntRange\x12\x10\n\x03min\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x10\n\x03max\x18\x02 \x01(\x05H\x01\x88\x01\x01\x42\x06\n\x04_minB\x06\n\x04_max\"\x96\x03\n\x0eSamplingRanges\x12\x36\n\x0btemperature\x18\x01 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12.\n\x05top_k\x18\x02 \x01(\x0b\x32\x1f.flare.runtime.SamplingIntRange\x12\x30\n\x05top_p\x18\x03 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12\x30\n\x05min_p\x18\x04 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12=\n\x12repetition_penalty\x18\x05 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12;\n\x10presence_penalty\x18\x06 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12<\n\x11\x66requency_penalty\x18\x07 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\"\xc5\x02\n\x10\x46rontendSettings\x12\x33\n\ttokenizer\x18\x01 \x01(\x0b\x32 .flare.runtime.TokenizerSettings\x12+\n\x05image\x18\x02 \x01(\x0b\x32\x1c.flare.runtime.ImageSettings\x12\x15\n\reos_token_ids\x18\x03 \x03(\r\x12:\n\x11template_encoding\x18\x04 \x01(\x0e\x32\x1f.flare.runtime.TemplateEncoding\x12\x34\n\x0eprompt_encoder\x18\x05 \x01(\x0e\x32\x1c.flare.runtime.PromptEncoder\x12\'\n\x1aimage_placeholder_token_id\x18\x06 \x01(\rH\x00\x88\x01\x01\x42\x1d\n\x1b_image_placeholder_token_id\"\xdc\x02\n\x11TokenizerSettings\x12\x16\n\tbos_token\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\teos_token\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tunk_token\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsep_token\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x16\n\tpad_token\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x16\n\tcls_token\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\nmask_token\x18\x07 \x01(\tH\x06\x88\x01\x01\x12;\n\x19\x61\x64\x64itional_special_tokens\x18\x08 \x01(\x0b\x32\x18.flare.runtime.TokenListB\x0c\n\n_bos_tokenB\x0c\n\n_eos_tokenB\x0c\n\n_unk_tokenB\x0c\n\n_sep_tokenB\x0c\n\n_pad_tokenB\x0c\n\n_cls_tokenB\r\n\x0b_mask_token\"\x1b\n\tTokenList\x12\x0e\n\x06values\x18\x01 \x03(\t\"\xfc\x03\n\rImageSettings\x12\x30\n\x04qwen\x18\x01 \x01(\x0b\x32 .flare.runtime.QwenImageSettingsH\x00\x12\x30\n\x04mimo\x18\x02 \x01(\x0b\x32 .flare.runtime.MimoImageSettingsH\x00\x12)\n\tdeepseek4\x18\x03 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12\'\n\x07moonvit\x18\x04 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12$\n\x04step\x18\x05 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12%\n\x05radio\x18\x06 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12J\n\x12\x64\x65\x65pseek_row_major\x18\x07 \x01(\x0b\x32,.flare.runtime.DeepseekRowMajorImageSettingsH\x00\x12#\n\x03glm\x18\x08 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12=\n\x0bpatch_merge\x18\t \x01(\x0b\x32&.flare.runtime.PatchMergeImageSettingsH\x00\x12+\n\x0bpadded_grid\x18\n \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x42\t\n\x07profile\"\xb9\x01\n\x17PatchMergeImageSettings\x12\x12\n\npatch_size\x18\x01 \x01(\r\x12\x12\n\nmin_pixels\x18\x02 \x01(\x04\x12\x12\n\nmax_pixels\x18\x03 \x01(\x04\x12\x12\n\nimage_mean\x18\x04 \x03(\x02\x12\x11\n\timage_std\x18\x05 \x03(\x02\x12\x13\n\x0bimage_token\x18\x06 \x01(\t\x12\x13\n\x0bstart_token\x18\x07 \x01(\t\x12\x11\n\tend_token\x18\x08 \x01(\t\"\x87\x01\n\x1d\x44\x65\x65pseekRowMajorImageSettings\x12\x12\n\nmin_pixels\x18\x01 \x01(\x04\x12\x12\n\nmax_tokens\x18\x02 \x01(\x04\x12#\n\x16max_width_height_ratio\x18\x03 \x01(\x04H\x00\x88\x01\x01\x42\x19\n\x17_max_width_height_ratio\"\x82\x01\n\x11QwenImageSettings\x12\x12\n\nmin_pixels\x18\x01 \x01(\x04\x12\x12\n\nmax_pixels\x18\x02 \x01(\x04\x12\x15\n\rposition_side\x18\x03 \x01(\r\x12\x16\n\x0eimage_token_id\x18\x04 \x01(\r\x12\x16\n\x0evideo_token_id\x18\x05 \x01(\r\"b\n\x11MimoImageSettings\x12\x12\n\nmin_pixels\x18\x01 \x01(\x04\x12\x12\n\nmax_pixels\x18\x02 \x01(\x04\x12\x12\n\nimage_mean\x18\x03 \x03(\x02\x12\x11\n\timage_std\x18\x04 \x03(\x02\"|\n\x06\x45ngram\x12\x11\n\ttoken_map\x18\x01 \x03(\r\x12\x14\n\x0cpad_token_id\x18\x02 \x01(\r\x12\x1d\n\x15\x63ompressed_vocabulary\x18\x03 \x01(\r\x12*\n\x06tables\x18\x04 \x03(\x0b\x32\x1a.flare.runtime.EngramTable\"\x80\x04\n\x0b\x45ngramTable\x12\r\n\x05layer\x18\x01 \x01(\r\x12\x0c\n\x04rows\x18\x02 \x01(\x04\x12\x17\n\x0f\x65mbedding_width\x18\x03 \x01(\r\x12\x13\n\x0bscale_block\x18\x04 \x01(\r\x12$\n\x07weights\x18\x05 \x01(\x0b\x32\x13.flare.runtime.Span\x12#\n\x06scales\x18\x06 \x01(\x0b\x32\x13.flare.runtime.Span\x12\'\n\x04hash\x18\x07 \x01(\x0b\x32\x19.flare.runtime.EngramHash\x12\x15\n\rweight_buffer\x18\x08 \x01(\t\x12\x14\n\x0cscale_buffer\x18\t \x01(\t\x12\x13\n\x0bhash_buffer\x18\n \x01(\t\x12\x13\n\x0bready_event\x18\x0b \x01(\t\x12\x35\n\x08\x65ncoding\x18\x0c \x01(\x0e\x32#.flare.runtime.EngramTable.Encoding\x12\x16\n\x0erequest_buffer\x18\r \x01(\t\x12\x14\n\x0cglobal_scale\x18\x0e \x01(\x02\x12/\n\x06shards\x18\x0f \x03(\x0b\x32\x1f.flare.runtime.EngramTableShard\"E\n\x08\x45ncoding\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x11\n\rFP8_E4M3_E8M0\x10\x01\x12\x15\n\x11NVFP4_E4M3_GLOBAL\x10\x02\"\x8c\x01\n\x10\x45ngramTableShard\x12\x0c\n\x04rank\x18\x01 \x01(\r\x12\x11\n\tfirst_row\x18\x02 \x01(\x04\x12\x0c\n\x04rows\x18\x03 \x01(\x04\x12$\n\x07weights\x18\x04 \x01(\x0b\x32\x13.flare.runtime.Span\x12#\n\x06scales\x18\x05 \x01(\x0b\x32\x13.flare.runtime.Span\"\xc3\x01\n\nEngramHash\x12\x30\n\x06method\x18\x01 \x01(\x0e\x32 .flare.runtime.EngramHash.Method\x12\x0e\n\x06orders\x18\x02 \x03(\r\x12\x13\n\x0bmultipliers\x18\x03 \x03(\x04\x12\x0e\n\x06primes\x18\x04 \x03(\x04\x12\x0f\n\x07offsets\x18\x05 \x03(\x04\"=\n\x06Method\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nXOR_MODULO\x10\x01\x12\x12\n\x0e\x45OS_XOR_MODULO\x10\x02\"K\n\x05\x41sset\x12\x0c\n\x04name\x18\x01 \x01(\t\x12$\n\x07payload\x18\x02 \x01(\x0b\x32\x13.flare.runtime.Span\x12\x0e\n\x06sha256\x18\x03 \x01(\t\"\xd3\x04\n\x0b\x42undleIndex\x12\x14\n\x0c\x61rchitecture\x18\x01 \x03(\x05\x12.\n\x0b\x61llocations\x18\x02 \x03(\x0b\x32\x19.flare.runtime.Allocation\x12&\n\x07modules\x18\x03 \x03(\x0b\x32\x15.flare.runtime.Module\x12*\n\tfunctions\x18\x04 \x03(\x0b\x32\x17.flare.runtime.Function\x12)\n\x06graphs\x18\x05 \x03(\x0b\x32\x19.flare.runtime.GraphIndex\x12\x30\n\x0creservations\x18\x06 \x03(\x0b\x32\x1a.flare.runtime.Reservation\x12+\n\texecution\x18\x07 \x01(\x0b\x32\x18.flare.runtime.Execution\x12\'\n\x07\x63ontext\x18\x08 \x01(\x0b\x32\x16.flare.runtime.Context\x12<\n\tresources\x18\t \x03(\x0b\x32).flare.runtime.BundleIndex.ResourcesEntry\x12\x0e\n\x06\x65vents\x18\n \x03(\t\x12*\n\x04roce\x18\x0b \x01(\x0b\x32\x1c.flare.runtime.RoceTransport\x12\x36\n\x0frebuild_weights\x18\x0c \x01(\x0b\x32\x1d.flare.runtime.RebuildWeights\x1a\x45\n\x0eResourcesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.flare.runtime.Span:\x02\x38\x01\"\xf8\x01\n\rRoceTransport\x12\x11\n\tproxy_abi\x18\x01 \x01(\r\x12\r\n\x05world\x18\x02 \x01(\r\x12\x0c\n\x04rank\x18\x03 \x01(\r\x12\x16\n\x0eregion_address\x18\x04 \x01(\x04\x12\x14\n\x0cregion_bytes\x18\x05 \x01(\x04\x12\x18\n\x10\x63ounters_address\x18\x06 \x01(\x04\x12\x16\n\x0e\x63ounters_bytes\x18\x07 \x01(\x04\x12\x12\n\nslot_bytes\x18\x08 \x01(\x04\x12\x12\n\npipe_bytes\x18\t \x01(\x04\x12\x13\n\x0b\x63trl_offset\x18\n \x01(\x04\x12\x14\n\x0cvarlen_bytes\x18\x10 \x01(\x04J\x04\x08\x0b\x10\x10\"v\n\nGraphIndex\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x66lags\x18\x02 \x01(\x04\x12\x10\n\x08resource\x18\x03 \x01(\t\x12\x12\n\nnode_count\x18\x04 \x01(\x04\x12\x12\n\nedge_count\x18\x05 \x01(\x04\x12\x11\n\tfunctions\x18\x06 \x03(\x04\"M\n\nPeerMemory\x12\r\n\x05owner\x18\x01 \x01(\r\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\x04\x12\x0e\n\x06nbytes\x18\x03 \x01(\x04\x12\x0f\n\x07readers\x18\x04 \x03(\r\"M\n\x07\x43ontext\x12\r\n\x05\x66lags\x18\x01 \x01(\r\x12\r\n\x05\x63\x61\x63he\x18\x02 \x01(\x05\x12$\n\x06limits\x18\x03 \x03(\x0b\x32\x14.flare.runtime.Limit\"\"\n\x05Limit\x12\n\n\x02id\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\x04\".\n\x0bReservation\x12\x0f\n\x07\x61\x64\x64ress\x18\x01 \x01(\x04\x12\x0e\n\x06nbytes\x18\x02 \x01(\x04\"8\n\x04Span\x12\x0e\n\x06offset\x18\x01 \x01(\x04\x12\x0e\n\x06nbytes\x18\x02 \x01(\x04\x12\x10\n\x08\x66ilename\x18\x03 \x01(\t\"\xa2\x02\n\nAllocation\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06nbytes\x18\x02 \x01(\x04\x12\x11\n\talignment\x18\x03 \x01(\x04\x12\x11\n\x04\x66ile\x18\x04 \x01(\tH\x00\x88\x01\x01\x12\'\n\ndata_spans\x18\x05 \x03(\x0b\x32\x13.flare.runtime.Span\x12.\n\x0brelocations\x18\x06 \x03(\x0b\x32\x19.flare.runtime.Relocation\x12\x14\n\x07\x61\x64\x64ress\x18\x07 \x01(\x04H\x01\x88\x01\x01\x12%\n\x06memory\x18\x08 \x01(\x0e\x32\x15.flare.runtime.Memory\x12%\n\x06sizing\x18\t \x01(\x0b\x32\x15.flare.runtime.SizingB\x07\n\x05_fileB\n\n\x08_address\"X\n\x06Sizing\x12)\n\x08\x63\x61pacity\x18\x01 \x01(\x0e\x32\x17.flare.runtime.Capacity\x12\x12\n\nunit_bytes\x18\x02 \x01(\x04\x12\x0f\n\x07maximum\x18\x03 \x01(\x04\"L\n\x06Module\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x66ile\x18\x02 \x01(\t\x12&\n\x07globals\x18\x03 \x03(\x0b\x32\x15.flare.runtime.Global\"f\n\x06Global\x12\x0e\n\x06symbol\x18\x01 \x01(\t\x12\x0c\n\x04\x66ile\x18\x02 \x01(\t\x12\x0e\n\x06nbytes\x18\x03 \x01(\x04\x12.\n\x0brelocations\x18\x04 \x03(\x0b\x32\x19.flare.runtime.Relocation\"y\n\x08\x46unction\x12\x0e\n\x06module\x18\x01 \x01(\t\x12\x0e\n\x06symbol\x18\x02 \x01(\t\x12\x17\n\x0fparameter_sizes\x18\x03 \x03(\x04\x12\x34\n\nattributes\x18\x04 \x03(\x0b\x32 .flare.runtime.FunctionAttribute\".\n\x11\x46unctionAttribute\x12\n\n\x02id\x18\x01 \x01(\x05\x12\r\n\x05value\x18\x02 \x01(\x05\"/\n\tReference\x12\x12\n\nallocation\x18\x01 \x01(\t\x12\x0e\n\x06offset\x18\x02 \x01(\x04\"s\n\nRelocation\x12\x0e\n\x06offset\x18\x01 \x01(\x04\x12(\n\x06target\x18\x02 \x01(\x0b\x32\x18.flare.runtime.Reference\x12+\n\x04kind\x18\x03 \x01(\x0e\x32\x1d.flare.runtime.RelocationKind\"D\n\x04\x42lob\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12.\n\x0brelocations\x18\x02 \x03(\x0b\x32\x19.flare.runtime.Relocation\";\n\tAttribute\x12\n\n\x02id\x18\x01 \x01(\x05\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.flare.runtime.Blob\"\x9d\x01\n\x06Kernel\x12\x10\n\x08\x66unction\x18\x01 \x01(\x04\x12\x0c\n\x04grid\x18\x02 \x03(\r\x12\r\n\x05\x62lock\x18\x03 \x03(\r\x12\x0e\n\x06shared\x18\x04 \x01(\r\x12&\n\targuments\x18\x05 \x03(\x0b\x32\x13.flare.runtime.Blob\x12,\n\nattributes\x18\x06 \x03(\x0b\x32\x18.flare.runtime.Attribute\"\x8a\x01\n\x06Memset\x12-\n\x0b\x64\x65stination\x18\x01 \x01(\x0b\x32\x18.flare.runtime.Reference\x12\r\n\x05pitch\x18\x02 \x01(\x04\x12\r\n\x05value\x18\x03 \x01(\r\x12\x14\n\x0c\x65lement_size\x18\x04 \x01(\r\x12\r\n\x05width\x18\x05 \x01(\x04\x12\x0e\n\x06height\x18\x06 \x01(\x04\"o\n\x04\x43opy\x12(\n\x06source\x18\x01 \x01(\x0b\x32\x18.flare.runtime.Reference\x12-\n\x0b\x64\x65stination\x18\x02 \x01(\x0b\x32\x18.flare.runtime.Reference\x12\x0e\n\x06nbytes\x18\x03 \x01(\x04\"\x07\n\x05\x45mpty\"\x1a\n\tEventWait\x12\r\n\x05\x65vent\x18\x01 \x01(\t\"\xe1\x01\n\x04Node\x12\'\n\x06kernel\x18\x01 \x01(\x0b\x32\x15.flare.runtime.KernelH\x00\x12\'\n\x06memset\x18\x02 \x01(\x0b\x32\x15.flare.runtime.MemsetH\x00\x12#\n\x04\x63opy\x18\x03 \x01(\x0b\x32\x13.flare.runtime.CopyH\x00\x12%\n\x05\x65mpty\x18\x04 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12.\n\nevent_wait\x18\x05 \x01(\x0b\x32\x18.flare.runtime.EventWaitH\x00\x42\x0b\n\toperation\"9\n\x04\x45\x64ge\x12\x0e\n\x06source\x18\x01 \x01(\x04\x12\x13\n\x0b\x64\x65stination\x18\x02 \x01(\x04\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"\xfc\x01\n\tExecution\x12\x13\n\x0b\x61\x62i_version\x18\x01 \x01(\r\x12\x12\n\nmax_active\x18\x02 \x01(\x04\x12\x19\n\x11input_frame_bytes\x18\x03 \x01(\x04\x12&\n\x07\x62uffers\x18\x04 \x03(\x0b\x32\x15.flare.runtime.Buffer\x12-\n\x06graphs\x18\x05 \x03(\x0b\x32\x1d.flare.runtime.ExecutionGraph\x12\x31\n\rtuning_graphs\x18\x06 \x03(\x0b\x32\x1a.flare.runtime.TuningGraph\x12!\n\x19\x63ompletion_token_capacity\x18\x07 \x01(\r\"C\n\x0bTuningGraph\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07outputs\x18\x02 \x03(\t\x12\x15\n\rstatus_buffer\x18\x03 \x01(\t\"\x82\x01\n\x06\x42uffer\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06nbytes\x18\x02 \x01(\x04\x12\r\n\x05\x64type\x18\x03 \x01(\t\x12\r\n\x05shape\x18\x04 \x03(\x04\x12\x0f\n\x07strides\x18\x05 \x03(\x04\x12+\n\treference\x18\x06 \x01(\x0b\x32\x18.flare.runtime.Reference\"\x95\x04\n\x0e\x45xecutionGraph\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04kind\x18\x02 \x01(\t\x12\x15\n\rsampling_mode\x18\x03 \x01(\t\x12\x16\n\x0e\x62\x61tch_capacity\x18\x04 \x01(\x04\x12\x16\n\x0etoken_capacity\x18\x05 \x01(\x04\x12\x33\n\x0einput_bindings\x18\x06 \x03(\x0b\x32\x1b.flare.runtime.InputBinding\x12\x19\n\x0c\x64raft_tokens\x18\x07 \x01(\rH\x00\x88\x01\x01\x12\x1a\n\rverify_tokens\x18\x08 \x01(\rH\x01\x88\x01\x01\x12\x30\n\x0c\x64raft_source\x18\t \x01(\x0e\x32\x1a.flare.runtime.DraftSource\x12 \n\x13speculation_enabled\x18\n \x01(\x08H\x02\x88\x01\x01\x12\x1d\n\x15\x65stimated_duration_ns\x18\x0b \x01(\x04\x12\x18\n\x10prefill_capacity\x18\x0c \x01(\x04\x12\x1d\n\x15packed_token_capacity\x18\r \x01(\x04\x12\x18\n\x10\x63onfidence_width\x18\x0e \x01(\r\x12\x33\n\x0etuned_bindings\x18\x0f \x03(\x0b\x32\x1b.flare.runtime.TunedBindingB\x0f\n\r_draft_tokensB\x10\n\x0e_verify_tokensB\x16\n\x14_speculation_enabled\"H\n\x0cInputBinding\x12\x13\n\x0b\x62uffer_name\x18\x01 \x01(\t\x12\x13\n\x0bhost_offset\x18\x02 \x01(\x04\x12\x0e\n\x06nbytes\x18\x03 \x01(\x04*\x8e\x01\n\x0bHistoryKind\x12\x17\n\x13HISTORY_UNSPECIFIED\x10\x00\x12\x13\n\x0fPAGED_KEY_VALUE\x10\x01\x12\x10\n\x0cPAGED_LATENT\x10\x02\x12\x12\n\x0eSLIDING_WINDOW\x10\x03\x12\x13\n\x0fRECURRENT_STATE\x10\x04\x12\x16\n\x12\x43ONTINUATION_STATE\x10\x05*\xfa\x06\n\x0bValueSource\x12\x15\n\x11VALUE_UNSPECIFIED\x10\x00\x12\x0c\n\x08\x43ONSTANT\x10\x01\x12\x10\n\x0cREQUEST_SLOT\x10\x02\x12\x16\n\x12REQUEST_GENERATION\x10\x03\x12\r\n\tLIVE_ROWS\x10\x04\x12\x10\n\x0cQUERY_LENGTH\x10\x05\x12\x12\n\x0eSAMPLE_ENABLED\x10\x06\x12\x10\n\x0cPROMPT_TOKEN\x10\x07\x12\x11\n\rPENDING_TOKEN\x10\x08\x12\x1c\n\x18PROMPT_AND_PENDING_TOKEN\x10\t\x12\x13\n\x0fPROMPT_POSITION\x10\n\x12\x17\n\x13IMAGE_FEATURE_INDEX\x10\x0b\x12\x16\n\x12IMAGE_LEFT_CONTEXT\x10\x0c\x12\x17\n\x13IMAGE_RIGHT_CONTEXT\x10\r\x12\x0e\n\nPAGE_TABLE\x10\x0e\x12\x13\n\x0fLIVE_STATE_ROWS\x10\x0f\x12\x12\n\x0eLIVE_STATE_ROW\x10\x10\x12\x14\n\x10PREFIX_STATE_ROW\x10\x11\x12\x16\n\x12SNAPSHOT_STATE_ROW\x10\x12\x12\x13\n\x0f\x43ONSUMED_TOKENS\x10\x13\x12\x1c\n\x18SNAPSHOT_CONSUMED_TOKENS\x10\x14\x12\x0f\n\x0bPREFILL_ROW\x10\x15\x12\x11\n\rSAMPLING_SEED\x10\x16\x12\x17\n\x13SAMPLING_PARAMETERS\x10\x17\x12\x12\n\x0eSAMPLING_TOP_K\x10\x18\x12\x12\n\x0eMAX_NEW_TOKENS\x10\x19\x12\x11\n\rEOS_TOKEN_IDS\x10\x1a\x12\r\n\tEOS_COUNT\x10\x1b\x12\x14\n\x10PROMPT_HISTOGRAM\x10\x1c\x12\x10\n\x0cGRAMMAR_MASK\x10\x1d\x12\x13\n\x0fGRAMMAR_ENABLED\x10\x1e\x12\x12\n\x0e\x43OPY_TOKEN_IDS\x10\x1f\x12\x14\n\x10\x43OPY_TOKEN_COUNT\x10 \x12\x18\n\x14\x43OPY_CONSUMED_TOKENS\x10!\x12\x13\n\x0f\x43OPY_GENERATION\x10\"\x12\x15\n\x11\x43OPY_GRAMMAR_MASK\x10#\x12\x16\n\x12\x43OPY_GRAMMAR_COUNT\x10$\x12\x10\n\x0cIMAGE_TENSOR\x10%\x12\x16\n\x12IMAGE_FEATURE_SLOT\x10&\x12\x17\n\x13PROMPT_TARGET_TOKEN\x10\'\x12\x1c\n\x18SPECULATIVE_PREFIX_LIMIT\x10(*|\n\rInputIndexing\x12\x18\n\x14INDEXING_UNSPECIFIED\x10\x00\x12\n\n\x06SCALAR\x10\x01\x12\x12\n\x0eSUBMISSION_ROW\x10\x02\x12\x0f\n\x0bREQUEST_ROW\x10\x03\x12\x10\n\x0cPACKED_TOKEN\x10\x04\x12\x0e\n\nACTIVE_ROW\x10\x05*T\n\x0fStateAccessKind\x12\x1c\n\x18STATE_ACCESS_UNSPECIFIED\x10\x00\x12\x08\n\x04READ\x10\x01\x12\t\n\x05WRITE\x10\x02\x12\x0e\n\nREAD_WRITE\x10\x03*\xf5\x01\n\x0eGraphOperation\x12\x1f\n\x1bGRAPH_OPERATION_UNSPECIFIED\x10\x00\x12\x08\n\x04INIT\x10\x01\x12\x0b\n\x07RESTORE\x10\x02\x12\x0c\n\x08SNAPSHOT\x10\x03\x12\x0b\n\x07PREFILL\x10\x04\x12\n\n\x06\x44\x45\x43ODE\x10\x05\x12\n\n\x06RESUME\x10\x06\x12\n\n\x06VISION\x10\x07\x12\x0f\n\x0bSPECULATIVE\x10\x08\x12\t\n\x05MIXED\x10\t\x12\x12\n\x0eSCORED_PREFILL\x10\x0c\"\x04\x08\n\x10\n\"\x04\x08\x0b\x10\x0b\"\x04\x08\r\x10\r*\x08PROPOSAL*\x0cVERIFICATION*\x12PACKED_SPECULATIVE*\xa1\x01\n\x0eToolParserKind\x12\x1b\n\x17TOOL_PARSER_UNSPECIFIED\x10\x00\x12\x0e\n\nQWEN_CODER\x10\x01\x12\x15\n\x11GLM_ARGUMENT_TAGS\x10\x02\x12\x08\n\x04\x44SML\x10\x03\x12\x08\n\x04XTML\x10\x04\x12\x0f\n\x0bMINIMAX_XML\x10\x05\x12\x0e\n\nGEMMA_CALL\x10\x06\x12\x08\n\x04\x41TEM\x10\x07\x12\x0c\n\x08TML_JSON\x10\x08*\xa6\x01\n\x13ReasoningParserKind\x12 \n\x1cREASONING_PARSER_UNSPECIFIED\x10\x00\x12\x0e\n\nTHINK_TAGS\x10\x01\x12\x0e\n\nXTML_THINK\x10\x02\x12\x0c\n\x08MM_THINK\x10\x03\x12\x13\n\x0f\x43HANNEL_THOUGHT\x10\x04\x12\x15\n\x11RECIPIENT_HEADERS\x10\x05\x12\x13\n\x0f\x43ONTENT_HEADERS\x10\x06*\xf4\x02\n\x14ImagePreparationKind\x12!\n\x1dIMAGE_PREPARATION_UNSPECIFIED\x10\x00\x12\x1a\n\x16QWEN_IMAGE_PREPARATION\x10\x01\x12\x1a\n\x16MIMO_IMAGE_PREPARATION\x10\x02\x12\x1e\n\x1a\x44\x45\x45PSEEK_IMAGE_PREPARATION\x10\x03\x12\x1d\n\x19MOONVIT_IMAGE_PREPARATION\x10\x04\x12\x1a\n\x16STEP_IMAGE_PREPARATION\x10\x05\x12\x1b\n\x17RADIO_IMAGE_PREPARATION\x10\x06\x12(\n$DEEPSEEK_ROW_MAJOR_IMAGE_PREPARATION\x10\x07\x12\x19\n\x15GLM_IMAGE_PREPARATION\x10\x08\x12!\n\x1dPATCH_MERGE_IMAGE_PREPARATION\x10\t\x12!\n\x1dPADDED_GRID_IMAGE_PREPARATION\x10\n*\xb6\x02\n\x0fImageLayoutKind\x12\x1c\n\x18IMAGE_LAYOUT_UNSPECIFIED\x10\x00\x12\x15\n\x11QWEN_IMAGE_LAYOUT\x10\x01\x12\x15\n\x11MIMO_IMAGE_LAYOUT\x10\x02\x12\x19\n\x15\x44\x45\x45PSEEK_IMAGE_LAYOUT\x10\x03\x12\x18\n\x14MOONVIT_IMAGE_LAYOUT\x10\x04\x12\x15\n\x11STEP_IMAGE_LAYOUT\x10\x05\x12\x16\n\x12RADIO_IMAGE_LAYOUT\x10\x06\x12#\n\x1f\x44\x45\x45PSEEK_ROW_MAJOR_IMAGE_LAYOUT\x10\x07\x12\x14\n\x10GLM_IMAGE_LAYOUT\x10\x08\x12\x1a\n\x16\x42RACKETED_IMAGE_LAYOUT\x10\t\x12\x1c\n\x18\x43LASS_TOKEN_IMAGE_LAYOUT\x10\n*P\n\rPromptEncoder\x12\x10\n\x0cPROMPT_JINJA\x10\x00\x12\x0f\n\x0b\x44\x45\x45PSEEK_V4\x10\x01\x12\x10\n\x0c\x44\x45\x45PSEEK_V41\x10\x02\x12\n\n\x06TML_V0\x10\x03*B\n\x10TemplateEncoding\x12\x11\n\rTEMPLATE_TEXT\x10\x00\x12\x1b\n\x17TEMPLATE_TOKEN_SEGMENTS\x10\x01*/\n\x06Memory\x12\n\n\x06\x44\x45VICE\x10\x00\x12\x0f\n\x0bMAPPED_HOST\x10\x01\x12\x08\n\x04PEER\x10\x02*(\n\x08\x43\x61pacity\x12\x0e\n\nSTATE_ROWS\x10\x00\x12\x0c\n\x08KV_PAGES\x10\x01*-\n\x0eRelocationKind\x12\x0b\n\x07POINTER\x10\x00\x12\x0e\n\nTENSOR_MAP\x10\x01*4\n\x0b\x44raftSource\x12\x0c\n\x08NO_DRAFT\x10\x00\x12\x07\n\x03MTP\x10\x01\x12\x0e\n\nCOPY_DRAFT\x10\x02\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -33,36 +33,36 @@ DESCRIPTOR._loaded_options = None _globals['_BUNDLEINDEX_RESOURCESENTRY']._loaded_options = None _globals['_BUNDLEINDEX_RESOURCESENTRY']._serialized_options = b'8\001' - _globals['_HISTORYKIND']._serialized_start=14669 - _globals['_HISTORYKIND']._serialized_end=14811 - _globals['_VALUESOURCE']._serialized_start=14814 - _globals['_VALUESOURCE']._serialized_end=15704 - _globals['_INPUTINDEXING']._serialized_start=15706 - _globals['_INPUTINDEXING']._serialized_end=15830 - _globals['_STATEACCESSKIND']._serialized_start=15832 - _globals['_STATEACCESSKIND']._serialized_end=15916 - _globals['_GRAPHOPERATION']._serialized_start=15919 - _globals['_GRAPHOPERATION']._serialized_end=16164 - _globals['_TOOLPARSERKIND']._serialized_start=16167 - _globals['_TOOLPARSERKIND']._serialized_end=16328 - _globals['_REASONINGPARSERKIND']._serialized_start=16331 - _globals['_REASONINGPARSERKIND']._serialized_end=16497 - _globals['_IMAGEPREPARATIONKIND']._serialized_start=16500 - _globals['_IMAGEPREPARATIONKIND']._serialized_end=16872 - _globals['_IMAGELAYOUTKIND']._serialized_start=16875 - _globals['_IMAGELAYOUTKIND']._serialized_end=17185 - _globals['_PROMPTENCODER']._serialized_start=17187 - _globals['_PROMPTENCODER']._serialized_end=17267 - _globals['_TEMPLATEENCODING']._serialized_start=17269 - _globals['_TEMPLATEENCODING']._serialized_end=17335 - _globals['_MEMORY']._serialized_start=17337 - _globals['_MEMORY']._serialized_end=17384 - _globals['_CAPACITY']._serialized_start=17386 - _globals['_CAPACITY']._serialized_end=17426 - _globals['_RELOCATIONKIND']._serialized_start=17428 - _globals['_RELOCATIONKIND']._serialized_end=17473 - _globals['_DRAFTSOURCE']._serialized_start=17475 - _globals['_DRAFTSOURCE']._serialized_end=17527 + _globals['_HISTORYKIND']._serialized_start=14696 + _globals['_HISTORYKIND']._serialized_end=14838 + _globals['_VALUESOURCE']._serialized_start=14841 + _globals['_VALUESOURCE']._serialized_end=15731 + _globals['_INPUTINDEXING']._serialized_start=15733 + _globals['_INPUTINDEXING']._serialized_end=15857 + _globals['_STATEACCESSKIND']._serialized_start=15859 + _globals['_STATEACCESSKIND']._serialized_end=15943 + _globals['_GRAPHOPERATION']._serialized_start=15946 + _globals['_GRAPHOPERATION']._serialized_end=16191 + _globals['_TOOLPARSERKIND']._serialized_start=16194 + _globals['_TOOLPARSERKIND']._serialized_end=16355 + _globals['_REASONINGPARSERKIND']._serialized_start=16358 + _globals['_REASONINGPARSERKIND']._serialized_end=16524 + _globals['_IMAGEPREPARATIONKIND']._serialized_start=16527 + _globals['_IMAGEPREPARATIONKIND']._serialized_end=16899 + _globals['_IMAGELAYOUTKIND']._serialized_start=16902 + _globals['_IMAGELAYOUTKIND']._serialized_end=17212 + _globals['_PROMPTENCODER']._serialized_start=17214 + _globals['_PROMPTENCODER']._serialized_end=17294 + _globals['_TEMPLATEENCODING']._serialized_start=17296 + _globals['_TEMPLATEENCODING']._serialized_end=17362 + _globals['_MEMORY']._serialized_start=17364 + _globals['_MEMORY']._serialized_end=17411 + _globals['_CAPACITY']._serialized_start=17413 + _globals['_CAPACITY']._serialized_end=17453 + _globals['_RELOCATIONKIND']._serialized_start=17455 + _globals['_RELOCATIONKIND']._serialized_end=17500 + _globals['_DRAFTSOURCE']._serialized_start=17502 + _globals['_DRAFTSOURCE']._serialized_end=17554 _globals['_PROTECTEDINDEX']._serialized_start=38 _globals['_PROTECTEDINDEX']._serialized_end=191 _globals['_SIGNEDGROUP']._serialized_start=193 @@ -82,149 +82,149 @@ _globals['_MODELINDEX']._serialized_start=988 _globals['_MODELINDEX']._serialized_end=1451 _globals['_REBUILDMANIFEST']._serialized_start=1454 - _globals['_REBUILDMANIFEST']._serialized_end=2280 - _globals['_REBUILDSPECULATION']._serialized_start=2283 - _globals['_REBUILDSPECULATION']._serialized_end=2793 - _globals['_REBUILDWEIGHTS']._serialized_start=2795 - _globals['_REBUILDWEIGHTS']._serialized_end=2896 - _globals['_REBUILDTENSOR']._serialized_start=2899 - _globals['_REBUILDTENSOR']._serialized_end=3061 - _globals['_SERVINGCONTRACT']._serialized_start=3064 - _globals['_SERVINGCONTRACT']._serialized_end=3907 - _globals['_MODELCONTRACT']._serialized_start=3910 - _globals['_MODELCONTRACT']._serialized_end=4177 - _globals['_VISIONCAPACITY']._serialized_start=4180 - _globals['_VISIONCAPACITY']._serialized_end=4324 - _globals['_CACHECONTRACT']._serialized_start=4327 - _globals['_CACHECONTRACT']._serialized_end=4461 - _globals['_HISTORYREGION']._serialized_start=4464 - _globals['_HISTORYREGION']._serialized_end=4614 - _globals['_PAGETABLEVIEW']._serialized_start=4616 - _globals['_PAGETABLEVIEW']._serialized_end=4668 - _globals['_VALUEBINDING']._serialized_start=4671 - _globals['_VALUEBINDING']._serialized_end=4848 - _globals['_STATEVIEW']._serialized_start=4850 - _globals['_STATEVIEW']._serialized_end=4925 - _globals['_STATEACCESS']._serialized_start=4927 - _globals['_STATEACCESS']._serialized_end=5028 - _globals['_COMPLETIONCURSOR']._serialized_start=5030 - _globals['_COMPLETIONCURSOR']._serialized_end=5109 - _globals['_TUNEDBINDING']._serialized_start=5111 - _globals['_TUNEDBINDING']._serialized_end=5163 - _globals['_GRAPHCONTRACT']._serialized_start=5166 - _globals['_GRAPHCONTRACT']._serialized_end=5544 - _globals['_TOOLPARSER']._serialized_start=5546 - _globals['_TOOLPARSER']._serialized_end=5653 - _globals['_REASONINGPARSER']._serialized_start=5656 - _globals['_REASONINGPARSER']._serialized_end=5805 - _globals['_PREFIXBOUNDARY']._serialized_start=5808 - _globals['_PREFIXBOUNDARY']._serialized_end=6010 - _globals['_IMAGESPAN']._serialized_start=6012 - _globals['_IMAGESPAN']._serialized_end=6133 - _globals['_IMAGETOKENLAYOUT']._serialized_start=6135 - _globals['_IMAGETOKENLAYOUT']._serialized_end=6216 - _globals['_FRONTENDCONTRACT']._serialized_start=6219 - _globals['_FRONTENDCONTRACT']._serialized_end=6717 - _globals['_CLUSTER']._serialized_start=6719 - _globals['_CLUSTER']._serialized_end=6768 - _globals['_DEFAULTSAMPLINGPARAMETERS']._serialized_start=6771 - _globals['_DEFAULTSAMPLINGPARAMETERS']._serialized_end=7167 - _globals['_SAMPLINGFLOATRANGE']._serialized_start=7169 - _globals['_SAMPLINGFLOATRANGE']._serialized_end=7241 - _globals['_SAMPLINGINTRANGE']._serialized_start=7243 - _globals['_SAMPLINGINTRANGE']._serialized_end=7313 - _globals['_SAMPLINGRANGES']._serialized_start=7316 - _globals['_SAMPLINGRANGES']._serialized_end=7722 - _globals['_FRONTENDSETTINGS']._serialized_start=7725 - _globals['_FRONTENDSETTINGS']._serialized_end=8050 - _globals['_TOKENIZERSETTINGS']._serialized_start=8053 - _globals['_TOKENIZERSETTINGS']._serialized_end=8401 - _globals['_TOKENLIST']._serialized_start=8403 - _globals['_TOKENLIST']._serialized_end=8430 - _globals['_IMAGESETTINGS']._serialized_start=8433 - _globals['_IMAGESETTINGS']._serialized_end=8941 - _globals['_PATCHMERGEIMAGESETTINGS']._serialized_start=8944 - _globals['_PATCHMERGEIMAGESETTINGS']._serialized_end=9129 - _globals['_DEEPSEEKROWMAJORIMAGESETTINGS']._serialized_start=9132 - _globals['_DEEPSEEKROWMAJORIMAGESETTINGS']._serialized_end=9267 - _globals['_QWENIMAGESETTINGS']._serialized_start=9270 - _globals['_QWENIMAGESETTINGS']._serialized_end=9400 - _globals['_MIMOIMAGESETTINGS']._serialized_start=9402 - _globals['_MIMOIMAGESETTINGS']._serialized_end=9500 - _globals['_ENGRAM']._serialized_start=9502 - _globals['_ENGRAM']._serialized_end=9626 - _globals['_ENGRAMTABLE']._serialized_start=9629 - _globals['_ENGRAMTABLE']._serialized_end=10141 - _globals['_ENGRAMTABLE_ENCODING']._serialized_start=10072 - _globals['_ENGRAMTABLE_ENCODING']._serialized_end=10141 - _globals['_ENGRAMTABLESHARD']._serialized_start=10144 - _globals['_ENGRAMTABLESHARD']._serialized_end=10284 - _globals['_ENGRAMHASH']._serialized_start=10287 - _globals['_ENGRAMHASH']._serialized_end=10482 - _globals['_ENGRAMHASH_METHOD']._serialized_start=10421 - _globals['_ENGRAMHASH_METHOD']._serialized_end=10482 - _globals['_ASSET']._serialized_start=10484 - _globals['_ASSET']._serialized_end=10559 - _globals['_BUNDLEINDEX']._serialized_start=10562 - _globals['_BUNDLEINDEX']._serialized_end=11157 - _globals['_BUNDLEINDEX_RESOURCESENTRY']._serialized_start=11088 - _globals['_BUNDLEINDEX_RESOURCESENTRY']._serialized_end=11157 - _globals['_ROCETRANSPORT']._serialized_start=11160 - _globals['_ROCETRANSPORT']._serialized_end=11408 - _globals['_GRAPHINDEX']._serialized_start=11410 - _globals['_GRAPHINDEX']._serialized_end=11528 - _globals['_PEERMEMORY']._serialized_start=11530 - _globals['_PEERMEMORY']._serialized_end=11607 - _globals['_CONTEXT']._serialized_start=11609 - _globals['_CONTEXT']._serialized_end=11686 - _globals['_LIMIT']._serialized_start=11688 - _globals['_LIMIT']._serialized_end=11722 - _globals['_RESERVATION']._serialized_start=11724 - _globals['_RESERVATION']._serialized_end=11770 - _globals['_SPAN']._serialized_start=11772 - _globals['_SPAN']._serialized_end=11828 - _globals['_ALLOCATION']._serialized_start=11831 - _globals['_ALLOCATION']._serialized_end=12121 - _globals['_SIZING']._serialized_start=12123 - _globals['_SIZING']._serialized_end=12211 - _globals['_MODULE']._serialized_start=12213 - _globals['_MODULE']._serialized_end=12289 - _globals['_GLOBAL']._serialized_start=12291 - _globals['_GLOBAL']._serialized_end=12393 - _globals['_FUNCTION']._serialized_start=12395 - _globals['_FUNCTION']._serialized_end=12516 - _globals['_FUNCTIONATTRIBUTE']._serialized_start=12518 - _globals['_FUNCTIONATTRIBUTE']._serialized_end=12564 - _globals['_REFERENCE']._serialized_start=12566 - _globals['_REFERENCE']._serialized_end=12613 - _globals['_RELOCATION']._serialized_start=12615 - _globals['_RELOCATION']._serialized_end=12730 - _globals['_BLOB']._serialized_start=12732 - _globals['_BLOB']._serialized_end=12800 - _globals['_ATTRIBUTE']._serialized_start=12802 - _globals['_ATTRIBUTE']._serialized_end=12861 - _globals['_KERNEL']._serialized_start=12864 - _globals['_KERNEL']._serialized_end=13021 - _globals['_MEMSET']._serialized_start=13024 - _globals['_MEMSET']._serialized_end=13162 - _globals['_COPY']._serialized_start=13164 - _globals['_COPY']._serialized_end=13275 - _globals['_EMPTY']._serialized_start=13277 - _globals['_EMPTY']._serialized_end=13284 - _globals['_EVENTWAIT']._serialized_start=13286 - _globals['_EVENTWAIT']._serialized_end=13312 - _globals['_NODE']._serialized_start=13315 - _globals['_NODE']._serialized_end=13540 - _globals['_EDGE']._serialized_start=13542 - _globals['_EDGE']._serialized_end=13599 - _globals['_EXECUTION']._serialized_start=13602 - _globals['_EXECUTION']._serialized_end=13854 - _globals['_TUNINGGRAPH']._serialized_start=13856 - _globals['_TUNINGGRAPH']._serialized_end=13923 - _globals['_BUFFER']._serialized_start=13926 - _globals['_BUFFER']._serialized_end=14056 - _globals['_EXECUTIONGRAPH']._serialized_start=14059 - _globals['_EXECUTIONGRAPH']._serialized_end=14592 - _globals['_INPUTBINDING']._serialized_start=14594 - _globals['_INPUTBINDING']._serialized_end=14666 + _globals['_REBUILDMANIFEST']._serialized_end=2307 + _globals['_REBUILDSPECULATION']._serialized_start=2310 + _globals['_REBUILDSPECULATION']._serialized_end=2820 + _globals['_REBUILDWEIGHTS']._serialized_start=2822 + _globals['_REBUILDWEIGHTS']._serialized_end=2923 + _globals['_REBUILDTENSOR']._serialized_start=2926 + _globals['_REBUILDTENSOR']._serialized_end=3088 + _globals['_SERVINGCONTRACT']._serialized_start=3091 + _globals['_SERVINGCONTRACT']._serialized_end=3934 + _globals['_MODELCONTRACT']._serialized_start=3937 + _globals['_MODELCONTRACT']._serialized_end=4204 + _globals['_VISIONCAPACITY']._serialized_start=4207 + _globals['_VISIONCAPACITY']._serialized_end=4351 + _globals['_CACHECONTRACT']._serialized_start=4354 + _globals['_CACHECONTRACT']._serialized_end=4488 + _globals['_HISTORYREGION']._serialized_start=4491 + _globals['_HISTORYREGION']._serialized_end=4641 + _globals['_PAGETABLEVIEW']._serialized_start=4643 + _globals['_PAGETABLEVIEW']._serialized_end=4695 + _globals['_VALUEBINDING']._serialized_start=4698 + _globals['_VALUEBINDING']._serialized_end=4875 + _globals['_STATEVIEW']._serialized_start=4877 + _globals['_STATEVIEW']._serialized_end=4952 + _globals['_STATEACCESS']._serialized_start=4954 + _globals['_STATEACCESS']._serialized_end=5055 + _globals['_COMPLETIONCURSOR']._serialized_start=5057 + _globals['_COMPLETIONCURSOR']._serialized_end=5136 + _globals['_TUNEDBINDING']._serialized_start=5138 + _globals['_TUNEDBINDING']._serialized_end=5190 + _globals['_GRAPHCONTRACT']._serialized_start=5193 + _globals['_GRAPHCONTRACT']._serialized_end=5571 + _globals['_TOOLPARSER']._serialized_start=5573 + _globals['_TOOLPARSER']._serialized_end=5680 + _globals['_REASONINGPARSER']._serialized_start=5683 + _globals['_REASONINGPARSER']._serialized_end=5832 + _globals['_PREFIXBOUNDARY']._serialized_start=5835 + _globals['_PREFIXBOUNDARY']._serialized_end=6037 + _globals['_IMAGESPAN']._serialized_start=6039 + _globals['_IMAGESPAN']._serialized_end=6160 + _globals['_IMAGETOKENLAYOUT']._serialized_start=6162 + _globals['_IMAGETOKENLAYOUT']._serialized_end=6243 + _globals['_FRONTENDCONTRACT']._serialized_start=6246 + _globals['_FRONTENDCONTRACT']._serialized_end=6744 + _globals['_CLUSTER']._serialized_start=6746 + _globals['_CLUSTER']._serialized_end=6795 + _globals['_DEFAULTSAMPLINGPARAMETERS']._serialized_start=6798 + _globals['_DEFAULTSAMPLINGPARAMETERS']._serialized_end=7194 + _globals['_SAMPLINGFLOATRANGE']._serialized_start=7196 + _globals['_SAMPLINGFLOATRANGE']._serialized_end=7268 + _globals['_SAMPLINGINTRANGE']._serialized_start=7270 + _globals['_SAMPLINGINTRANGE']._serialized_end=7340 + _globals['_SAMPLINGRANGES']._serialized_start=7343 + _globals['_SAMPLINGRANGES']._serialized_end=7749 + _globals['_FRONTENDSETTINGS']._serialized_start=7752 + _globals['_FRONTENDSETTINGS']._serialized_end=8077 + _globals['_TOKENIZERSETTINGS']._serialized_start=8080 + _globals['_TOKENIZERSETTINGS']._serialized_end=8428 + _globals['_TOKENLIST']._serialized_start=8430 + _globals['_TOKENLIST']._serialized_end=8457 + _globals['_IMAGESETTINGS']._serialized_start=8460 + _globals['_IMAGESETTINGS']._serialized_end=8968 + _globals['_PATCHMERGEIMAGESETTINGS']._serialized_start=8971 + _globals['_PATCHMERGEIMAGESETTINGS']._serialized_end=9156 + _globals['_DEEPSEEKROWMAJORIMAGESETTINGS']._serialized_start=9159 + _globals['_DEEPSEEKROWMAJORIMAGESETTINGS']._serialized_end=9294 + _globals['_QWENIMAGESETTINGS']._serialized_start=9297 + _globals['_QWENIMAGESETTINGS']._serialized_end=9427 + _globals['_MIMOIMAGESETTINGS']._serialized_start=9429 + _globals['_MIMOIMAGESETTINGS']._serialized_end=9527 + _globals['_ENGRAM']._serialized_start=9529 + _globals['_ENGRAM']._serialized_end=9653 + _globals['_ENGRAMTABLE']._serialized_start=9656 + _globals['_ENGRAMTABLE']._serialized_end=10168 + _globals['_ENGRAMTABLE_ENCODING']._serialized_start=10099 + _globals['_ENGRAMTABLE_ENCODING']._serialized_end=10168 + _globals['_ENGRAMTABLESHARD']._serialized_start=10171 + _globals['_ENGRAMTABLESHARD']._serialized_end=10311 + _globals['_ENGRAMHASH']._serialized_start=10314 + _globals['_ENGRAMHASH']._serialized_end=10509 + _globals['_ENGRAMHASH_METHOD']._serialized_start=10448 + _globals['_ENGRAMHASH_METHOD']._serialized_end=10509 + _globals['_ASSET']._serialized_start=10511 + _globals['_ASSET']._serialized_end=10586 + _globals['_BUNDLEINDEX']._serialized_start=10589 + _globals['_BUNDLEINDEX']._serialized_end=11184 + _globals['_BUNDLEINDEX_RESOURCESENTRY']._serialized_start=11115 + _globals['_BUNDLEINDEX_RESOURCESENTRY']._serialized_end=11184 + _globals['_ROCETRANSPORT']._serialized_start=11187 + _globals['_ROCETRANSPORT']._serialized_end=11435 + _globals['_GRAPHINDEX']._serialized_start=11437 + _globals['_GRAPHINDEX']._serialized_end=11555 + _globals['_PEERMEMORY']._serialized_start=11557 + _globals['_PEERMEMORY']._serialized_end=11634 + _globals['_CONTEXT']._serialized_start=11636 + _globals['_CONTEXT']._serialized_end=11713 + _globals['_LIMIT']._serialized_start=11715 + _globals['_LIMIT']._serialized_end=11749 + _globals['_RESERVATION']._serialized_start=11751 + _globals['_RESERVATION']._serialized_end=11797 + _globals['_SPAN']._serialized_start=11799 + _globals['_SPAN']._serialized_end=11855 + _globals['_ALLOCATION']._serialized_start=11858 + _globals['_ALLOCATION']._serialized_end=12148 + _globals['_SIZING']._serialized_start=12150 + _globals['_SIZING']._serialized_end=12238 + _globals['_MODULE']._serialized_start=12240 + _globals['_MODULE']._serialized_end=12316 + _globals['_GLOBAL']._serialized_start=12318 + _globals['_GLOBAL']._serialized_end=12420 + _globals['_FUNCTION']._serialized_start=12422 + _globals['_FUNCTION']._serialized_end=12543 + _globals['_FUNCTIONATTRIBUTE']._serialized_start=12545 + _globals['_FUNCTIONATTRIBUTE']._serialized_end=12591 + _globals['_REFERENCE']._serialized_start=12593 + _globals['_REFERENCE']._serialized_end=12640 + _globals['_RELOCATION']._serialized_start=12642 + _globals['_RELOCATION']._serialized_end=12757 + _globals['_BLOB']._serialized_start=12759 + _globals['_BLOB']._serialized_end=12827 + _globals['_ATTRIBUTE']._serialized_start=12829 + _globals['_ATTRIBUTE']._serialized_end=12888 + _globals['_KERNEL']._serialized_start=12891 + _globals['_KERNEL']._serialized_end=13048 + _globals['_MEMSET']._serialized_start=13051 + _globals['_MEMSET']._serialized_end=13189 + _globals['_COPY']._serialized_start=13191 + _globals['_COPY']._serialized_end=13302 + _globals['_EMPTY']._serialized_start=13304 + _globals['_EMPTY']._serialized_end=13311 + _globals['_EVENTWAIT']._serialized_start=13313 + _globals['_EVENTWAIT']._serialized_end=13339 + _globals['_NODE']._serialized_start=13342 + _globals['_NODE']._serialized_end=13567 + _globals['_EDGE']._serialized_start=13569 + _globals['_EDGE']._serialized_end=13626 + _globals['_EXECUTION']._serialized_start=13629 + _globals['_EXECUTION']._serialized_end=13881 + _globals['_TUNINGGRAPH']._serialized_start=13883 + _globals['_TUNINGGRAPH']._serialized_end=13950 + _globals['_BUFFER']._serialized_start=13953 + _globals['_BUFFER']._serialized_end=14083 + _globals['_EXECUTIONGRAPH']._serialized_start=14086 + _globals['_EXECUTIONGRAPH']._serialized_end=14619 + _globals['_INPUTBINDING']._serialized_start=14621 + _globals['_INPUTBINDING']._serialized_end=14693 # @@protoc_insertion_point(module_scope) diff --git a/ignition/recipes/deepseek41/bootstrap.py b/ignition/recipes/deepseek41/bootstrap.py index a05fffaac..a95f30305 100644 --- a/ignition/recipes/deepseek41/bootstrap.py +++ b/ignition/recipes/deepseek41/bootstrap.py @@ -101,6 +101,9 @@ class PreparationConfig: autotune_effort: str = "default" joint_tuning: bool = False tuning_replica: TuningReplica | None = None + # Prefill reductions keep each rank's contiguous row shard and gather the branch input that + # every row's consumers read. + row_shard_prefill: bool = False devices: tuple[int, ...] = (0, 1, 2, 3) # Device ordinals are host-local when each rank runs on its own Spark. hosts: tuple[str, ...] | None = None @@ -413,10 +416,11 @@ def prepare_rank(snapshot, config, *, rank, group, progress=None, core_storage=N if (weights.config.decoder_start is not None and weights.config.text["candidate_source_layer_id"] >= weights.config.decoder_start): candidate_tokens = min(candidate_tokens, 128) + row_shard = shard if config.row_shard_prefill else None prefill_storage = Scratch.allocate(config.prefill_chunk_size, 1, shard.world_size, device, - candidate_tokens=candidate_tokens) + candidate_tokens=candidate_tokens, row_shard=row_shard) scored_storage = (Scratch.allocate(config.prefill_chunk_size, 1, shard.world_size, device, - candidate_tokens=config.prefill_chunk_size) + candidate_tokens=config.prefill_chunk_size, row_shard=row_shard) if config.prompt_logprobs else None) decoder_storage = Scratch.allocate(config.decode_tokens, verification_samples, shard.world_size, device) vision_state = vision_inputs = vision_prefill = vision_storage = feature_pool = None @@ -507,6 +511,9 @@ def add(name, forward, kind, rows, tokens, fields, *, enabled=None, packed_token input_vocabulary=INPUT_VOCABULARY if vision_state is not None else None, validate_prefill=vision_state.validate_prefill if vision_state is not None and mode == "prefill" else None, ) + if (config.row_shard_prefill and mode == "prefill" and decoder.row_shard is None + and progress is not None): + progress(f"{name}: row-sharded prefill falls back to full-row prefill for this capacity") forward = PreparedGeneration(decoder, batch, completion, frame, sequence, runtime=collective, coordinator=coordinator, session=session, name=name + ".generation", workspace=arena, context_factory=PreparedContext if kind == "scored_prefill" else _CONTEXT) diff --git a/ignition/recipes/deepseek41/decoder.py b/ignition/recipes/deepseek41/decoder.py index 8859636ed..9ececf991 100644 --- a/ignition/recipes/deepseek41/decoder.py +++ b/ignition/recipes/deepseek41/decoder.py @@ -8,7 +8,9 @@ import torch -from ignition.components.collectives import PreparedAllReduce +from ignition.components.collectives import (PreparedAllReduce, PreparedChunkedGatherShards, + PreparedGatherShards, PreparedReduceScatter) +from ignition.preparation.persistent import persistent_zeros from ignition.components.embedding import PreparedEmbedding from ignition.components.expert_overlap import ConcurrentSharedExpert from ignition.components.ring_commit import PreparedRingCommit @@ -25,6 +27,128 @@ from .engram import EngramScratch, PreparedEngram, PreparedTableRequests +# Row-shard eligibility uses 512-row blocks. This constant does not bound mHC calls; +# callers select bounded calls only by passing max_rows_per_call to PreparedMHC. +MHC_CALL_ROWS = 512 + + +@dataclass(frozen=True) +class RowShard: + """One rank's contiguous rows of a row-sharded prefill decoder. + + Every rank declares the same shard sizes, so a traced forward keeps one layout and the ranks + differ only in the values the collectives exchange. These tensors are declaration surfaces: + they size the prepared reductions, gathers and mHC calls, and `functional()` leaves the + graph's own copies to the traced forward, where compilation plans them. + """ + + world: int + rank: int + rows: int + start: int + rows_index: torch.Tensor + attention_output: torch.Tensor + mlp_output: torch.Tensor + residual: torch.Tensor + pre_mix: torch.Tensor + attn_mix: torch.Tensor + attention_hc: MHCScratch + ffn_hc: MHCScratch + # Every graph's `rows_index` is a view of this table, one segment per shard row count (see + # `_row_table_layout`). One compiled forward binds every rank, and binding compares storage + # offsets, so a segment sits at the same offset on every rank and only its contents, this rank's + # row numbers, differ. + rows_table: torch.Tensor | None = None + + @classmethod + def allocate(cls, tokens, world, rank, device): + """This rank's shard declarations for a decoder of `tokens` rows, or None when rows do not divide.""" + if world < 2 or tokens % world or tokens // world % MHC_CALL_ROWS: + return None + rows, start = tokens // world, rank * (tokens // world) + def zeros(*shape, dtype=torch.bfloat16): + return torch.zeros(shape, dtype=dtype, device=device) + def scratch(): + return MHCScratch.allocate(rows, 5120, device) + layout = _row_table_layout(rows) + rows_values = torch.cat([torch.arange(rank * count, (rank + 1) * count, dtype=torch.int64) + for count, _ in layout]) + # The table is snapshot storage: prefill scratch is exported as zeros, and a captured graph + # reads these row numbers as data. `persistent_zeros` routes to the core-owned arena while a + # preparation exports, and to ordinary storage on a CPU test. + rows_table = persistent_zeros(rows_values.shape, rows_values.dtype, device).copy_(rows_values) + first = layout[-1][1] + return cls(world, rank, rows, start, rows_table[first:first + rows], zeros(rows, 5120), + zeros(rows, 5120), zeros(rows, 4, 5120), zeros(rows, 4, dtype=torch.float32), + zeros(rows, 4, dtype=torch.float32), scratch(), scratch(), rows_table) + + def graph(self, tokens): + """The shard a decoder of `tokens` rows uses, or None when its rows do not divide.""" + rows = tokens // self.world + if tokens % self.world or rows % MHC_CALL_ROWS or rows > self.rows: + return None + # A narrower graph holds fewer rows per rank, so this rank's block begins earlier than it + # does in the captured layout: the graph reads its row count's segment of the table. + start = self.rank * rows + layout = _row_table_layout(self.rows) + index = [count for count, _ in layout].index(rows) + first = layout[index][1] + return replace(self, rows=rows, start=start, + rows_index=self.rows_table[first:first + rows], + attention_output=self.attention_output[:rows], mlp_output=self.mlp_output[:rows], + residual=self.residual[:rows], pre_mix=self.pre_mix[:rows], attn_mix=self.attn_mix[:rows], + attention_hc=self.attention_hc.rows(rows), ffn_hc=self.ffn_hc.rows(rows)) + + +def _row_table_layout(rows): + """Each shard row count up to `rows` a graph can use, with its segment's first table element. + + The layout depends only on `rows`, which every rank shares, so a segment's storage offset is the + same on every rank. + """ + layout, first = [], 0 + for count in range(1, rows + 1): + if count % MHC_CALL_ROWS == 0: + layout.append((count, first)) + first += count + return layout + + +def row_shard_ready(scratch, shard, runtime): + """True when a runtime serves every row-sharded reduction and gather over this scratch.""" + return (PreparedReduceScatter.fits(runtime, scratch.attention.output, shard.attention_output) + and PreparedReduceScatter.fits(runtime, scratch.mlp_partial, shard.mlp_output) + and PreparedGatherShards.fits(runtime, shard.attention_hc.normalized, + scratch.attention_hc.normalized) + and PreparedGatherShards.fits(runtime, shard.ffn_hc.normalized, scratch.ffn_hc.normalized) + and PreparedGatherShards.fits(runtime, shard.pre_mix.view(torch.bfloat16), + scratch.pre_mix.view(torch.bfloat16)) + and PreparedChunkedGatherShards.fits(runtime, shard.residual, scratch.residual)) + + +def shard_schedule(layers, stop, sharded, aux_layers=()): + """Per-layer `(shard, defer, reseed)` for a prefill decoder whose shard exists. + + Layers below `stop` that are not Engram layers keep this rank's rows. An Engram layer reads and + writes full rows, so it runs full-row, and the layers on either side of it change row dimension: + a layer that lands on the far side of such a change runs its own post, so the full-row residual + its consumer reads is complete, and the first sharded layer after a full-row run reseeds the + shard from the rows it owns. + """ + def sharded_at(index): + return (sharded is not None and index < stop and index < len(layers) + and index not in aux_layers and layers[index].engram is None) + schedule = [] + for index in range(len(layers)): + shard = sharded_at(index) + # Deferred post hands this layer's four streams to the next layer's attention mHC, which + # reads them only when both layers hold the same rows. + defer = (index + 1 < len(layers) and layers[index + 1].engram is None + and index + 1 not in aux_layers and sharded_at(index + 1) == shard) + schedule.append((shard, defer, shard and index > 0 and not sharded_at(index - 1))) + return schedule + + @dataclass(frozen=True) class Scratch: """BF16 decoder activations and FP32 target vocabulary logits.""" @@ -50,6 +174,7 @@ class Scratch: hidden: torch.Tensor local_logits: torch.Tensor logits: torch.Tensor + row_shard: RowShard | None = None def rows(self, tokens, samples): def select(value, count): @@ -58,14 +183,19 @@ def select(value, count): if is_dataclass(value): return type(value)(*(select(getattr(value, field.name), count) for field in fields(value))) raise TypeError("decoder scratch must contain tensors and tensor dataclasses") - return type(self)(*(select(getattr(self, field.name), samples if field.name in - ("hidden", "local_logits", "logits") else tokens) for field in fields(self))) + def field_value(name): + if name == "row_shard": + return None if self.row_shard is None else self.row_shard.graph(tokens) + return select(getattr(self, name), samples if name in + ("hidden", "local_logits", "logits") else tokens) + return type(self)(*(field_value(field.name) for field in fields(self))) @classmethod def allocate(cls, tokens, samples, world, device, *, experts=384, top_k=6, - candidate_tokens=None): + candidate_tokens=None, row_shard=None): def zeros(*shape, dtype=torch.bfloat16): return torch.zeros(shape, dtype=dtype, device=device) + shard = None if row_shard is None else RowShard.allocate(tokens, world, row_shard.rank, device) residual, engram_residual = zeros(tokens, 4, 5120), zeros(tokens, 4, 5120) pre_mix, attn_mix = zeros(tokens, 4, dtype=torch.float32), zeros(tokens, 4, dtype=torch.float32) attention_hc = MHCScratch.allocate(tokens, 5120, device) @@ -84,7 +214,7 @@ def zeros(*shape, dtype=torch.bfloat16): routed, partial = zeros(tokens, 5120), zeros(tokens, 5120) return cls(embedding, residual, engram_residual, engram, pre_mix, attn_mix, attention_hc, ffn_hc, attention, attention_output, shared, router_logits, router, routed, partial, mlp_output, - hidden, local_logits, logits) + hidden, local_logits, logits, shard) @@ -129,9 +259,11 @@ class PreparedLayer: def __init__(self, index, weights, tables, scratch, metadata, pool, rotary, *, shard, mode, runtime, coordinator, session, name, workspace, carry, defer_post, boundary=None, replay_start=None, window_staging=None, mla_compute_mode="auto", - window_metadata=None, examples=None): + window_metadata=None, examples=None, row_shard=None, reseed=False): s, text = scratch, pool.config.text - self.scratch, self.carry = s, carry + self.scratch, self.carry, self.row_shard = s, carry, row_shard + # A full-row layer owns the rows since this shard last ran, so the shard is refilled here. + self.reseed = reseed and row_shard is not None self.metadata = metadata self.image_mask = (torch.zeros(metadata.token_ids.shape, dtype=torch.bool, device=metadata.token_ids.device) if mode == "prefill" and weights.experts.vision_correction is not None else None) @@ -142,23 +274,42 @@ def __init__(self, index, weights, tables, scratch, metadata, pool, rotary, *, s metadata, pool, scratch=s.engram, session=session, name=name + ".engram", workspace=workspace, runtime=runtime, coordinator=coordinator, examples=examples) - source = s.engram_residual if self.engram is not None else s.residual + # A row shard reduces into this rank's rows and normalizes them there; the full-size scratch + # holds the branch input every consumer reads, which a gather fills before that consumer runs. + attention_scratch = s.attention_hc if row_shard is None else row_shard.attention_hc + ffn_scratch = s.ffn_hc if row_shard is None else row_shard.ffn_hc + pre_mix = s.pre_mix if row_shard is None else row_shard.pre_mix + attn_mix = s.attn_mix if row_shard is None else row_shard.attn_mix + residual = s.residual if row_shard is None else row_shard.residual + attention_output = s.attention_output if row_shard is None else row_shard.attention_output + mlp_output = s.mlp_output if row_shard is None else row_shard.mlp_output + source = s.engram_residual if self.engram is not None else residual options = dict(rms_eps=text["rms_norm_eps"], norm_eps=text["rms_norm_eps"], hc_eps=text["hc_eps"], sinkhorn_iters=text["hc_sinkhorn_iters"]) - incoming = ((s.mlp_output, s.ffn_hc.residual, s.ffn_hc.post, s.ffn_hc.comb) if carry else (source,)) + incoming = ((mlp_output, ffn_scratch.residual, ffn_scratch.post, ffn_scratch.comb) if carry else (source,)) self.attention_hc = None if boundary is not None else PreparedMHC( - incoming, _mhc_weights(weights, "attn", "attn_norm"), s.attention_hc, - session=session, name=name + ".attn_hc", workspace=workspace, pre_mix=s.pre_mix, pre_out=s.attn_mix, **options) + incoming, _mhc_weights(weights, "attn", "attn_norm"), attention_scratch, + session=session, name=name + ".attn_hc", workspace=workspace, pre_mix=pre_mix, pre_out=attn_mix, **options) self.attention = PreparedAttention(index, weights.attention, s.attention_hc.normalized, s.attention, metadata, pool, rotary, mode=mode, world=shard.world_size, session=session, name=name + ".attention", workspace=workspace, replay_start=replay_start, window_staging=window_staging, mla_compute_mode=mla_compute_mode, window_metadata=window_metadata, examples=examples) - self.attention_reduce = PreparedAllReduce(s.attention.output, s.attention_output, runtime=runtime, - coordinator=coordinator, session=session, name=name + ".attention_reduce") - self.ffn_hc = PreparedMHC((s.attention_output, s.attention_hc.residual, s.attention_hc.post, s.attention_hc.comb), - _mhc_weights(weights, "ffn", "ffn_norm"), s.ffn_hc, session=session, name=name + ".ffn_hc", workspace=workspace, - pre_mix=s.attn_mix, pre_out=s.pre_mix, **options) + self.attention_gather = None if row_shard is None else PreparedGatherShards( + attention_scratch.normalized, s.attention_hc.normalized, runtime=runtime, coordinator=coordinator, + session=session, name=name + ".attention_gather") + if row_shard is None: + self.attention_reduce = PreparedAllReduce(s.attention.output, s.attention_output, runtime=runtime, + coordinator=coordinator, session=session, name=name + ".attention_reduce") + else: + self.attention_reduce = PreparedReduceScatter(s.attention.output, attention_output, runtime=runtime, + coordinator=coordinator, session=session, name=name + ".attention_reduce") + self.ffn_hc = PreparedMHC((attention_output, attention_scratch.residual, attention_scratch.post, + attention_scratch.comb), _mhc_weights(weights, "ffn", "ffn_norm"), ffn_scratch, session=session, + name=name + ".ffn_hc", workspace=workspace, pre_mix=attn_mix, pre_out=pre_mix, **options) + self.ffn_gather = None if row_shard is None else PreparedGatherShards( + ffn_scratch.normalized, s.ffn_hc.normalized, runtime=runtime, coordinator=coordinator, + session=session, name=name + ".ffn_gather") self.router_projection = UnquantizedProjection(s.ffn_hc.normalized, weights.experts.router, s.router_logits, session=session, name=name + ".router") self.router = PreparedRouter(s.router_logits, s.router, session=session, name=name + ".topk", @@ -178,15 +329,33 @@ def __init__(self, index, weights, tables, scratch, metadata, pool, rotary, *, s parallel_plans=(self.router_projection.plan, self.router.plan, self.experts.plan), session=session, workspace=workspace, defer_launch=s.ffn_hc.normalized.shape[0] > 4) - self.mlp_reduce = PreparedAllReduce(s.mlp_partial, s.mlp_output, runtime=runtime, - coordinator=coordinator, session=session, name=name + ".mlp_reduce") - self.post = None if defer_post else PreparedMHCPost(s.mlp_output, s.ffn_hc.residual, - s.ffn_hc.post, s.ffn_hc.comb, s.residual, session=session, name=name + ".post") + if row_shard is None: + self.mlp_reduce = PreparedAllReduce(s.mlp_partial, s.mlp_output, runtime=runtime, + coordinator=coordinator, session=session, name=name + ".mlp_reduce") + else: + self.mlp_reduce = PreparedReduceScatter(s.mlp_partial, mlp_output, runtime=runtime, + coordinator=coordinator, session=session, name=name + ".mlp_reduce") + self.post = None if defer_post else PreparedMHCPost(mlp_output, ffn_scratch.residual, + ffn_scratch.post, ffn_scratch.comb, residual, session=session, name=name + ".post") + # A deferred post runs in the next layer's attention mHC over the same rows, so only a layer + # that runs its own post widens this rank's four-stream residual back to every row. + self.residual_gather = self.mix_gather = None + if row_shard is not None and not defer_post: + self.residual_gather = PreparedChunkedGatherShards(row_shard.residual, s.residual, runtime=runtime, + session=session, coordinator=coordinator, name=name + ".residual_gather") + # A BF16 view gathers the FP32 lagged mix through the same prepared surface. + self.mix_gather = PreparedGatherShards(row_shard.pre_mix.view(torch.bfloat16), + s.pre_mix.view(torch.bfloat16), runtime=runtime, coordinator=coordinator, + session=session, name=name + ".mix_gather") if self.attention_hc is not None: self.attention_hc.functional() + if self.attention_gather is not None: + self.attention_gather.functional(session=session) self.attention_reduce.functional(session=session) self.ffn_hc.functional() + if self.ffn_gather is not None: + self.ffn_gather.functional(session=session) self.router_projection.functional(session=session) self.router.functional(session=session) self.experts.functional(session=session) @@ -197,11 +366,17 @@ def __init__(self, index, weights, tables, scratch, metadata, pool, rotary, *, s self.mlp_reduce.functional(session=session) if self.post is not None: self.post.functional() + if self.residual_gather is not None: + self.residual_gather.functional(session=session) + self.mix_gather.functional(session=session) self.has_image_mask = self.image_mask is not None register_finalizer(session, self.release_examples) def release_examples(self): self.scratch = self.metadata = self.image_mask = None + if self.row_shard is not None: + self.row_shard = replace(self.row_shard, attention_output=None, mlp_output=None, + residual=None, pre_mix=None, attn_mix=None, attention_hc=None, ffn_hc=None) if self.concurrent_shared is not None: self.shared = None @@ -211,6 +386,12 @@ def __call__(self, incoming, pre_mix, metadata, indices, selection, *, boundary= return incoming, next_mix, records, selection def _run(self, incoming, pre_mix, metadata, indices, selection, *, boundary=None, window_cache=None): + shard = self.row_shard + if self.reseed: + # The full-row layers since this shard last ran own every row, so take this rank's rows + # out of them through the row numbers the graph reads. + incoming = tuple(value.index_select(0, shard.rows_index) for value in incoming) + pre_mix = pre_mix.index_select(0, shard.rows_index) if self.engram is not None: incoming = (self.engram(incoming[0], metadata),) if boundary is None: @@ -218,6 +399,8 @@ def _run(self, incoming, pre_mix, metadata, indices, selection, *, boundary=None residual, post, comb, normalized = self.attention_hc(*incoming, pre_mix=pre_mix, pre_out=attn_mix) else: (residual, post, comb, normalized), attn_mix = boundary + if self.attention_gather is not None: + normalized = self.attention_gather(normalized) attention_normalized = normalized attended, records, selection, observations = self.attention._run(normalized, metadata, selection, indices=indices, window_cache=window_cache) @@ -225,6 +408,8 @@ def _run(self, incoming, pre_mix, metadata, indices, selection, *, boundary=None next_mix = torch.zeros_like(attn_mix) residual, post, comb, normalized = self.ffn_hc(attention_output, residual, post, comb, pre_mix=attn_mix, pre_out=next_mix) + if self.ffn_gather is not None: + normalized = self.ffn_gather(normalized) pending = None if self.concurrent_shared is None else self.concurrent_shared.start(normalized) image_mask = metadata.token_ids >= 129280 if self.has_image_mask else None router_logits = self.router_projection(normalized) @@ -236,6 +421,14 @@ def _run(self, incoming, pre_mix, metadata, indices, selection, *, boundary=None output = self.mlp_reduce(routed + shared) incoming = ((self.post(output, residual, post, comb),) if self.post is not None else (output, residual, post, comb)) + if self.residual_gather is not None: + # Every row's consumer follows, so widen this rank's rows back out: the four-stream + # residual and the lagged mix the next layer's mHC reads. + residual, = incoming + full_mix = torch.empty((metadata.token_ids.numel(), 4), dtype=next_mix.dtype, + device=next_mix.device) + self.mix_gather(next_mix.view(torch.bfloat16), full_mix.view(torch.bfloat16)) + incoming, next_mix = (self.residual_gather(residual),), full_mix observations = {"attention." + key: value for key, value in observations.items()} observations.update({"attention_hc.normalized": attention_normalized, "router.ids": route.ids, "output": incoming[0]}) @@ -265,6 +458,14 @@ def __init__(self, weights, tables, scratch, metadata, pool, *, mode, runtime, self.aux_layers = () if weights.draft is None else weights.config.draft.target_layers if self.packing is not None and any(layer <= decoder_start for layer in self.aux_layers): raise ValueError("bounded decoder feedback requires draft target layers after its global-KV boundary") + # Layers below `stop` run row-sharded; the layer at `stop - 1` materializes every row again + # so the decoder suffix keeps its full-row boundary and replay gather. + stop = (len(weights.layers) if self.packing is None else decoder_start) - 1 + self.row_shard = scratch.row_shard + if (self.row_shard is None or mode != "prefill" or stop < 1 + or not row_shard_ready(scratch, self.row_shard, runtime)): + self.row_shard = None + schedule = shard_schedule(weights.layers, stop, self.row_shard, self.aux_layers) self.auxiliary_shape = (self.output_metadata.token_ids.numel(), len(self.aux_layers) * 5120) self.auxiliary = torch.zeros(self.auxiliary_shape, dtype=torch.bfloat16, device=metadata.positions.device) self.safe_tokens = torch.zeros_like(metadata.token_ids) if self.vision is not None else metadata.token_ids @@ -304,8 +505,8 @@ def __init__(self, weights, tables, scratch, metadata, pool, *, mode, runtime, source_indices=self.window_metadata) layer = replace(layer, attention={key: value for key, value in layer.attention.items() if key not in ("compressor", "index_k", "index_norm")}) - defer = (index + 1 < len(weights.layers) and weights.layers[index + 1].engram is None - and index + 1 not in self.aux_layers) + sharded, defer, reseed = schedule[index] + sharded = self.row_shard if sharded else None self.layers.append(PreparedLayer(index, layer, tables, layer_scratch, layer_metadata, pool, self.rotary, shard=weights.shard, mode=mode, runtime=runtime, coordinator=coordinator, session=session, name=f"{name}.layer{index}", workspace=workspace, @@ -313,7 +514,7 @@ def __init__(self, weights, tables, scratch, metadata, pool, *, mode, runtime, window_staging=None if self.window_commit is None else self.window_commit.staging[index], mla_compute_mode=mla_compute_mode, window_metadata=(self.window_metadata if layer_metadata is metadata else self.suffix_window_metadata), - examples=examples)) + examples=examples, row_shard=sharded, reseed=reseed)) carry = defer self.scoring = None if scoring is None else scoring.prepare(scratch.embedding, weights.head, vocabulary=129280, session=session, coordinator=coordinator, name=name + ".prompt", @@ -348,6 +549,9 @@ def _prepare_head(self, *, vocabulary_parallel, vocabulary_runtime, coordinator, def release_outputs(self): self.scratch = self.metadata = self.output_scratch = self.output_metadata = None + if self.row_shard is not None: + self.row_shard = replace(self.row_shard, attention_output=None, mlp_output=None, + residual=None, pre_mix=None, attn_mix=None, attention_hc=None, ffn_hc=None) self.auxiliary = self.safe_tokens = self.head_hidden = self.local_logits = None def __call__(self, metadata): @@ -362,6 +566,10 @@ def __call__(self, metadata): pre_mix = torch.zeros((embedding.shape[0], 4), dtype=torch.float32, device=embedding.device) pre_mix[:, 0].fill_(1) incoming = (residual,) + if self.row_shard is not None: + # The first layer keeps this rank's rows, so enter with them. + incoming = (residual.index_select(0, self.row_shard.rows_index),) + pre_mix = pre_mix.index_select(0, self.row_shard.rows_index) indices = self.window_metadata(metadata) selection = (torch.full((embedding.shape[0], 512), -1, dtype=torch.int32, device=embedding.device), None, None) staging = (None if self.staging_shape is None else torch.zeros( diff --git a/ignition/recipes/deepseek41/recipe.py b/ignition/recipes/deepseek41/recipe.py index 4a13fe3c5..917dad35a 100644 --- a/ignition/recipes/deepseek41/recipe.py +++ b/ignition/recipes/deepseek41/recipe.py @@ -29,7 +29,7 @@ def prepare(snapshot, plan, *, compile_forwards, kernel_cache, progress, retain_ collective_ladder=plan.collective_ladder, prompt_logprobs=plan.prompt_logprobs, compile_workers=plan.compile_workers, autotune_effort=plan.autotune_effort, tuning_replica=plan.tuning_replica, native_dense_activations=plan.native_dense_activations, - joint_tuning=plan.joint_tuning, + joint_tuning=plan.joint_tuning, row_shard_prefill=plan.row_shard_prefill, lock_gpu_clocks=plan.lock_gpu_clocks and plan.autotune_effort != "off", image_concurrency=plan.image_concurrency, devices=tuple(range(plan.world_size)) if ranks is None else (0,) * plan.world_size, diff --git a/tests/cli/test_commands.py b/tests/cli/test_commands.py index 34a58ce3e..c52f7725e 100644 --- a/tests/cli/test_commands.py +++ b/tests/cli/test_commands.py @@ -315,6 +315,13 @@ def test_shard_hyperconnections_flag(command): assert parser.parse_args([command, "qwen3.8-flash", "--shard-hyperconnections"]).shard_hyperconnections +@pytest.mark.parametrize("command", ["export", "debug-serve"]) +def test_row_shard_prefill_flag(command): + parser = _parser() + assert not parser.parse_args([command, "deepseek-v4.1-flash"]).row_shard_prefill + assert parser.parse_args([command, "deepseek-v4.1-flash", "--row-shard-prefill"]).row_shard_prefill + + @pytest.mark.parametrize("command", ["export", "debug-serve"]) def test_collective_tuning_preparation_flag_is_explicit(command): assert _parser().parse_args([command, "qwen3.8-flash"]).collective_tuning is None diff --git a/tests/cli/test_planning.py b/tests/cli/test_planning.py index ec8dc756b..6e8b3e397 100644 --- a/tests/cli/test_planning.py +++ b/tests/cli/test_planning.py @@ -195,6 +195,48 @@ def test_joint_tuning_is_opt_in_and_serialized_without_changing_shapes(monkeypat replace(joint, autotune_effort="off") +def test_row_shard_prefill_requires_deepseek_tp_and_survives_rebuild_serialization(monkeypatch): + """The option travels through `plan` and `fit` beside an explicit MXFP8 scope.""" + from types import SimpleNamespace + from google.protobuf import json_format + from ignition.preparation import rebuild_manifest, runtime_core_pb2 as proto + + monkeypatch.setattr("ignition.preparation.source_identity.source_identity", lambda: "test-source") + model = replace(SOLO, name="deepseek41") + gpus = (gpu(0, 95), gpu(1, 95)) + settings = dict(selected=(0, 1), concurrency=4, reserve_bytes=0, speculation=DRAFTING, + online_mxfp8_scope="all") + ordinary = planning.plan(model, model.checkpoints[0], gpus, **settings) + sharded = planning.plan(model, model.checkpoints[0], gpus, row_shard_prefill=True, **settings) + assert not ordinary.row_shard_prefill and ordinary.online_mxfp8_scope == "all" + assert sharded.row_shard_prefill and sharded.online_mxfp8_scope == "all" + assert sharded.to_dict()["row_shard_prefill"] is True + # Turning the flag off restores the plan the same arguments produce without it. + assert replace(sharded, row_shard_prefill=False) == ordinary + fitted = planning.fit(model, model.checkpoints[0], gpus, concurrency=4, reserve_bytes=0, + speculation=DRAFTING, online_mxfp8_scope="all", row_shard_prefill=True) + assert fitted.row_shard_prefill and fitted.online_mxfp8_scope == "all" + for plan in (ordinary, sharded, fitted): + manifest = rebuild_manifest.from_plan(SimpleNamespace(directory=None), plan, compiled_forwards=True) + encoded = json_format.ParseDict(manifest, proto.RebuildManifest()) + assert proto.RebuildManifest.FromString(encoded.SerializeToString()).row_shard_prefill == plan.row_shard_prefill + with pytest.raises(UsageError, match="DeepSeek V4.1"): + planning.plan(SOLO, SOLO.checkpoints[0], gpus, row_shard_prefill=True, **settings) + with pytest.raises(UsageError, match="tensor parallelism"): + planning.plan(model, model.checkpoints[0], (gpu(0, 95),), selected=(0,), concurrency=4, + reserve_bytes=0, speculation=DRAFTING, row_shard_prefill=True) + # The plan rejects three ranks for row divisibility and rejects SM103 eligibility. + three = recipe("deepseek41", 60, (3, 4), speculation=DRAFTING) + with pytest.raises(UsageError, match="TP3"): + planning.plan(three, three.checkpoints[0], (gpu(0, 95), gpu(1, 95), gpu(2, 95)), + selected=(0, 1, 2), concurrency=4, reserve_bytes=0, speculation=DRAFTING, + row_shard_prefill=True) + with pytest.raises(UsageError, match="SM103"): + planning.plan(model, model.checkpoints[0], (gpu(0, 95, (10, 3)), gpu(1, 95, (10, 3))), + selected=(0, 1), concurrency=4, reserve_bytes=0, speculation=DRAFTING, + row_shard_prefill=True) + + def test_image_concurrency_defaults_and_explicit_capacity(): vision = replace(SOLO, max_images=4) settings = dict(concurrency=32, reserve_bytes=0, speculation=None, selected=None) diff --git a/tests/components/test_row_shard_collectives.py b/tests/components/test_row_shard_collectives.py new file mode 100644 index 000000000..b3815fc15 --- /dev/null +++ b/tests/components/test_row_shard_collectives.py @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Row-sharded reductions and shard gathers keep the all-reduce path's layout and eligibility.""" + +from types import SimpleNamespace + +import pytest +import torch + + +def _surfaces(monkeypatch, *, abi=11, roce=True): + """Enable the ABI 11 row-shard surfaces on the module the wrappers test at run time.""" + import b12x.comm.roce._proxy as proxy + from ignition.components import collectives + + monkeypatch.setattr(proxy, "ABI_VERSION", abi) + monkeypatch.setattr(collectives, "_roce", lambda runtime: roce) + + +def _runtime(*, world=4, pipe_bytes=1 << 26, gather=None): + calls = [] + + def reduce_scatter(inp, *, out, plan): + calls.append(("reduce_scatter", tuple(inp.shape), tuple(out.shape))) + + def gather_shards(inp, *, out, plan): + calls.append(("gather_shards", tuple(inp.shape), tuple(out.shape))) + + return SimpleNamespace(world_size=world, pipe_bytes=pipe_bytes, _group=object(), + should_reduce_scatter=lambda inp: True, should_gather_shards=lambda inp: True, + reduce_scatter=reduce_scatter, gather_shards=gather or gather_shards, calls=calls) + + +@pytest.fixture +def prepared(monkeypatch): + """Prepare row-shard operations without a device or a preparation session.""" + from ignition.components import collectives + + _surfaces(monkeypatch) + surfaces, plan = [], SimpleNamespace(request=lambda **kwargs: SimpleNamespace(**kwargs)) + monkeypatch.setattr(collectives, "_roce_plan", + lambda runtime, coordinator, surface: surfaces.append(surface) or plan) + monkeypatch.setattr(collectives, "declare", lambda *args, **kwargs: None) + monkeypatch.setattr(torch.distributed, "get_process_group_ranks", lambda group: [0, 1, 2, 3]) + return surfaces, plan + + +@pytest.mark.parametrize("abi,roce,drop,expected", [ + (11, True, None, True), (10, True, None, False), (11, False, None, False), + (11, True, "should_reduce_scatter", False), (11, True, "should_gather_shards", False)]) +def test_row_shard_surfaces_require_abi_eleven_and_both_runtime_surfaces(monkeypatch, abi, roce, drop, expected): + from ignition.components import collectives + + _surfaces(monkeypatch, abi=abi, roce=roce) + runtime = _runtime() + if drop is not None: + delattr(runtime, drop) + assert collectives.row_shard_surfaces(runtime) is expected + + +def test_reduce_scatter_fits_splits_whole_row_packs_into_the_shard(monkeypatch): + from ignition.components.collectives import PreparedReduceScatter + + _surfaces(monkeypatch) + runtime = _runtime() + shard = torch.zeros(2, 5120, dtype=torch.bfloat16) + assert PreparedReduceScatter.fits(runtime, torch.zeros(8, 5120, dtype=torch.bfloat16), shard) + # A four-stream residual shards along its rows, not its trailing dims. + assert PreparedReduceScatter.fits(runtime, torch.zeros(8, 4, 5120, dtype=torch.bfloat16), + torch.zeros(2, 4, 5120, dtype=torch.bfloat16)) + assert not PreparedReduceScatter.fits(runtime, torch.zeros(8, 5120, dtype=torch.bfloat16), torch.zeros(4, 5120, dtype=torch.bfloat16)) + assert not PreparedReduceScatter.fits(runtime, torch.zeros(6, 5120, dtype=torch.bfloat16), shard) + assert not PreparedReduceScatter.fits(runtime, torch.zeros(8, 5120, dtype=torch.bfloat16)[:, :1], torch.zeros(2, 1, dtype=torch.bfloat16)) + assert not PreparedReduceScatter.fits(runtime, torch.zeros(8, 7, dtype=torch.bfloat16), torch.zeros(2, 7, dtype=torch.bfloat16)) + assert not PreparedReduceScatter.fits(runtime, torch.zeros(8, 5120), torch.zeros(2, 5120)) + assert not PreparedReduceScatter.fits(_runtime(pipe_bytes=1 << 15), torch.zeros(8, 5120, dtype=torch.bfloat16), shard) + + +def test_gather_shards_fits_gathers_every_shard_row_and_chunked_gathers_accept_one_row_per_rank(monkeypatch): + from ignition.components.collectives import PreparedChunkedGatherShards, PreparedGatherShards + + _surfaces(monkeypatch) + runtime = _runtime() + shard = torch.zeros(2, 4, 5120, dtype=torch.bfloat16) + assert PreparedGatherShards.fits(runtime, shard, torch.zeros(8, 4, 5120, dtype=torch.bfloat16)) + assert not PreparedGatherShards.fits(runtime, shard, torch.zeros(4, 4, 5120, dtype=torch.bfloat16)) + assert not PreparedGatherShards.fits(_runtime(pipe_bytes=1 << 18), shard, + torch.zeros(8, 4, 5120, dtype=torch.bfloat16)) + assert PreparedChunkedGatherShards.fits(runtime, shard, torch.zeros(8, 4, 5120, dtype=torch.bfloat16)) + # One shard row per rank is the smallest chunk, so nothing below its message fits. + assert not PreparedChunkedGatherShards.fits(_runtime(pipe_bytes=4 * 40960 - 1), shard, + torch.zeros(8, 4, 5120, dtype=torch.bfloat16)) + + +@pytest.mark.parametrize("wrapper", ["PreparedReduceScatter", "PreparedGatherShards"]) +def test_single_rank_row_shard_operations_copy_the_source(monkeypatch, wrapper): + from ignition.components import collectives + + operation = getattr(collectives, wrapper) + source = torch.arange(8, dtype=torch.bfloat16).reshape(2, 4) + output = torch.empty_like(source) + prepared = operation(source, output, runtime=None, session=None, coordinator=None, name="row_shard") + assert prepared.single + assert prepared(source) is output + assert torch.equal(source, output) + with pytest.raises(ValueError, match="single-rank"): + operation(source, torch.empty(4, 4, dtype=torch.bfloat16), runtime=None, session=None, + coordinator=None, name="row_shard") + + +def test_prepared_row_shard_operations_request_their_surface_and_launch_the_runtime(prepared): + from ignition.components.collectives import PreparedGatherShards, PreparedReduceScatter + + surfaces, plan = prepared + runtime = _runtime() + source = torch.zeros(8, 5120, dtype=torch.bfloat16) + shard = torch.zeros(2, 5120, dtype=torch.bfloat16) + reduce = PreparedReduceScatter(source, shard, runtime=runtime, session=object(), + coordinator=SimpleNamespace(hosts=()), name="layer.attention_reduce") + gather_output = torch.zeros(8, 5120, dtype=torch.bfloat16) + gather = PreparedGatherShards(shard, gather_output, runtime=runtime, session=object(), + coordinator=SimpleNamespace(hosts=()), name="layer.attention_gather") + assert surfaces == ["AllReduce.reduce_scatter", "AllReduce.gather_shards"] + assert reduce.plan is plan and gather.plan is plan + assert reduce(source) is shard and gather(shard) is gather_output + assert runtime.calls == [("reduce_scatter", (8, 5120), (2, 5120)), + ("gather_shards", (2, 5120), (8, 5120))] + + +def test_chunked_gather_scatters_each_chunk_into_its_rank_major_window(prepared): + from ignition.components.collectives import PreparedChunkedGatherShards + + rows, world, columns = 5, 2, 8 + row_bytes = columns * 2 + # Two shard rows per gathered message, so the five shard rows arrive in three chunks. + runtime = _runtime(world=world, pipe_bytes=row_bytes * 2 * world) + + def gather(inp, *, out, plan): + # A gather publishes rank-major rows, so a single-rank test writes the shard to each rank. + out.view(world, -1, columns)[:] = inp + runtime.gather_shards = gather + shard = torch.arange(rows * columns, dtype=torch.bfloat16).reshape(rows, columns) + output = torch.zeros(rows * world, columns, dtype=torch.bfloat16) + operation = PreparedChunkedGatherShards(shard, output, runtime=runtime, session=object(), + coordinator=SimpleNamespace(hosts=()), name="layer.residual_gather") + assert operation.bounds == [(0, 2), (2, 4), (4, 5)] + assert operation(shard) is output + for rank in range(world): + assert torch.equal(output.view(world, rows, columns)[rank], shard) + + +def test_a_functional_chunked_gather_allocates_its_staging_and_output_in_the_forward(prepared): + """A captured graph owns every operand, so the wrapper declares shapes and allocates on call.""" + from ignition.components.collectives import PreparedChunkedGatherShards + + rows, world, columns = 2, 2, 8 + row_bytes = columns * 2 + + def gather(inp, *, out, plan): + out.view(world, -1, columns)[:] = inp + + runtime = _runtime(world=world, pipe_bytes=row_bytes * rows * world) + runtime.gather_shards = gather + shard = torch.arange(rows * columns, dtype=torch.bfloat16).reshape(rows, columns) + operation = PreparedChunkedGatherShards(shard, torch.zeros(rows * world, columns, dtype=torch.bfloat16), + runtime=runtime, session=object(), + coordinator=SimpleNamespace(hosts=()), name="layer.residual_gather") + assert operation.functional(session=object()) is operation + assert operation.output is None + first = operation(shard) + second = operation(shard) + assert first.shape == second.shape == (rows * world, columns) and first.dtype is torch.bfloat16 + assert first.data_ptr() != second.data_ptr() + for rank in range(world): + assert torch.equal(second.view(world, rows, columns)[rank], shard) diff --git a/tests/recipes/test_deepseek41_expert_overlap.py b/tests/recipes/test_deepseek41_expert_overlap.py index c3617cc5e..ac1ae42b7 100644 --- a/tests/recipes/test_deepseek41_expert_overlap.py +++ b/tests/recipes/test_deepseek41_expert_overlap.py @@ -62,6 +62,10 @@ def test_layer_consumes_standalone_router_results_with_runtime_image_mask(): layer = object.__new__(decoder.PreparedLayer) layer.engram = layer.concurrent_shared = None layer.has_image_mask = True + # Built without __init__, so give the layer an unsharded layer's shard state. + layer.row_shard, layer.reseed = None, False + layer.attention_gather = layer.ffn_gather = None + layer.residual_gather = layer.mix_gather = None residual = torch.arange(12, dtype=torch.float32).reshape(3, 4) metadata = SimpleNamespace(token_ids=torch.tensor([7, 129280, 9])) selection = (torch.zeros(3, 1, dtype=torch.int32), None, None) diff --git a/tests/recipes/test_deepseek41_row_shard.py b/tests/recipes/test_deepseek41_row_shard.py new file mode 100644 index 000000000..61dca28e8 --- /dev/null +++ b/tests/recipes/test_deepseek41_row_shard.py @@ -0,0 +1,271 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""A row-sharded prefill keeps one capture layout and falls back to full rows when it cannot.""" + +from types import SimpleNamespace + +import pytest +import torch + + +def _surfaces(monkeypatch, *, abi=11, pipe_bytes=1 << 26): + import b12x.comm.roce._proxy as proxy + from ignition.components import collectives + + monkeypatch.setattr(proxy, "ABI_VERSION", abi) + monkeypatch.setattr(collectives, "_roce", lambda runtime: True) + return SimpleNamespace(world_size=2, pipe_bytes=pipe_bytes, _group=object(), + should_reduce_scatter=lambda inp: True, should_gather_shards=lambda inp: True) + + +def test_row_shard_takes_contiguous_rank_rows_and_gates_on_world_and_call_rows(): + from ignition.recipes.deepseek41.decoder import MHC_CALL_ROWS, RowShard + + shard = RowShard.allocate(1024, 2, 1, "cpu") + assert (shard.world, shard.rows, MHC_CALL_ROWS) == (2, 512, 512) + assert shard.rows_index.tolist() == list(range(512, 1024)) + assert shard.residual.shape == (512, 4, 5120) and shard.residual.dtype is torch.bfloat16 + assert shard.pre_mix.shape == (512, 4) and shard.pre_mix.dtype is torch.float32 + assert shard.attention_hc.residual.shape == shard.ffn_hc.residual.shape == (512, 4, 5120) + assert shard.attention_hc.normalized.shape == (512, 5120) + assert RowShard.allocate(1024, 1, 0, "cpu") is None + assert RowShard.allocate(1025, 2, 0, "cpu") is None + # One rank's rows must hold whole mHC calls so its call shape matches the full-row pin. + assert RowShard.allocate(1024, 4, 0, "cpu") is None + + +def test_row_shard_graph_shrinks_capacity_and_rejects_rows_that_do_not_divide(): + from ignition.recipes.deepseek41.decoder import RowShard + + shard = RowShard.allocate(2048, 2, 0, "cpu") + smaller = shard.graph(1024) + assert smaller.rows == 512 and smaller.rows_index.tolist() == list(range(512)) + assert smaller.residual.shape == (512, 4, 5120) and smaller.attention_hc.normalized.shape == (512, 5120) + assert shard.graph(2048).rows == 1024 + # Every rank's block of a narrower graph starts at rank * rows of that graph, not of the + # allocated layout, and stays inside the graph's rows. + for rank in range(4): + wide = RowShard.allocate(8192, 4, rank, "cpu") + for tokens in (2048, 4096, 6144, 8192): + narrow = wide.graph(tokens) + rows = tokens // 4 + assert narrow.rows_index.tolist() == list(range(rank * rows, (rank + 1) * rows)) + assert wide.rows_index.tolist() == list(range(rank * 2048, (rank + 1) * 2048)) + assert shard.graph(1023) is None + assert shard.graph(1536) is None + assert shard.graph(1024 + 2) is None + + +def test_decoder_scratch_carries_a_row_shard_only_when_the_bootstrap_asks(): + from ignition.recipes.deepseek41.decoder import Scratch + + plain = Scratch.allocate(1024, 1, 2, "cpu", candidate_tokens=16) + assert plain.row_shard is None + asked = Scratch.allocate(1024, 1, 2, "cpu", candidate_tokens=16, + row_shard=SimpleNamespace(rank=1)) + assert asked.row_shard.rows == 512 and asked.row_shard.rows_index[0].item() == 512 + assert asked.rows(1024, 1).row_shard.rows == 512 + # A shard keeps whole 512-row mHC calls, so a narrower graph has no shard. + assert asked.rows(512, 1).row_shard is None + + +class _Recorder: + """A stand-in for a prepared component that returns the surface the forward reads.""" + + def __init__(self, value=None): + self.value, self.calls, self.kwargs = value, [], {} + + def __call__(self, *args, **kwargs): + self.calls.append(args) + self.kwargs = kwargs + return self.value + + def _run(self, *args, **kwargs): + self.calls.append(args) + self.kwargs = kwargs + return self.value + + +def _metadata(tokens=1024): + return SimpleNamespace(token_ids=torch.zeros(tokens, dtype=torch.int64), + positions=torch.arange(tokens)) + + +def _recording_layer(scratch, *, reseed=False): + """A `PreparedLayer` whose `_run` calls recorders, so a test can read the operands it passes.""" + from ignition.recipes.deepseek41 import decoder + + shard = scratch.row_shard + layer = object.__new__(decoder.PreparedLayer) + layer.scratch, layer.row_shard, layer.reseed = scratch, shard, reseed + layer.engram = layer.concurrent_shared = None + layer.image_mask, layer.has_image_mask = None, False + attended = shard.attention_output if shard is not None else scratch.attention_output + attention_branch = scratch.attention_hc if shard is None else shard.attention_hc + ffn_branch = scratch.ffn_hc if shard is None else shard.ffn_hc + layer.attention = _Recorder((attended, {}, None, {})) + layer.attention_hc = _Recorder((attention_branch.residual, attention_branch.post, + attention_branch.comb, attention_branch.normalized)) + layer.ffn_hc = _Recorder((ffn_branch.residual, ffn_branch.post, ffn_branch.comb, ffn_branch.normalized)) + layer.post = None if shard is None else _Recorder(shard.residual) + layer.shared = _Recorder(scratch.shared.output) + layer.attention_reduce = _Recorder(attended) + layer.mlp_reduce = _Recorder() + layer.router_projection = _Recorder(scratch.router_logits) + layer.router = _Recorder(SimpleNamespace(ids=torch.zeros(1, dtype=torch.int32), weights=torch.zeros(1), + selected_logits=None)) + layer.experts = _Recorder(scratch.routed) + if shard is None: + layer.attention_gather = layer.ffn_gather = None + layer.residual_gather = layer.mix_gather = None + else: + layer.attention_gather, layer.ffn_gather = _Recorder(), _Recorder() + layer.residual_gather, layer.mix_gather = _Recorder(), _Recorder() + return layer + + +@pytest.mark.parametrize("row_shard", [False, True]) +def test_the_forward_normalizes_the_ffn_from_the_attention_branch_state(row_shard): + """A prepared mHC rewrites the residual, post and comb surfaces it reads. + + The feed-forward mHC must therefore take the attention branch's surfaces — the shard's copy + when the layer shards, the full scratch when it does not — which the attention mHC wrote on + this call. The feed-forward scratch still holds the previous layer's post until this call + writes it, so passing its surfaces normalizes a stale mix. + """ + from ignition.recipes.deepseek41 import decoder + + scratch = decoder.Scratch.allocate(1024, 8, 2, "cpu", + row_shard=SimpleNamespace(rank=1) if row_shard else None) + branch = scratch.attention_hc if scratch.row_shard is None else scratch.row_shard.attention_hc + output = scratch.attention_output if scratch.row_shard is None else scratch.row_shard.attention_output + residual_source = scratch.residual if scratch.row_shard is None else scratch.row_shard.residual + mix_source = scratch.pre_mix if scratch.row_shard is None else scratch.row_shard.pre_mix + layer = _recording_layer(scratch) + layer._run((residual_source,), mix_source, _metadata(), None, None) + attention_output, residual, post, comb = layer.ffn_hc.calls[0] + assert attention_output.data_ptr() == output.data_ptr() + for received, expected in zip((residual, post, comb), + (branch.residual, branch.post, branch.comb), strict=True): + assert received.data_ptr() == expected.data_ptr() + + +def test_a_sharded_forward_gathers_the_lagged_mix_through_its_bf16_view(): + """The mix gather was prepared over BF16 packs of the FP32 lagged mix, so it takes that view.""" + from ignition.recipes.deepseek41 import decoder + + scratch = decoder.Scratch.allocate(1024, 8, 2, "cpu", row_shard=SimpleNamespace(rank=1)) + shard = scratch.row_shard + layer = _recording_layer(scratch) + _, full_mix, _, _, _ = layer._run((shard.residual,), shard.pre_mix, _metadata(), None, None) + mix, gathered = layer.mix_gather.calls[0] + assert mix.dtype is torch.bfloat16 and mix.shape == (shard.rows, 8) + assert mix.is_contiguous() and mix.stride() == (8, 1) + assert gathered.dtype is torch.bfloat16 and gathered.shape == (1024, 8) + assert full_mix.dtype is torch.float32 and full_mix.shape == (1024, 4) + (residual,) = layer.residual_gather.calls[0] + assert residual.data_ptr() == shard.residual.data_ptr() + + +def test_a_resuming_shard_refills_from_the_rows_its_rank_owns(): + """The full-row layers own the rows in between, so a resuming shard takes its own rows out. + + Taking rows from zero instead would hand rank one another rank's state. Each 128-row block + of the full residual and lagged mix carries its own value, so the two differ. + """ + from ignition.recipes.deepseek41 import decoder + + scratch = decoder.Scratch.allocate(1024, 8, 2, "cpu", row_shard=SimpleNamespace(rank=1)) + shard = scratch.row_shard + blocks = (torch.arange(1024) // 128).to(torch.float32) + scratch.residual.copy_(blocks[:, None, None].expand(1024, 4, 5120)) + scratch.pre_mix.copy_(blocks[:, None].expand(1024, 4)) + layer = _recording_layer(scratch, reseed=True) + layer._run((scratch.residual,), scratch.pre_mix, _metadata(), None, None) + (refilled,) = layer.attention_hc.calls[0] + assert torch.equal(refilled, scratch.residual.index_select(0, shard.rows_index)) + assert torch.equal(layer.attention_hc.kwargs["pre_mix"], + scratch.pre_mix.index_select(0, shard.rows_index)) + assert not torch.equal(refilled, scratch.residual[:shard.rows]) + + +@pytest.mark.parametrize("abi,pipe_bytes,ready", [ + (11, 1 << 26, True), (10, 1 << 26, False), (11, 1 << 22, False)]) +def test_row_shard_ready_requires_the_abi_and_capacity_for_every_surface(monkeypatch, abi, pipe_bytes, ready): + from ignition.recipes.deepseek41.decoder import RowShard, row_shard_ready + + runtime = _surfaces(monkeypatch, abi=abi, pipe_bytes=pipe_bytes) + rows = 512 + scratch = SimpleNamespace( + attention=SimpleNamespace(output=torch.zeros(1024, 5120, dtype=torch.bfloat16)), + attention_hc=SimpleNamespace(normalized=torch.zeros(1024, 5120, dtype=torch.bfloat16)), + ffn_hc=SimpleNamespace(normalized=torch.zeros(1024, 5120, dtype=torch.bfloat16)), + mlp_partial=torch.zeros(1024, 5120, dtype=torch.bfloat16), + pre_mix=torch.zeros(1024, 4, dtype=torch.float32), + residual=torch.zeros(1024, 4, 5120, dtype=torch.bfloat16)) + shard = SimpleNamespace( + attention_output=torch.zeros(rows, 5120, dtype=torch.bfloat16), + mlp_output=torch.zeros(rows, 5120, dtype=torch.bfloat16), + attention_hc=SimpleNamespace(normalized=torch.zeros(rows, 5120, dtype=torch.bfloat16)), + ffn_hc=SimpleNamespace(normalized=torch.zeros(rows, 5120, dtype=torch.bfloat16)), + pre_mix=torch.zeros(rows, 4, dtype=torch.float32), + residual=torch.zeros(rows, 4, 5120, dtype=torch.bfloat16)) + assert shard.pre_mix.view(torch.bfloat16).shape == (rows, 8) + assert row_shard_ready(scratch, shard, runtime) is ready + + +def test_every_row_count_a_graph_can_use_is_one_segment_of_the_table(): + """One allocation holds every segment, so no narrower graph needs storage of its own. + + The table is int64, and each graph's row numbers are a view at that row count's segment + offset: 512 rows at 0, then 1,024 at 512, 1,536 at 1,536 and 2,048 at 3,072. + """ + from ignition.recipes.deepseek41.decoder import RowShard + + captured = RowShard.allocate(8192, 4, 1, "cpu") + assert captured.rows_table.dtype is torch.int64 + assert captured.rows_index.data_ptr() == captured.rows_table.data_ptr() + 3072 * 8 + assert captured.rows_index.tolist() == list(range(2048, 4096)) + offsets = {8192: 3072, 6144: 1536, 4096: 512, 2048: 0} + for tokens, first in offsets.items(): + graph = captured.graph(tokens) + assert graph.rows_table is captured.rows_table + assert graph.rows_index.storage_offset() == first + rows = tokens // 4 + assert graph.rows_index.tolist() == list(range(rows, 2 * rows)) + assert graph.rows_index.data_ptr() == captured.rows_table.data_ptr() + first * 8 + + +def test_the_row_table_is_written_once_into_persistent_storage(monkeypatch): + """Prefill scratch exports as zeros, so the table takes the persistent arena instead.""" + from ignition.recipes.deepseek41 import decoder + + calls = [] + + def persistent(shape, dtype, device, **kwargs): + calls.append((tuple(shape), dtype, device, kwargs)) + return torch.zeros(shape, dtype=dtype) + + monkeypatch.setattr(decoder, "persistent_zeros", persistent) + shard = decoder.RowShard.allocate(4096, 4, 1, "cpu") + assert calls == [((1536,), torch.int64, "cpu", {})] + assert shard.rows_table.tolist() == [*range(512, 1024), *range(1024, 2048)] + assert shard.residual.dtype is torch.bfloat16 + + +def test_every_rank_binds_row_shard_tables_at_one_storage_offset(): + """One compiled forward binds every rank, and binding compares storage offsets. + + Each graph's row numbers therefore sit at the same offset on every rank; only their contents + differ. + """ + from ignition.recipes.deepseek41.decoder import RowShard + + shards = [RowShard.allocate(8192, 4, rank, "cpu") for rank in range(4)] + for tokens in (8192, 6144, 4096, 2048): + graphs = [shard.graph(tokens) for shard in shards] + rows = tokens // 4 + assert len({graph.rows_index.storage_offset() for graph in graphs}) == 1 + for rank, graph in enumerate(graphs): + assert graph.rows_index.tolist() == list(range(rank * rows, (rank + 1) * rows)) diff --git a/tests/recipes/test_deepseek41_row_shard_release.py b/tests/recipes/test_deepseek41_row_shard_release.py new file mode 100644 index 000000000..59956521d --- /dev/null +++ b/tests/recipes/test_deepseek41_row_shard_release.py @@ -0,0 +1,35 @@ +"""Row-shard declaration scratch must not survive prepared-owner finalization.""" + +import gc +from types import SimpleNamespace +import weakref + +import pytest +import torch + +from ignition.recipes.deepseek41.decoder import PreparedDecoder, PreparedLayer, RowShard + + +@pytest.mark.parametrize("owner_type,release", [ + (PreparedLayer, "release_examples"), (PreparedDecoder, "release_outputs")]) +def test_prepared_owner_releases_row_shard_scratch(owner_type, release): + shard = RowShard.allocate(1024, 2, 1, "cpu") + examples = [weakref.ref(value) for value in ( + shard.attention_output, shard.mlp_output, shard.residual, shard.pre_mix, shard.attn_mix, + *shard.attention_hc.bind_args().values(), *shard.ffn_hc.bind_args().values())] + rows_index = shard.rows_index + prepared = object.__new__(owner_type) + prepared.row_shard = shard + prepared.scratch = SimpleNamespace(row_shard=shard) + if owner_type is PreparedDecoder: + prepared.output_scratch = prepared.scratch + prepared.concurrent_shared = None + getattr(prepared, release)() + del shard + gc.collect() + assert all(example() is None for example in examples) + assert prepared.row_shard.rows_index is rows_index + assert torch.equal(rows_index, torch.arange(512, 1024)) + for name in ("attention_output", "mlp_output", "residual", "pre_mix", "attn_mix", + "attention_hc", "ffn_hc"): + assert getattr(prepared.row_shard, name) is None diff --git a/tests/recipes/test_deepseek41_shard_schedule.py b/tests/recipes/test_deepseek41_shard_schedule.py new file mode 100644 index 000000000..00714da28 --- /dev/null +++ b/tests/recipes/test_deepseek41_shard_schedule.py @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Row-shard scheduling across the Engram layers of a DeepSeek V4.1 prefill decoder. + +An Engram layer reads and writes full rows, so it runs full-row. Because the row dimension +therefore changes twice inside the decoder, the shard has to end before such a layer and resume +after it, and only a layer that runs its own post may change the dimension. +""" + +from dataclasses import dataclass + +from ignition.recipes.deepseek41.decoder import shard_schedule + + +@dataclass(frozen=True) +class _Layer: + engram: object = None + + +SHARD = object() + + +def layers(count=40, engram=()): + return tuple(_Layer("engram" if index in engram else None) for index in range(count)) + + +def flags(schedule, column): + return {index for index, entry in enumerate(schedule) if entry[column]} + + +def test_shard_resumes_after_each_engram_layer(): + schedule = shard_schedule(layers(engram=(1, 14)), 19, SHARD) + assert flags(schedule, 0) == set(range(19)) - {1, 14} + + +def test_shard_refills_from_full_rows_where_it_resumes(): + schedule = shard_schedule(layers(engram=(1, 14)), 19, SHARD) + assert flags(schedule, 2) == {2, 15} + + +def test_only_same_row_layers_defer_their_post(): + schedule = shard_schedule(layers(engram=(1, 14)), 19, SHARD) + # Layers 19-38 defer their post while both neighbouring layers remain full-row. + assert flags(schedule, 1) == {*range(2, 13), *range(15, 18), *range(19, 39)} + + +def test_engram_layer_itself_stays_full_row(): + schedule = shard_schedule(layers(engram=(1, 14)), 19, SHARD) + for index in (1, 14): + assert schedule[index][0] is False + assert schedule[index][1] is False + + +def test_tail_from_stop_on_stays_full_row(): + schedule = shard_schedule(layers(engram=(1, 14)), 19, SHARD) + assert not any(entry[0] for entry in schedule[19:]) + + +def test_engram_first_layer_does_not_disable_the_rest(): + schedule = shard_schedule(layers(engram=(0,)), 19, SHARD) + assert flags(schedule, 0) == set(range(1, 19)) + assert flags(schedule, 2) == {1} + assert schedule[0][0] is False + + +def test_aux_layer_runs_full_row_and_restarts_the_shard(): + schedule = shard_schedule(layers(), 19, SHARD, aux_layers=(10,)) + assert 10 not in flags(schedule, 0) + assert 9 not in flags(schedule, 1) + assert 11 in flags(schedule, 2) + + +def test_stop_below_one_shards_nothing(): + schedule = shard_schedule(layers(engram=(1, 14)), 0, SHARD) + assert not any(entry[0] for entry in schedule) + assert not any(entry[2] for entry in schedule) + + +def test_without_a_shard_every_layer_keeps_the_legacy_defer(): + schedule = shard_schedule(layers(engram=(1, 14)), 19, None) + expected = [index + 1 not in (1, 14) for index in range(39)] + [False] + assert [entry[1] for entry in schedule] == expected + assert not any(entry[0] or entry[2] for entry in schedule) From da5ec45c1af7aaf46dea20a04c2746fa3d5b1c22 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sat, 14 Nov 2026 23:28:24 -0600 Subject: [PATCH 2/2] Keep DeepSeek V4.1 prefill attention on rank-owned rows Tensor-parallel prefill gathers normalized rows before attention and reduces its output afterward. Compute all attention heads and output groups for each rank's own rows, then gather the cache projections required by full-row consumers. This removes those exchanges while preserving complete cache writes. Load replicated query/output projections only for eligible prefill layers, retain partitioned decode copies, and carry selection state across row transitions. Account for values, both retained MXFP8 scale layouts and FP32 sinks: 17 layers add 2,121,273,600 numeric bytes per rank. Preserve main's metadata and online-conversion options. Document the qualified hardware and retained-core scope separately from unrun dependency/mode configurations. The benefit is less attention communication at a retained-weight cost. Grouping all eight output groups changes BF16 rounding association; comparisons use operator-derived bounds rather than exact partitioned-output equality. Validation includes accounting/layout tests and combined-stack GPU/native checks. On 32 captured native rows, all 163,840 projection outputs satisfy the unchanged operator budget and match native BF16 bits. Existing 7b35510b/ABI-14 cores pass checkpoint/Restore, inactive-state, replay and allocation checks; 4fd836dc component/compiler checks pass separately. Floating cache differences, baseline-reproduced replay failures and measured dependency scopes remain explicit; no fresh default-dependency whole-model export or isolated speedup is claimed. Computational source is unchanged from the reviewed 8d38f298 tree. Design-Signed-Off-By: Codex (GPT-6); design-review=ed22 Signed-off-by: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> --- docs/TODO.txt | 6 + docs/cli-design.md | 1 + docs/model-preparation.md | 50 ++ helios/proto/runtime_core.proto | 2 + ignition/cli/__init__.py | 4 + ignition/cli/bench.py | 2 +- ignition/cli/preparation.py | 5 +- ignition/cli/serve.py | 14 +- ignition/planning.py | 34 +- ignition/preparation/rebuild_manifest.py | 2 + ignition/preparation/runtime_core_pb2.py | 352 +++++----- ignition/recipes/declaration.py | 21 +- ignition/recipes/deepseek41/attention.py | 203 ++++-- ignition/recipes/deepseek41/bootstrap.py | 24 +- ignition/recipes/deepseek41/config.py | 127 +++- ignition/recipes/deepseek41/decoder.py | 235 ++++++- ignition/recipes/deepseek41/recipe.py | 3 + ignition/recipes/deepseek41/weights.py | 45 +- tests/cli/test_planning.py | 113 +++ .../test_attention_output_rank_sum_cuda.py | 224 ++++++ tests/recipes/test_deepseek41_dspark_cuda.py | 1 + .../recipes/test_deepseek41_expert_overlap.py | 2 + .../test_deepseek41_preparation_lifetimes.py | 124 ++++ tests/recipes/test_deepseek41_row_shard.py | 3 + .../test_deepseek41_row_shard_attention.py | 654 ++++++++++++++++++ .../recipes/test_deepseek41_shard_schedule.py | 30 +- 26 files changed, 1954 insertions(+), 327 deletions(-) create mode 100644 tests/components/test_attention_output_rank_sum_cuda.py create mode 100644 tests/recipes/test_deepseek41_preparation_lifetimes.py create mode 100644 tests/recipes/test_deepseek41_row_shard_attention.py diff --git a/docs/TODO.txt b/docs/TODO.txt index f8b00a219..9c1df3e13 100644 --- a/docs/TODO.txt +++ b/docs/TODO.txt @@ -16,6 +16,12 @@ host memory admission. Its nine-worker assertion requires 296 GiB available under the 32 GiB worker allowance and 8 GiB headroom; smaller hosts admit fewer. +[ ] Extend DeepSeek V4.1 row-sharded-attention validation beyond the retained + SM121 TP4 native profile: fresh default-dependency whole-model preparation, + other supported GPU/TP geometries, scored-prefill modes, and remaining + shard-tail capacities. Use operator-specific bounds and complete state checks. + See docs/model-preparation.md for the qualified execution scope. + [ ] Ensure Rust bridge updates relink the WinUI executable during incremental builds. MSBuild can retain flare-winui.exe after the bridge library changes; verify loose and single-file outputs without requiring /t:Rebuild. diff --git a/docs/cli-design.md b/docs/cli-design.md index 2070b7304..ffc0974d4 100644 --- a/docs/cli-design.md +++ b/docs/cli-design.md @@ -462,6 +462,7 @@ explicit flags take precedence. | `--joint-tuning` | Jointly select DeepSeek V4.1 QKV projection, query normalization, and query expansion plans by timing complete prepared layers across every prefill, decode, packed verification, and draft shape. Requires MXFP8 query normalization and autotuning enabled. Command line only. | off | | `--shard-hyperconnections` | Partition Qwen3.8 Flash target and MTP hyperconnection bottlenecks across TP ranks and all-reduce BF16 gate logits before sigmoid. Requires TP2 or TP4. Command line only; changing it requires re-export. | off (replicated) | | `--row-shard-prefill` | Replace each DeepSeek V4.1 prefill tensor-parallel all-reduce with a reduce-scatter, run the hyperconnection on the rank's contiguous row shard, and gather the normalized branch input back to every rank. The collective pair preserves the pipelined two-shot reduction's arithmetic; whole-decoder numerical equivalence remains unqualified. Preparation requires the runtime's ABI 11 or later row-shard surfaces; the proxy pinned in this tree records ABI 14. Requires TP2 or more; three ranks and SM103 are refused, and a layer whose row count or pipe capacity does not fit keeps the all-reduce path. Command line only; changing it requires re-export. | off | +| `--row-shard-attention` | Also run DeepSeek V4.1 prefill attention on the rank's row shard: every rank loads all 64 attention heads and all 8 output groups for the layers a prepared prefill runs on the shard, which the plan charges as resident weights (17 layers, 2.12 GB per rank at TP4 by default; 37, 4.62 GB with `--prompt-logprobs`; 35, 4.37 GB for scored `--mtp-only`), gathers the KV latent and the compressed-latent projections so its caches hold every row, and needs no attention reduction. Changes reduction rounding relative to the all-reduce path; see the [DeepSeek V4.1 preparation guide](model-preparation.md#deepseek-v41-flash). Requires `--row-shard-prefill`; a layer whose gathers do not fit keeps the partitioned path. Command line only; changing it requires re-export. | off | | `--collective-ladder PATH` | JSON file of final joint collective geometries and runtime profiles. Required when the resolved collective tuning policy is `runtime`; rejected with `static`. See [offline ladders](../ignition/preparation/ladders/README.md). | None | | `--collective-tuning static\|runtime` | Static autotunes collectives during preparation; runtime uses the required ladder file for final compiled geometry and captures runtime submode/active-CTA selection for eligible BF16 PCIe reductions. Overrides `[preparation] collective_tuning`, then the recipe declaration. | Static | | `--lock-gpu-clocks` | Lock each tuning GPU at its maximum supported graphics frequency during b12x and Inductor autotuning, then reset driver clock defaults. Frozen warmup, capture, and serving remain unlocked. Requires root or noninteractive sudo access to `nvidia-smi`. Command line only; inactive with `--autotune-effort off`. | off | diff --git a/docs/model-preparation.md b/docs/model-preparation.md index ea2c33aeb..e874bbf8e 100644 --- a/docs/model-preparation.md +++ b/docs/model-preparation.md @@ -599,6 +599,56 @@ seams determine the measured contexts. The option defaults to off and requires MXFP8 query normalization on the selected GPUs and `--autotune-effort default` or `audit`. It adds preparation time without changing the serving graph catalog. +`ignition export deepseek-v4.1-flash --row-shard-prefill --row-shard-attention` +computes prefill attention on the row shard the reduced prefill already leaves on +each rank. At TP4, a layer that a prepared prefill runs on the shard loads all 64 +attention heads and all 8 output groups of its query and output projections +beside the rank's partitioned 16 heads and 2 groups. Decode, the 128-row suffix +layers, and layers without sharded prefill use the partitioned projections. +Packed prefill stops at the decoder boundary, so 17 of the 40 backbone layers +hold replicated projections and attention sinks, totaling 2.12 GB per rank: 1.59 +GB outside that rank's partition and 0.53 GB for an additional copy of its +partition. Each replicated layer retains 124,780,800 bytes: 112 MiB of MXFP8 +values, two 3.5 MiB scale representations, and 256 bytes of FP32 sinks. Scored +prefill replays no suffix and reaches every layer below the last, so +`--prompt-logprobs` raises the totals to 37 layers and 4.62 GB: 3.46 GB outside +the partition and 1.15 GB duplicating it. A draft-only plan builds no prefill +graph without the draft, and those graphs are what keep a draft target layer on +full rows, so the scored plan under `--mtp-only` replicates 35 layers and 4.37 +GB. The plan's per-rank weight estimate charges the same layers, so the page +budget it sizes leaves room for them instead of spending it on KV pages. The +weight estimate determines planned page, context, and snapshot capacities; +changing those capacities requires re-export. Existing cores retain their +captured storage and capacities. Each rank gathers the 512-wide KV latent plus +the two compressed-latent projection inputs before the full-row window, +index, and compressed-latent cache writes. This preserves full-row cache +coverage; numerical comparisons use the operator-specific bounds described below. The +attention output of a rank's own query rows is complete once +the grouped output projection has summed all eight groups, so no attention +reduction runs at all. That sum happens in one kernel here, where the +partitioned path sums four rank partials through an FP32 all-reduce, so prefill +logits are expected to agree with the all-reduce path to BF16 rounding rather +than exactly; comparisons require operator-specific rounding bounds. The option requires `--row-shard-prefill`, is +recorded in the plan and the core's rebuild metadata, and changes prepared weight +layouts, so changing it requires a fresh export and `--reuse-weights` requires +matching layouts; `--prompt-logprobs` and `--mtp-only` select the replicated +layers too and carry the same requirements. A prefill capacity whose shard rows +or gathers do not fit keeps the partitioned attention path for that graph and +prints one preparation line naming the capacity. The replicated weights load for +the layers a prefill kind reaches, not for the capacities that shard, so a +capacity that keeps the partitioned path leaves them resident; disabling the +option and re-exporting is the only way to recover that memory. Row-sharded +attention has compiled collective and exported-core native validation on four +SM121 GB10 hosts with the retained FlashInfer `7b35510b`/ABI-14 cores. Checks +cover partial prefill, exact checkpoint/Restore and inactive-state preservation, +replay, scratch reuse, and retained-memory/allocation stability. GPU component +and compiler-boundary checks also pass with the pinned FlashInfer `4fd836dc`; +a whole-model core prepared with that revision has not been exported for these +checks. The grouped output projection satisfies the tested operator rounding +budget; floating cache differences and baseline replay variation remain visible. +These results do not qualify other hardware/TP configurations, scored-prefill +modes, or every captured shape; follow-up checks are tracked in docs/TODO.txt. + `--speculative-token-capacity` bounds physical target rows; `--speculative-graph-budget` independently controls the number of packed rounds. For each power-of-two request bucket `B`, the catalog retains capacities diff --git a/helios/proto/runtime_core.proto b/helios/proto/runtime_core.proto index 603c682d7..672e4ac5c 100644 --- a/helios/proto/runtime_core.proto +++ b/helios/proto/runtime_core.proto @@ -116,6 +116,8 @@ message RebuildManifest { bool force_moe_decode_a16 = 27; // Preparation-only row-sharded prefill collectives; requires proxy ABI 11 or later. bool row_shard_prefill = 28; + // Preparation-only row-sharded prefill attention with replicated attention weights. + bool row_shard_attention = 29; // Preparation-only online conversion group; omitted means all eligible projections. string online_mxfp8_scope = 30; // Preparation context group size; zero/absent retains full-TP context sharding when enabled. diff --git a/ignition/cli/__init__.py b/ignition/cli/__init__.py index a7a27be09..50300e40b 100644 --- a/ignition/cli/__init__.py +++ b/ignition/cli/__init__.py @@ -236,6 +236,10 @@ def _preparation_arguments(running, *, model_default="defaults.model"): help="reduce DeepSeek V4.1 prefill into one contiguous row shard per rank, run the" " hyperconnection on that shard, and gather the branch input every row reads" " (default: tensor-parallel all-reduce)") + advanced.add_argument("--row-shard-attention", action="store_true", + help="also run DeepSeek V4.1 prefill attention on that row shard, replicating every" + " head and output group on every rank and gathering the KV latent instead of" + " the attention output; requires --row-shard-prefill (default: off)") advanced.add_argument("--lock-gpu-clocks", action="store_true", help="lock each tuning GPU to its maximum supported graphics clock during" " autotuning; requires root or noninteractive sudo for nvidia-smi") diff --git a/ignition/cli/bench.py b/ignition/cli/bench.py index 6a218bdcd..82f438914 100644 --- a/ignition/cli/bench.py +++ b/ignition/cli/bench.py @@ -166,7 +166,7 @@ def run(args): lock_gpu_clocks=args.lock_gpu_clocks, native_dense_activations=args.native_dense_activations, force_moe_decode_a16=args.force_moe_decode_a16, shard_hyperconnections=args.shard_hyperconnections, joint_tuning=args.joint_tuning, - row_shard_prefill=args.row_shard_prefill, + row_shard_prefill=args.row_shard_prefill, row_shard_attention=args.row_shard_attention, image_concurrency=args.image_concurrency, mixed_prefill=args.mixed_prefill, prompt_logprobs=args.prompt_logprobs, expert_scale_compression=args.expert_scale_compression, online_mxfp8=args.online_mxfp8, online_mxfp8_scope=args.online_mxfp8_scope, diff --git a/ignition/cli/preparation.py b/ignition/cli/preparation.py index fecedb919..54192e44b 100644 --- a/ignition/cli/preparation.py +++ b/ignition/cli/preparation.py @@ -52,6 +52,7 @@ def derive(configuration: Configuration, model: ResolvedModel, host: Host, *, gp image_concurrency: int | None = None, mla_compute_mode: str | None = None, lock_gpu_clocks: bool = False, native_dense_activations: bool = False, joint_tuning: bool = False, shard_hyperconnections: bool = False, row_shard_prefill: bool = False, + row_shard_attention: bool = False, force_moe_decode_a16: bool | None = None) -> planning.Plan: """Select GPUs and estimate memory for one checkpoint from the configuration and command-line intent.""" from ignition.preparation.collective_ladder import resolve as resolve_ladder @@ -66,13 +67,13 @@ def derive(configuration: Configuration, model: ResolvedModel, host: Host, *, gp stream_overlap=stream_overlap, roce_ring=roce_ring, roce_chain=roce_chain, mixed_prefill=mixed_prefill, prompt_logprobs=prompt_logprobs, expert_scale_compression=expert_scale_compression, image_concurrency=image_capacity_setting(configuration, model, image_concurrency), - online_mxfp8=online_mxfp8, online_mxfp8_scope=online_mxfp8_scope, language_model_only=language_model_only, mla_compute_mode=mla_compute_mode) + online_mxfp8=online_mxfp8, online_mxfp8_scope=online_mxfp8_scope, language_model_only=language_model_only, mla_compute_mode=mla_compute_mode, + row_shard_prefill=row_shard_prefill, row_shard_attention=row_shard_attention) result = replace(result, compile_workers=workers, hub_cache=configuration.directory("hub_cache"), autotune_effort=tuning_effort(configuration, autotune_effort), collective_tuning=mode, collective_ladder=ladder, force_moe_decode_a16=configuration.get("preparation.force_moe_decode_a16", force_moe_decode_a16).value, joint_tuning=joint_tuning, shard_hyperconnections=shard_hyperconnections, - row_shard_prefill=row_shard_prefill, lock_gpu_clocks=lock_gpu_clocks, native_dense_activations=native_dense_activations) return tuning_devices(result, host, tuning_gpus) diff --git a/ignition/cli/serve.py b/ignition/cli/serve.py index 971275740..009d585b7 100644 --- a/ignition/cli/serve.py +++ b/ignition/cli/serve.py @@ -201,7 +201,8 @@ def _export_hosts(args): expert_scale_compression=args.expert_scale_compression, online_mxfp8=args.online_mxfp8, online_mxfp8_scope=args.online_mxfp8_scope, language_model_only=args.language_model_only, - mla_compute_mode=args.mla_compute_mode) + mla_compute_mode=args.mla_compute_mode, + row_shard_prefill=args.row_shard_prefill, row_shard_attention=args.row_shard_attention) collective_mode = collective_tuning_mode(configuration, model, args.collective_tuning) plan = replace(plan, compile_workers=compiler_workers(configuration, args.compile_workers), hub_cache=hub_cache, autotune_effort=tuning_effort(configuration, args.autotune_effort), @@ -209,8 +210,7 @@ def _export_hosts(args): collective_ladder=resolve_collective_ladder(collective_mode, args.collective_ladder), lock_gpu_clocks=args.lock_gpu_clocks, native_dense_activations=args.native_dense_activations, force_moe_decode_a16=configuration.get("preparation.force_moe_decode_a16", args.force_moe_decode_a16).value, - shard_hyperconnections=args.shard_hyperconnections, joint_tuning=args.joint_tuning, - row_shard_prefill=args.row_shard_prefill) + shard_hyperconnections=args.shard_hyperconnections, joint_tuning=args.joint_tuning) environment.apply(configuration, local, plan.recipe.b12x_environment) options = _server_options(configuration, model, plan) options.pop("api_key") @@ -278,7 +278,7 @@ def __call__(self, message): lock_gpu_clocks=args.lock_gpu_clocks, native_dense_activations=args.native_dense_activations, force_moe_decode_a16=configuration.get("preparation.force_moe_decode_a16", args.force_moe_decode_a16).value, shard_hyperconnections=args.shard_hyperconnections, joint_tuning=args.joint_tuning, - row_shard_prefill=args.row_shard_prefill, + row_shard_prefill=args.row_shard_prefill, row_shard_attention=args.row_shard_attention, image_concurrency=args.image_concurrency, speculation=speculation(args, model, configuration), prefill_chunk=args.prefill_chunk_size, context_parallel=context_parallel(args, model), context_parallel_size=getattr(args, "dcp_size", None), @@ -469,7 +469,8 @@ def debug_serve(args) -> int: cache_states=args.prefix_cache_states, mixed_prefill=args.mixed_prefill, prompt_logprobs=args.prompt_logprobs, expert_scale_compression=args.expert_scale_compression, online_mxfp8=args.online_mxfp8, online_mxfp8_scope=args.online_mxfp8_scope, - language_model_only=args.language_model_only, mla_compute_mode=args.mla_compute_mode) + language_model_only=args.language_model_only, mla_compute_mode=args.mla_compute_mode, + row_shard_prefill=args.row_shard_prefill, row_shard_attention=args.row_shard_attention) collective_mode = collective_tuning_mode(configuration, model, args.collective_tuning) plan = replace(plan, compile_workers=compiler_workers(configuration, args.compile_workers), hub_cache=hub_cache, autotune_effort=tuning_effort(configuration, args.autotune_effort), @@ -477,8 +478,7 @@ def debug_serve(args) -> int: collective_ladder=resolve_collective_ladder(collective_mode, args.collective_ladder), lock_gpu_clocks=args.lock_gpu_clocks, native_dense_activations=args.native_dense_activations, force_moe_decode_a16=configuration.get("preparation.force_moe_decode_a16", args.force_moe_decode_a16).value, - shard_hyperconnections=args.shard_hyperconnections, joint_tuning=args.joint_tuning, - row_shard_prefill=args.row_shard_prefill) + shard_hyperconnections=args.shard_hyperconnections, joint_tuning=args.joint_tuning) if rendezvous is None: plan = tuning_devices(plan, host, args.tuning_gpus) local = plan.gpus diff --git a/ignition/planning.py b/ignition/planning.py index f37c54829..c1d5e8690 100644 --- a/ignition/planning.py +++ b/ignition/planning.py @@ -195,6 +195,9 @@ class Plan: # Prefill replaces each tensor-parallel all-reduce with a reduce-scatter, an mHC on one row # shard, and a shard gather when the RoCE runtime serves those surfaces. row_shard_prefill: bool = False + # Row-sharded prefill also computes attention on the shard, which replicates every attention + # head and output group on every rank and gathers the KV latent instead of the branch input. + row_shard_attention: bool = False lock_gpu_clocks: bool = False native_dense_activations: bool = False force_moe_decode_a16: bool = False @@ -279,7 +282,8 @@ def __post_init__(self): if self.world_size < 2: raise UsageError("--row-shard-prefill requires tensor parallelism") # The recipe's 4,096-row chunk cannot be divided evenly among three ranks. - # A 6,144-row chunk would divide evenly. + # A 6,144-row chunk would divide evenly, but attention sharding also requires + # an even partition of the eight head blocks. if self.world_size == 3: raise UsageError("--row-shard-prefill has no TP3 row partition; select four GPUs") # The collectives are architecture independent, but the projections and experts on @@ -287,6 +291,8 @@ def __post_init__(self): if any(gpu.capability == (10, 3) for gpu in self.gpus): raise UsageError("--row-shard-prefill is not qualified on SM103;" " select SM120 or SM121 GPUs") + if self.row_shard_attention and not self.row_shard_prefill: + raise UsageError("--row-shard-attention requires --row-shard-prefill") if self.joint_tuning: if self.recipe.name != "deepseek41": raise UsageError("--joint-tuning requires DeepSeek V4.1") @@ -407,6 +413,7 @@ def to_dict(self) -> dict: "joint_tuning": self.joint_tuning, "shard_hyperconnections": self.shard_hyperconnections, "row_shard_prefill": self.row_shard_prefill, + "row_shard_attention": self.row_shard_attention, "lock_gpu_clocks": self.lock_gpu_clocks, "native_dense_activations": self.native_dense_activations, "force_moe_decode_a16": self.force_moe_decode_a16, @@ -450,7 +457,7 @@ def fit(recipe: Recipe, checkpoint: Checkpoint, gpus: tuple[Gpu, ...], *, concur roce_chain: bool = False, image_concurrency: int | None = None, online_mxfp8: bool = False, online_mxfp8_scope: str | None = None, language_model_only: bool = False, mla_compute_mode: str | None = None, - row_shard_prefill: bool = False) -> Plan: + row_shard_prefill: bool = False, row_shard_attention: bool = False) -> Plan: """Size pages and the prefix cache to the supply; the result may not fit. Weights come from the checkpoint's stored tensors and graphs and scratch from @@ -516,12 +523,15 @@ def fit(recipe: Recipe, checkpoint: Checkpoint, gpus: tuple[Gpu, ...], *, concur costs = replace(costs, graph_bytes=costs.graph_bytes - credit) plan = Plan(recipe, checkpoint, gpus, concurrency, recipe.prefill_chunk if prefill_chunk is None else prefill_chunk, speculation, 0, 0, 0, reserve_bytes, - recipe.weights_bytes(checkpoint, len(gpus), speculation, expert_scale_compression), costs.graph_bytes, + recipe.weights_bytes(checkpoint, len(gpus), speculation, expert_scale_compression, + row_shard_attention=row_shard_attention, prompt_logprobs=prompt_logprobs, + mtp_only=speculation is not None and speculation.mtp_only), costs.graph_bytes, context_parallel=context_parallel and costs.page_tokens is not None, context_parallel_size=context_parallel_size, stream_overlap=stream_overlap and recipe.stream_overlap, mixed_prefill=mixed_prefill, prompt_logprobs=prompt_logprobs, expert_scale_compression=expert_scale_compression, roce_ring=roce_ring, roce_chain=roce_chain, - row_shard_prefill=row_shard_prefill, mla_compute_mode=mla_compute_mode, + row_shard_prefill=row_shard_prefill, row_shard_attention=row_shard_attention, + mla_compute_mode=mla_compute_mode, # Language-only plans prepare no vision encoder, so no image feature slots. image_concurrency=0 if language_model_only else (min(4, concurrency) if recipe.max_images else 0) if image_concurrency is None else image_concurrency, online_mxfp8=online_mxfp8, online_mxfp8_scope=online_mxfp8_scope, @@ -581,7 +591,7 @@ def plan(recipe: Recipe, checkpoint: Checkpoint, gpus: tuple[Gpu, ...], *, selec expert_scale_compression: str | None = None, roce_ring: bool = False, roce_chain: bool = False, image_concurrency: int | None = None, online_mxfp8: bool = False, online_mxfp8_scope: str | None = None, language_model_only: bool = False, mla_compute_mode: str | None = None, - row_shard_prefill: bool = False) -> Plan: + row_shard_prefill: bool = False, row_shard_attention: bool = False) -> Plan: """Select GPUs and size the plan, or raise a capacity error that lists working settings. Without `selected`, the plan uses the smallest supported tensor-parallel size @@ -619,7 +629,11 @@ def plan(recipe: Recipe, checkpoint: Checkpoint, gpus: tuple[Gpu, ...], *, selec if recipe.name != "deepseek41": raise UsageError("--vocabulary-parallel-sampling requires DeepSeek V4.1") world_sizes = tuple(size for size in world_sizes if size != 3) - sizes = " or ".join(map(str, world_sizes)) + # Three ranks have no per-rank block of the recipe's chunk, so a row-sharded plan has no + # three-rank size to select automatically or to offer in a capacity error. An explicit + # three-rank selection still reaches the plan's own refusal. + plan_sizes = tuple(size for size in world_sizes if size != 3) if row_shard_prefill else world_sizes + sizes = " or ".join(map(str, plan_sizes)) supported = [gpu for gpu in gpus if gpu.supported] chosen = None if selected is not None: @@ -637,7 +651,7 @@ def plan(recipe: Recipe, checkpoint: Checkpoint, gpus: tuple[Gpu, ...], *, selec for gpu in chosen: if not gpu.supported: raise CapacityError(f"GPU {gpu.index} ({gpu.name}) is not an {SUPPORTED_ARCHITECTURES} device") - elif len(supported) < world_sizes[0]: + elif len(supported) < plan_sizes[0]: raise CapacityError(f"{checkpoint.alias} needs {sizes} {SUPPORTED_ARCHITECTURES} GPUs; this host has {len(supported)}") def fitted(group, *, concurrency=concurrency, reserve_bytes=reserve_bytes, speculation=speculation): @@ -647,7 +661,7 @@ def fitted(group, *, concurrency=concurrency, reserve_bytes=reserve_bytes, specu stream_overlap=stream_overlap, cache_states=cache_states, mixed_prefill=mixed_prefill, prompt_logprobs=prompt_logprobs, expert_scale_compression=expert_scale_compression, roce_ring=roce_ring, roce_chain=roce_chain, - row_shard_prefill=row_shard_prefill, + row_shard_prefill=row_shard_prefill, row_shard_attention=row_shard_attention, image_concurrency=None if image_concurrency is None else min(image_concurrency, concurrency), online_mxfp8=online_mxfp8, online_mxfp8_scope=online_mxfp8_scope, language_model_only=language_model_only, mla_compute_mode=mla_compute_mode) @@ -663,7 +677,7 @@ def automatic(world_size, **settings): def candidates(**settings): if chosen is not None: return [fitted(chosen, **settings)] - return [automatic(size, **settings) for size in world_sizes if size <= len(supported)] + return [automatic(size, **settings) for size in plan_sizes if size <= len(supported)] def attempt(**settings): return next((candidate for candidate in candidates(**settings) if candidate.fits), None) @@ -690,7 +704,7 @@ def attempt(**settings): if speculation is not None and attempt(speculation=None) is not None: options.append("--no-mtp") if chosen is not None: - for size in world_sizes: + for size in plan_sizes: if len(chosen) < size <= len(supported) and (larger := automatic(size)).fits: options.append(f"--gpus {format_gpus(gpu.index for gpu in larger.gpus)}") break diff --git a/ignition/preparation/rebuild_manifest.py b/ignition/preparation/rebuild_manifest.py index 912cd184c..db73cd4d0 100644 --- a/ignition/preparation/rebuild_manifest.py +++ b/ignition/preparation/rebuild_manifest.py @@ -237,6 +237,8 @@ def from_plan(model, plan, *, compiled_forwards): result['shard_hyperconnections'] = True if plan.row_shard_prefill: result['row_shard_prefill'] = True + if plan.row_shard_attention: + result['row_shard_attention'] = True if plan.joint_tuning: result['joint_tuning'] = True if (s := plan.speculation) is not None: diff --git a/ignition/preparation/runtime_core_pb2.py b/ignition/preparation/runtime_core_pb2.py index 77059b150..293d7f6ac 100644 --- a/ignition/preparation/runtime_core_pb2.py +++ b/ignition/preparation/runtime_core_pb2.py @@ -24,7 +24,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12runtime_core.proto\x12\rflare.runtime\"\x99\x01\n\x0eProtectedIndex\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x0f\n\x07\x63ore_id\x18\x02 \x01(\x0c\x12\x0e\n\x06signer\x18\x03 \x01(\x0c\x12*\n\x06groups\x18\x04 \x03(\x0b\x32\x1a.flare.runtime.SignedGroup\x12)\n\x05ranks\x18\x05 \x03(\x0b\x32\x1a.flare.runtime.SignedGroup\"<\n\x0bSignedGroup\x12\x0c\n\x04kind\x18\x01 \x01(\r\x12\x0c\n\x04\x62ody\x18\x02 \x01(\x0c\x12\x11\n\tsignature\x18\x03 \x01(\x0c\"\xcc\x01\n\x11PublisherMetadata\x12(\n\x05index\x18\x01 \x01(\x0b\x32\x19.flare.runtime.ModelIndex\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x17\n\x0fofficial_source\x18\x03 \x01(\t\x12\x14\n\x0cgroup_hashes\x18\x04 \x03(\x0c\x12\x13\n\x0b\x64\x65velopment\x18\x05 \x01(\x08\x12\r\n\x05ranks\x18\x06 \x01(\r\x12\'\n\x07license\x18\x07 \x01(\x0b\x32\x16.flare.runtime.License\"%\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04link\x18\x02 \x01(\t\"F\n\x0fResourceCatalog\x12\x33\n\tresources\x18\x01 \x03(\x0b\x32 .flare.runtime.ProtectedResource\"^\n\x11ProtectedResource\x12\x0e\n\x06offset\x18\x01 \x01(\x04\x12\x14\n\x0cstored_bytes\x18\x02 \x01(\x04\x12\x13\n\x0bplain_bytes\x18\x03 \x01(\x04\x12\x0e\n\x06sha256\x18\x04 \x03(\x0c\"`\n\nStoredData\x12\x0e\n\x06nbytes\x18\x01 \x01(\x04\x12\x0e\n\x06sha256\x18\x02 \x01(\x0c\x12\x0e\n\x06offset\x18\x03 \x01(\x04\x12\"\n\x05parts\x18\x04 \x03(\x0b\x32\x13.flare.runtime.Span\"\xd9\x01\n\x0eRankProtection\x12\x0c\n\x04rank\x18\x01 \x01(\r\x12*\n\x06\x62undle\x18\x02 \x01(\x0b\x32\x1a.flare.runtime.BundleIndex\x12\x33\n\x0b\x65xecutables\x18\x03 \x01(\x0b\x32\x1e.flare.runtime.ResourceCatalog\x12/\n\x07support\x18\x04 \x01(\x0b\x32\x1e.flare.runtime.ResourceCatalog\x12\'\n\x04\x64\x61ta\x18\x05 \x01(\x0b\x32\x19.flare.runtime.StoredData\"\xcf\x03\n\nModelIndex\x12\x0f\n\x07version\x18\x01 \x01(\r\x12)\n\x05ranks\x18\x02 \x03(\x0b\x32\x1a.flare.runtime.BundleIndex\x12.\n\x0bpeer_memory\x18\x03 \x03(\x0b\x32\x19.flare.runtime.PeerMemory\x12$\n\x06\x61ssets\x18\x05 \x03(\x0b\x32\x14.flare.runtime.Asset\x12%\n\x06\x65ngram\x18\x06 \x01(\x0b\x32\x15.flare.runtime.Engram\x12\x43\n\x11sampling_defaults\x18\x07 \x01(\x0b\x32(.flare.runtime.DefaultSamplingParameters\x12\x31\n\x08\x66rontend\x18\x08 \x01(\x0b\x32\x1f.flare.runtime.FrontendSettings\x12\'\n\x07\x63luster\x18\t \x01(\x0b\x32\x16.flare.runtime.Cluster\x12\x30\n\x08\x63ontract\x18\n \x01(\x0b\x32\x1e.flare.runtime.ServingContract\x12/\n\x07rebuild\x18\x0b \x01(\x0b\x32\x1e.flare.runtime.RebuildManifestJ\x04\x08\x04\x10\x05\"\xd5\x06\n\x0fRebuildManifest\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x0e\n\x06recipe\x18\x02 \x01(\t\x12\x17\n\x0fsource_identity\x18\x03 \x01(\t\x12\x1d\n\x15\x63heckpoint_repository\x18\x04 \x01(\t\x12\x1b\n\x13\x63heckpoint_revision\x18\x05 \x01(\t\x12\x17\n\x0ftensor_parallel\x18\x06 \x01(\r\x12\x17\n\x0fmax_concurrency\x18\x07 \x01(\x04\x12\x15\n\rprefill_chunk\x18\x08 \x01(\x04\x12\x19\n\x11image_concurrency\x18\t \x01(\x04\x12\x18\n\x10\x63ontext_parallel\x18\n \x01(\x08\x12\x16\n\x0estream_overlap\x18\x0b \x01(\x08\x12\x15\n\rmixed_prefill\x18\x0c \x01(\x08\x12%\n\x18\x65xpert_scale_compression\x18\r \x01(\tH\x00\x88\x01\x01\x12\x18\n\x10mla_compute_mode\x18\x0e \x01(\t\x12\x11\n\troce_ring\x18\x0f \x01(\x08\x12\x14\n\x0conline_mxfp8\x18\x10 \x01(\x08\x12\x1b\n\x13language_model_only\x18\x11 \x01(\x08\x12\x36\n\x0bspeculation\x18\x12 \x01(\x0b\x32!.flare.runtime.RebuildSpeculation\x12\x19\n\x11\x63ompiled_forwards\x18\x13 \x01(\x08\x12\x15\n\rreserve_bytes\x18\x14 \x01(\x04\x12\x18\n\x10local_checkpoint\x18\x15 \x01(\x08\x12\x12\n\nroce_chain\x18\x16 \x01(\x08\x12\x14\n\x0cjoint_tuning\x18\x17 \x01(\x08\x12\x1e\n\x16shard_hyperconnections\x18\x18 \x01(\x08\x12\x19\n\x11\x63ollective_tuning\x18\x19 \x01(\t\x12\x1e\n\x16\x63ollective_ladder_json\x18\x1a \x01(\t\x12\x1c\n\x14\x66orce_moe_decode_a16\x18\x1b \x01(\x08\x12\x19\n\x11row_shard_prefill\x18\x1c \x01(\x08\x12\x1a\n\x12online_mxfp8_scope\x18\x1e \x01(\t\x12\x1d\n\x15\x63ontext_parallel_size\x18\x1f \x01(\rB\x1b\n\x19_expert_scale_compression\"\xfe\x03\n\x12RebuildSpeculation\x12\x14\n\x0c\x64raft_tokens\x18\x01 \x01(\r\x12\x10\n\x08\x61\x64\x61ptive\x18\x02 \x01(\x08\x12\x17\n\x0fproposal_source\x18\x03 \x01(\t\x12\x16\n\x0ehistory_drafts\x18\x04 \x01(\x08\x12\x16\n\x0etoken_capacity\x18\x05 \x01(\x04\x12\x1b\n\x13packed_verification\x18\x06 \x01(\x08\x12\x1e\n\x16\x64raft_prefix_selection\x18\x07 \x01(\t\x12$\n\x17\x64raft_head_quantization\x18\x08 \x01(\tH\x00\x88\x01\x01\x12%\n\x18markov_head_quantization\x18\t \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0cgraph_budget\x18\n \x01(\x04H\x02\x88\x01\x01\x12$\n\x1cvocabulary_parallel_sampling\x18\x0b \x01(\x08\x12\x0e\n\x06method\x18\x0c \x01(\t\x12\x10\n\x08mtp_only\x18\r \x01(\x08\x12$\n\x17\x64raft_body_quantization\x18\x0e \x01(\tH\x03\x88\x01\x01\x42\x1a\n\x18_draft_head_quantizationB\x1b\n\x19_markov_head_quantizationB\x0f\n\r_graph_budgetB\x1a\n\x18_draft_body_quantization\"e\n\x0eRebuildWeights\x12\x0f\n\x07version\x18\x01 \x01(\r\x12-\n\x07tensors\x18\x02 \x03(\x0b\x32\x1c.flare.runtime.RebuildTensor\x12\x13\n\x0bunavailable\x18\x03 \x03(\t\"\xa2\x01\n\rRebuildTensor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x64type\x18\x02 \x01(\t\x12\x15\n\relement_bytes\x18\x03 \x01(\r\x12\r\n\x05shape\x18\x04 \x03(\x04\x12\x0e\n\x06stride\x18\x05 \x03(\x04\x12\x13\n\x0blayout_type\x18\x06 \x01(\t\x12)\n\x07storage\x18\x07 \x01(\x0b\x32\x18.flare.runtime.Reference\"\xcb\x06\n\x0fServingContract\x12\x10\n\x08model_id\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x34\n\x0emodel_contract\x18\x03 \x01(\x0b\x32\x1c.flare.runtime.ModelContract\x12\x1b\n\x13max_sequence_tokens\x18\x04 \x01(\x04\x12\x1a\n\x12prefill_chunk_size\x18\x05 \x01(\x04\x12\r\n\x05pages\x18\x06 \x01(\x04\x12\x12\n\nstate_rows\x18\x07 \x01(\x04\x12\x11\n\tsnapshots\x18\x08 \x01(\x04\x12\x13\n\x0bpage_tokens\x18\t \x01(\x04\x12\x1a\n\x12speculative_widths\x18\x0b \x03(\x04\x12$\n\x17speculative_fixed_width\x18\x0c \x01(\x04H\x00\x88\x01\x01\x12\x16\n\x0ehistory_drafts\x18\r \x01(\x08\x12#\n\x1bspeculative_proposal_source\x18\x0e \x01(\t\x12\x1b\n\x0emax_new_tokens\x18\x0f \x01(\x04H\x01\x88\x01\x01\x12\x17\n\nmax_active\x18\x10 \x01(\x04H\x02\x88\x01\x01\x12\x14\n\x0cserved_names\x18\x11 \x03(\t\x12\x1a\n\rreserve_bytes\x18\x12 \x01(\x04H\x03\x88\x01\x01\x12\x1b\n\x0equeue_capacity\x18\x13 \x01(\x04H\x04\x88\x01\x01\x12\x1d\n\x10\x66rontend_workers\x18\x14 \x01(\x04H\x05\x88\x01\x01\x12\x1b\n\x0emax_body_bytes\x18\x15 \x01(\x04H\x06\x88\x01\x01\x12\x14\n\x0c\x63ontext_hint\x18\x16 \x01(\t\x12\x15\n\rcapacity_hint\x18\x17 \x01(\t\x12\"\n\x1aspeculative_token_capacity\x18\x18 \x01(\x04\x12 \n\x13\x64\x65\x63ode_weight_bytes\x18\x19 \x01(\x04H\x07\x88\x01\x01\x42\x1a\n\x18_speculative_fixed_widthB\x11\n\x0f_max_new_tokensB\r\n\x0b_max_activeB\x10\n\x0e_reserve_bytesB\x11\n\x0f_queue_capacityB\x13\n\x11_frontend_workersB\x11\n\x0f_max_body_bytesB\x16\n\x14_decode_weight_bytesJ\x04\x08\n\x10\x0b\"\x8b\x02\n\rModelContract\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x12\n\nvocabulary\x18\x02 \x01(\x04\x12\x18\n\x10input_vocabulary\x18\x03 \x01(\x04\x12-\n\x06vision\x18\x04 \x01(\x0b\x32\x1d.flare.runtime.VisionCapacity\x12+\n\x05\x63\x61\x63he\x18\x05 \x01(\x0b\x32\x1c.flare.runtime.CacheContract\x12,\n\x06graphs\x18\x06 \x03(\x0b\x32\x1c.flare.runtime.GraphContract\x12\x31\n\x08\x66rontend\x18\x07 \x01(\x0b\x32\x1f.flare.runtime.FrontendContract\"\x90\x01\n\x0eVisionCapacity\x12\x15\n\rfeature_slots\x18\x05 \x01(\x04\x12\x1c\n\x14\x66\x65\x61tures_per_request\x18\x01 \x01(\x04\x12\x12\n\nmax_images\x18\x02 \x01(\x04\x12\x1b\n\x13patches_per_feature\x18\x03 \x01(\x04\x12\x18\n\x10patch_capacities\x18\x04 \x03(\x04\"\x86\x01\n\rCacheContract\x12\x13\n\x0bpage_tokens\x18\x01 \x01(\x04\x12-\n\x07regions\x18\x02 \x03(\x0b\x32\x1c.flare.runtime.HistoryRegion\x12\x31\n\x0bpage_tables\x18\x03 \x03(\x0b\x32\x1c.flare.runtime.PageTableView\"\x96\x01\n\rHistoryRegion\x12(\n\x04kind\x18\x01 \x01(\x0e\x32\x1a.flare.runtime.HistoryKind\x12)\n\x07storage\x18\x02 \x01(\x0b\x32\x18.flare.runtime.StateView\x12\x19\n\x11tokens_per_record\x18\x03 \x01(\x04\x12\x15\n\rwindow_tokens\x18\x04 \x01(\x04\"4\n\rPageTableView\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\t\x12\x13\n\x0bpage_offset\x18\x02 \x01(\x04\"\xb1\x01\n\x0cValueBinding\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\t\x12*\n\x06source\x18\x02 \x01(\x0e\x32\x1a.flare.runtime.ValueSource\x12.\n\x08indexing\x18\x03 \x01(\x0e\x32\x1c.flare.runtime.InputIndexing\x12\x14\n\x0cvalue_offset\x18\x04 \x01(\x12\x12\x0f\n\x07padding\x18\x05 \x01(\x12\x12\x0e\n\x06tensor\x18\x06 \x01(\t\"K\n\tStateView\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\t\x12\x0e\n\x06offset\x18\x02 \x01(\x04\x12\x0e\n\x06stride\x18\x03 \x01(\x04\x12\x0e\n\x06nbytes\x18\x04 \x01(\x04\"e\n\x0bStateAccess\x12&\n\x04view\x18\x01 \x01(\x0b\x32\x18.flare.runtime.StateView\x12.\n\x06\x61\x63\x63\x65ss\x18\x02 \x01(\x0e\x32\x1e.flare.runtime.StateAccessKind\"O\n\x10\x43ompletionCursor\x12\'\n\x05state\x18\x01 \x01(\x0b\x32\x18.flare.runtime.StateView\x12\x12\n\nlag_tokens\x18\x02 \x01(\x04\"4\n\x0cTunedBinding\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\t\x12\x14\n\x0ctuning_graph\x18\x02 \x01(\t\"\xfa\x02\n\rGraphContract\x12\r\n\x05graph\x18\x01 \x01(\t\x12+\n\x06inputs\x18\x02 \x03(\x0b\x32\x1b.flare.runtime.ValueBinding\x12)\n\x05state\x18\x03 \x03(\x0b\x32\x1a.flare.runtime.StateAccess\x12\x1e\n\x16state_rows_per_request\x18\x04 \x01(\x04\x12\x35\n\x0c\x64raft_cursor\x18\x05 \x01(\x0b\x32\x1f.flare.runtime.CompletionCursor\x12\x18\n\x10\x65mbedding_tables\x18\x06 \x03(\r\x12/\n\rpending_token\x18\x07 \x01(\x0b\x32\x18.flare.runtime.StateView\x12+\n\x04kind\x18\x08 \x01(\x0e\x32\x1d.flare.runtime.GraphOperation\x12\x33\n\x0etuned_bindings\x18\t \x03(\x0b\x32\x1b.flare.runtime.TunedBinding\"k\n\nToolParser\x12+\n\x04kind\x18\x01 \x01(\x0e\x32\x1d.flare.runtime.ToolParserKind\x12\x19\n\x11wrapping_newlines\x18\x02 \x01(\r\x12\x15\n\rtag_separator\x18\x03 \x01(\t\"\x95\x01\n\x0fReasoningParser\x12\x30\n\x04kind\x18\x01 \x01(\x0e\x32\".flare.runtime.ReasoningParserKind\x12\x0c\n\x04open\x18\x02 \x01(\t\x12\r\n\x05\x63lose\x18\x03 \x01(\t\x12\x19\n\x11wrapping_newlines\x18\x04 \x01(\r\x12\x18\n\x10terminal_markers\x18\x05 \x03(\t\"\xca\x01\n\x0ePrefixBoundary\x12\r\n\x05roles\x18\x01 \x03(\t\x12\x18\n\x0b\x61ppend_role\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x1e\n\x16\x63ontinue_final_message\x18\x03 \x01(\x08\x12\x18\n\x10minimum_messages\x18\x04 \x01(\x04\x12\x16\n\x0erequired_start\x18\x05 \x01(\t\x12\x14\n\x0crequired_end\x18\x06 \x01(\t\x12\x17\n\x0frequired_marker\x18\x07 \x01(\tB\x0e\n\x0c_append_role\"y\n\tImageSpan\x12\x13\n\x0bstart_token\x18\x01 \x01(\r\x12\x11\n\tend_token\x18\x02 \x01(\r\x12\x12\n\nmax_tokens\x18\x03 \x01(\x04\x12\x17\n\x0finterior_tokens\x18\x04 \x03(\r\x12\x17\n\x0funscoped_tokens\x18\x05 \x03(\r\"Q\n\x10ImageTokenLayout\x12\x15\n\rfeature_token\x18\x01 \x01(\r\x12&\n\x04span\x18\x02 \x01(\x0b\x32\x18.flare.runtime.ImageSpan\"\xf2\x03\n\x10\x46rontendContract\x12(\n\x05tools\x18\x01 \x01(\x0b\x32\x19.flare.runtime.ToolParser\x12\x31\n\treasoning\x18\x02 \x01(\x0b\x32\x1e.flare.runtime.ReasoningParser\x12\x43\n\x11image_preparation\x18\x03 \x01(\x0e\x32#.flare.runtime.ImagePreparationKindH\x00\x88\x01\x01\x12\x39\n\x0cimage_layout\x18\x04 \x01(\x0e\x32\x1e.flare.runtime.ImageLayoutKindH\x01\x88\x01\x01\x12\x17\n\x0frejected_tokens\x18\x05 \x03(\r\x12\x1b\n\x13position_dimensions\x18\x06 \x01(\r\x12\x18\n\x10\x61ssistant_marker\x18\x07 \x01(\t\x12\x1b\n\x13message_end_markers\x18\x08 \x03(\t\x12\x36\n\x0fprefix_boundary\x18\t \x01(\x0b\x32\x1d.flare.runtime.PrefixBoundary\x12\x35\n\x0cimage_tokens\x18\n \x01(\x0b\x32\x1f.flare.runtime.ImageTokenLayoutB\x14\n\x12_image_preparationB\x0f\n\r_image_layout\"1\n\x07\x43luster\x12\r\n\x05world\x18\x01 \x01(\r\x12\x11\n\texport_id\x18\x03 \x01(\tJ\x04\x08\x02\x10\x03\"\x8c\x03\n\x19\x44\x65\x66\x61ultSamplingParameters\x12\x11\n\x04seed\x18\x01 \x01(\x04H\x00\x88\x01\x01\x12\x18\n\x0btemperature\x18\x02 \x01(\x02H\x01\x88\x01\x01\x12\x12\n\x05top_k\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05top_p\x18\x04 \x01(\x02H\x03\x88\x01\x01\x12\x12\n\x05min_p\x18\x05 \x01(\x02H\x04\x88\x01\x01\x12\x1f\n\x12repetition_penalty\x18\x06 \x01(\x02H\x05\x88\x01\x01\x12\x1d\n\x10presence_penalty\x18\x07 \x01(\x02H\x06\x88\x01\x01\x12\x1e\n\x11\x66requency_penalty\x18\x08 \x01(\x02H\x07\x88\x01\x01\x12-\n\x06ranges\x18\t \x01(\x0b\x32\x1d.flare.runtime.SamplingRangesB\x07\n\x05_seedB\x0e\n\x0c_temperatureB\x08\n\x06_top_kB\x08\n\x06_top_pB\x08\n\x06_min_pB\x15\n\x13_repetition_penaltyB\x13\n\x11_presence_penaltyB\x14\n\x12_frequency_penalty\"H\n\x12SamplingFloatRange\x12\x10\n\x03min\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12\x10\n\x03max\x18\x02 \x01(\x02H\x01\x88\x01\x01\x42\x06\n\x04_minB\x06\n\x04_max\"F\n\x10SamplingIntRange\x12\x10\n\x03min\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x10\n\x03max\x18\x02 \x01(\x05H\x01\x88\x01\x01\x42\x06\n\x04_minB\x06\n\x04_max\"\x96\x03\n\x0eSamplingRanges\x12\x36\n\x0btemperature\x18\x01 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12.\n\x05top_k\x18\x02 \x01(\x0b\x32\x1f.flare.runtime.SamplingIntRange\x12\x30\n\x05top_p\x18\x03 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12\x30\n\x05min_p\x18\x04 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12=\n\x12repetition_penalty\x18\x05 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12;\n\x10presence_penalty\x18\x06 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12<\n\x11\x66requency_penalty\x18\x07 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\"\xc5\x02\n\x10\x46rontendSettings\x12\x33\n\ttokenizer\x18\x01 \x01(\x0b\x32 .flare.runtime.TokenizerSettings\x12+\n\x05image\x18\x02 \x01(\x0b\x32\x1c.flare.runtime.ImageSettings\x12\x15\n\reos_token_ids\x18\x03 \x03(\r\x12:\n\x11template_encoding\x18\x04 \x01(\x0e\x32\x1f.flare.runtime.TemplateEncoding\x12\x34\n\x0eprompt_encoder\x18\x05 \x01(\x0e\x32\x1c.flare.runtime.PromptEncoder\x12\'\n\x1aimage_placeholder_token_id\x18\x06 \x01(\rH\x00\x88\x01\x01\x42\x1d\n\x1b_image_placeholder_token_id\"\xdc\x02\n\x11TokenizerSettings\x12\x16\n\tbos_token\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\teos_token\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tunk_token\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsep_token\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x16\n\tpad_token\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x16\n\tcls_token\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\nmask_token\x18\x07 \x01(\tH\x06\x88\x01\x01\x12;\n\x19\x61\x64\x64itional_special_tokens\x18\x08 \x01(\x0b\x32\x18.flare.runtime.TokenListB\x0c\n\n_bos_tokenB\x0c\n\n_eos_tokenB\x0c\n\n_unk_tokenB\x0c\n\n_sep_tokenB\x0c\n\n_pad_tokenB\x0c\n\n_cls_tokenB\r\n\x0b_mask_token\"\x1b\n\tTokenList\x12\x0e\n\x06values\x18\x01 \x03(\t\"\xfc\x03\n\rImageSettings\x12\x30\n\x04qwen\x18\x01 \x01(\x0b\x32 .flare.runtime.QwenImageSettingsH\x00\x12\x30\n\x04mimo\x18\x02 \x01(\x0b\x32 .flare.runtime.MimoImageSettingsH\x00\x12)\n\tdeepseek4\x18\x03 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12\'\n\x07moonvit\x18\x04 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12$\n\x04step\x18\x05 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12%\n\x05radio\x18\x06 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12J\n\x12\x64\x65\x65pseek_row_major\x18\x07 \x01(\x0b\x32,.flare.runtime.DeepseekRowMajorImageSettingsH\x00\x12#\n\x03glm\x18\x08 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12=\n\x0bpatch_merge\x18\t \x01(\x0b\x32&.flare.runtime.PatchMergeImageSettingsH\x00\x12+\n\x0bpadded_grid\x18\n \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x42\t\n\x07profile\"\xb9\x01\n\x17PatchMergeImageSettings\x12\x12\n\npatch_size\x18\x01 \x01(\r\x12\x12\n\nmin_pixels\x18\x02 \x01(\x04\x12\x12\n\nmax_pixels\x18\x03 \x01(\x04\x12\x12\n\nimage_mean\x18\x04 \x03(\x02\x12\x11\n\timage_std\x18\x05 \x03(\x02\x12\x13\n\x0bimage_token\x18\x06 \x01(\t\x12\x13\n\x0bstart_token\x18\x07 \x01(\t\x12\x11\n\tend_token\x18\x08 \x01(\t\"\x87\x01\n\x1d\x44\x65\x65pseekRowMajorImageSettings\x12\x12\n\nmin_pixels\x18\x01 \x01(\x04\x12\x12\n\nmax_tokens\x18\x02 \x01(\x04\x12#\n\x16max_width_height_ratio\x18\x03 \x01(\x04H\x00\x88\x01\x01\x42\x19\n\x17_max_width_height_ratio\"\x82\x01\n\x11QwenImageSettings\x12\x12\n\nmin_pixels\x18\x01 \x01(\x04\x12\x12\n\nmax_pixels\x18\x02 \x01(\x04\x12\x15\n\rposition_side\x18\x03 \x01(\r\x12\x16\n\x0eimage_token_id\x18\x04 \x01(\r\x12\x16\n\x0evideo_token_id\x18\x05 \x01(\r\"b\n\x11MimoImageSettings\x12\x12\n\nmin_pixels\x18\x01 \x01(\x04\x12\x12\n\nmax_pixels\x18\x02 \x01(\x04\x12\x12\n\nimage_mean\x18\x03 \x03(\x02\x12\x11\n\timage_std\x18\x04 \x03(\x02\"|\n\x06\x45ngram\x12\x11\n\ttoken_map\x18\x01 \x03(\r\x12\x14\n\x0cpad_token_id\x18\x02 \x01(\r\x12\x1d\n\x15\x63ompressed_vocabulary\x18\x03 \x01(\r\x12*\n\x06tables\x18\x04 \x03(\x0b\x32\x1a.flare.runtime.EngramTable\"\x80\x04\n\x0b\x45ngramTable\x12\r\n\x05layer\x18\x01 \x01(\r\x12\x0c\n\x04rows\x18\x02 \x01(\x04\x12\x17\n\x0f\x65mbedding_width\x18\x03 \x01(\r\x12\x13\n\x0bscale_block\x18\x04 \x01(\r\x12$\n\x07weights\x18\x05 \x01(\x0b\x32\x13.flare.runtime.Span\x12#\n\x06scales\x18\x06 \x01(\x0b\x32\x13.flare.runtime.Span\x12\'\n\x04hash\x18\x07 \x01(\x0b\x32\x19.flare.runtime.EngramHash\x12\x15\n\rweight_buffer\x18\x08 \x01(\t\x12\x14\n\x0cscale_buffer\x18\t \x01(\t\x12\x13\n\x0bhash_buffer\x18\n \x01(\t\x12\x13\n\x0bready_event\x18\x0b \x01(\t\x12\x35\n\x08\x65ncoding\x18\x0c \x01(\x0e\x32#.flare.runtime.EngramTable.Encoding\x12\x16\n\x0erequest_buffer\x18\r \x01(\t\x12\x14\n\x0cglobal_scale\x18\x0e \x01(\x02\x12/\n\x06shards\x18\x0f \x03(\x0b\x32\x1f.flare.runtime.EngramTableShard\"E\n\x08\x45ncoding\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x11\n\rFP8_E4M3_E8M0\x10\x01\x12\x15\n\x11NVFP4_E4M3_GLOBAL\x10\x02\"\x8c\x01\n\x10\x45ngramTableShard\x12\x0c\n\x04rank\x18\x01 \x01(\r\x12\x11\n\tfirst_row\x18\x02 \x01(\x04\x12\x0c\n\x04rows\x18\x03 \x01(\x04\x12$\n\x07weights\x18\x04 \x01(\x0b\x32\x13.flare.runtime.Span\x12#\n\x06scales\x18\x05 \x01(\x0b\x32\x13.flare.runtime.Span\"\xc3\x01\n\nEngramHash\x12\x30\n\x06method\x18\x01 \x01(\x0e\x32 .flare.runtime.EngramHash.Method\x12\x0e\n\x06orders\x18\x02 \x03(\r\x12\x13\n\x0bmultipliers\x18\x03 \x03(\x04\x12\x0e\n\x06primes\x18\x04 \x03(\x04\x12\x0f\n\x07offsets\x18\x05 \x03(\x04\"=\n\x06Method\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nXOR_MODULO\x10\x01\x12\x12\n\x0e\x45OS_XOR_MODULO\x10\x02\"K\n\x05\x41sset\x12\x0c\n\x04name\x18\x01 \x01(\t\x12$\n\x07payload\x18\x02 \x01(\x0b\x32\x13.flare.runtime.Span\x12\x0e\n\x06sha256\x18\x03 \x01(\t\"\xd3\x04\n\x0b\x42undleIndex\x12\x14\n\x0c\x61rchitecture\x18\x01 \x03(\x05\x12.\n\x0b\x61llocations\x18\x02 \x03(\x0b\x32\x19.flare.runtime.Allocation\x12&\n\x07modules\x18\x03 \x03(\x0b\x32\x15.flare.runtime.Module\x12*\n\tfunctions\x18\x04 \x03(\x0b\x32\x17.flare.runtime.Function\x12)\n\x06graphs\x18\x05 \x03(\x0b\x32\x19.flare.runtime.GraphIndex\x12\x30\n\x0creservations\x18\x06 \x03(\x0b\x32\x1a.flare.runtime.Reservation\x12+\n\texecution\x18\x07 \x01(\x0b\x32\x18.flare.runtime.Execution\x12\'\n\x07\x63ontext\x18\x08 \x01(\x0b\x32\x16.flare.runtime.Context\x12<\n\tresources\x18\t \x03(\x0b\x32).flare.runtime.BundleIndex.ResourcesEntry\x12\x0e\n\x06\x65vents\x18\n \x03(\t\x12*\n\x04roce\x18\x0b \x01(\x0b\x32\x1c.flare.runtime.RoceTransport\x12\x36\n\x0frebuild_weights\x18\x0c \x01(\x0b\x32\x1d.flare.runtime.RebuildWeights\x1a\x45\n\x0eResourcesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.flare.runtime.Span:\x02\x38\x01\"\xf8\x01\n\rRoceTransport\x12\x11\n\tproxy_abi\x18\x01 \x01(\r\x12\r\n\x05world\x18\x02 \x01(\r\x12\x0c\n\x04rank\x18\x03 \x01(\r\x12\x16\n\x0eregion_address\x18\x04 \x01(\x04\x12\x14\n\x0cregion_bytes\x18\x05 \x01(\x04\x12\x18\n\x10\x63ounters_address\x18\x06 \x01(\x04\x12\x16\n\x0e\x63ounters_bytes\x18\x07 \x01(\x04\x12\x12\n\nslot_bytes\x18\x08 \x01(\x04\x12\x12\n\npipe_bytes\x18\t \x01(\x04\x12\x13\n\x0b\x63trl_offset\x18\n \x01(\x04\x12\x14\n\x0cvarlen_bytes\x18\x10 \x01(\x04J\x04\x08\x0b\x10\x10\"v\n\nGraphIndex\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x66lags\x18\x02 \x01(\x04\x12\x10\n\x08resource\x18\x03 \x01(\t\x12\x12\n\nnode_count\x18\x04 \x01(\x04\x12\x12\n\nedge_count\x18\x05 \x01(\x04\x12\x11\n\tfunctions\x18\x06 \x03(\x04\"M\n\nPeerMemory\x12\r\n\x05owner\x18\x01 \x01(\r\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\x04\x12\x0e\n\x06nbytes\x18\x03 \x01(\x04\x12\x0f\n\x07readers\x18\x04 \x03(\r\"M\n\x07\x43ontext\x12\r\n\x05\x66lags\x18\x01 \x01(\r\x12\r\n\x05\x63\x61\x63he\x18\x02 \x01(\x05\x12$\n\x06limits\x18\x03 \x03(\x0b\x32\x14.flare.runtime.Limit\"\"\n\x05Limit\x12\n\n\x02id\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\x04\".\n\x0bReservation\x12\x0f\n\x07\x61\x64\x64ress\x18\x01 \x01(\x04\x12\x0e\n\x06nbytes\x18\x02 \x01(\x04\"8\n\x04Span\x12\x0e\n\x06offset\x18\x01 \x01(\x04\x12\x0e\n\x06nbytes\x18\x02 \x01(\x04\x12\x10\n\x08\x66ilename\x18\x03 \x01(\t\"\xa2\x02\n\nAllocation\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06nbytes\x18\x02 \x01(\x04\x12\x11\n\talignment\x18\x03 \x01(\x04\x12\x11\n\x04\x66ile\x18\x04 \x01(\tH\x00\x88\x01\x01\x12\'\n\ndata_spans\x18\x05 \x03(\x0b\x32\x13.flare.runtime.Span\x12.\n\x0brelocations\x18\x06 \x03(\x0b\x32\x19.flare.runtime.Relocation\x12\x14\n\x07\x61\x64\x64ress\x18\x07 \x01(\x04H\x01\x88\x01\x01\x12%\n\x06memory\x18\x08 \x01(\x0e\x32\x15.flare.runtime.Memory\x12%\n\x06sizing\x18\t \x01(\x0b\x32\x15.flare.runtime.SizingB\x07\n\x05_fileB\n\n\x08_address\"X\n\x06Sizing\x12)\n\x08\x63\x61pacity\x18\x01 \x01(\x0e\x32\x17.flare.runtime.Capacity\x12\x12\n\nunit_bytes\x18\x02 \x01(\x04\x12\x0f\n\x07maximum\x18\x03 \x01(\x04\"L\n\x06Module\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x66ile\x18\x02 \x01(\t\x12&\n\x07globals\x18\x03 \x03(\x0b\x32\x15.flare.runtime.Global\"f\n\x06Global\x12\x0e\n\x06symbol\x18\x01 \x01(\t\x12\x0c\n\x04\x66ile\x18\x02 \x01(\t\x12\x0e\n\x06nbytes\x18\x03 \x01(\x04\x12.\n\x0brelocations\x18\x04 \x03(\x0b\x32\x19.flare.runtime.Relocation\"y\n\x08\x46unction\x12\x0e\n\x06module\x18\x01 \x01(\t\x12\x0e\n\x06symbol\x18\x02 \x01(\t\x12\x17\n\x0fparameter_sizes\x18\x03 \x03(\x04\x12\x34\n\nattributes\x18\x04 \x03(\x0b\x32 .flare.runtime.FunctionAttribute\".\n\x11\x46unctionAttribute\x12\n\n\x02id\x18\x01 \x01(\x05\x12\r\n\x05value\x18\x02 \x01(\x05\"/\n\tReference\x12\x12\n\nallocation\x18\x01 \x01(\t\x12\x0e\n\x06offset\x18\x02 \x01(\x04\"s\n\nRelocation\x12\x0e\n\x06offset\x18\x01 \x01(\x04\x12(\n\x06target\x18\x02 \x01(\x0b\x32\x18.flare.runtime.Reference\x12+\n\x04kind\x18\x03 \x01(\x0e\x32\x1d.flare.runtime.RelocationKind\"D\n\x04\x42lob\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12.\n\x0brelocations\x18\x02 \x03(\x0b\x32\x19.flare.runtime.Relocation\";\n\tAttribute\x12\n\n\x02id\x18\x01 \x01(\x05\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.flare.runtime.Blob\"\x9d\x01\n\x06Kernel\x12\x10\n\x08\x66unction\x18\x01 \x01(\x04\x12\x0c\n\x04grid\x18\x02 \x03(\r\x12\r\n\x05\x62lock\x18\x03 \x03(\r\x12\x0e\n\x06shared\x18\x04 \x01(\r\x12&\n\targuments\x18\x05 \x03(\x0b\x32\x13.flare.runtime.Blob\x12,\n\nattributes\x18\x06 \x03(\x0b\x32\x18.flare.runtime.Attribute\"\x8a\x01\n\x06Memset\x12-\n\x0b\x64\x65stination\x18\x01 \x01(\x0b\x32\x18.flare.runtime.Reference\x12\r\n\x05pitch\x18\x02 \x01(\x04\x12\r\n\x05value\x18\x03 \x01(\r\x12\x14\n\x0c\x65lement_size\x18\x04 \x01(\r\x12\r\n\x05width\x18\x05 \x01(\x04\x12\x0e\n\x06height\x18\x06 \x01(\x04\"o\n\x04\x43opy\x12(\n\x06source\x18\x01 \x01(\x0b\x32\x18.flare.runtime.Reference\x12-\n\x0b\x64\x65stination\x18\x02 \x01(\x0b\x32\x18.flare.runtime.Reference\x12\x0e\n\x06nbytes\x18\x03 \x01(\x04\"\x07\n\x05\x45mpty\"\x1a\n\tEventWait\x12\r\n\x05\x65vent\x18\x01 \x01(\t\"\xe1\x01\n\x04Node\x12\'\n\x06kernel\x18\x01 \x01(\x0b\x32\x15.flare.runtime.KernelH\x00\x12\'\n\x06memset\x18\x02 \x01(\x0b\x32\x15.flare.runtime.MemsetH\x00\x12#\n\x04\x63opy\x18\x03 \x01(\x0b\x32\x13.flare.runtime.CopyH\x00\x12%\n\x05\x65mpty\x18\x04 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12.\n\nevent_wait\x18\x05 \x01(\x0b\x32\x18.flare.runtime.EventWaitH\x00\x42\x0b\n\toperation\"9\n\x04\x45\x64ge\x12\x0e\n\x06source\x18\x01 \x01(\x04\x12\x13\n\x0b\x64\x65stination\x18\x02 \x01(\x04\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"\xfc\x01\n\tExecution\x12\x13\n\x0b\x61\x62i_version\x18\x01 \x01(\r\x12\x12\n\nmax_active\x18\x02 \x01(\x04\x12\x19\n\x11input_frame_bytes\x18\x03 \x01(\x04\x12&\n\x07\x62uffers\x18\x04 \x03(\x0b\x32\x15.flare.runtime.Buffer\x12-\n\x06graphs\x18\x05 \x03(\x0b\x32\x1d.flare.runtime.ExecutionGraph\x12\x31\n\rtuning_graphs\x18\x06 \x03(\x0b\x32\x1a.flare.runtime.TuningGraph\x12!\n\x19\x63ompletion_token_capacity\x18\x07 \x01(\r\"C\n\x0bTuningGraph\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07outputs\x18\x02 \x03(\t\x12\x15\n\rstatus_buffer\x18\x03 \x01(\t\"\x82\x01\n\x06\x42uffer\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06nbytes\x18\x02 \x01(\x04\x12\r\n\x05\x64type\x18\x03 \x01(\t\x12\r\n\x05shape\x18\x04 \x03(\x04\x12\x0f\n\x07strides\x18\x05 \x03(\x04\x12+\n\treference\x18\x06 \x01(\x0b\x32\x18.flare.runtime.Reference\"\x95\x04\n\x0e\x45xecutionGraph\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04kind\x18\x02 \x01(\t\x12\x15\n\rsampling_mode\x18\x03 \x01(\t\x12\x16\n\x0e\x62\x61tch_capacity\x18\x04 \x01(\x04\x12\x16\n\x0etoken_capacity\x18\x05 \x01(\x04\x12\x33\n\x0einput_bindings\x18\x06 \x03(\x0b\x32\x1b.flare.runtime.InputBinding\x12\x19\n\x0c\x64raft_tokens\x18\x07 \x01(\rH\x00\x88\x01\x01\x12\x1a\n\rverify_tokens\x18\x08 \x01(\rH\x01\x88\x01\x01\x12\x30\n\x0c\x64raft_source\x18\t \x01(\x0e\x32\x1a.flare.runtime.DraftSource\x12 \n\x13speculation_enabled\x18\n \x01(\x08H\x02\x88\x01\x01\x12\x1d\n\x15\x65stimated_duration_ns\x18\x0b \x01(\x04\x12\x18\n\x10prefill_capacity\x18\x0c \x01(\x04\x12\x1d\n\x15packed_token_capacity\x18\r \x01(\x04\x12\x18\n\x10\x63onfidence_width\x18\x0e \x01(\r\x12\x33\n\x0etuned_bindings\x18\x0f \x03(\x0b\x32\x1b.flare.runtime.TunedBindingB\x0f\n\r_draft_tokensB\x10\n\x0e_verify_tokensB\x16\n\x14_speculation_enabled\"H\n\x0cInputBinding\x12\x13\n\x0b\x62uffer_name\x18\x01 \x01(\t\x12\x13\n\x0bhost_offset\x18\x02 \x01(\x04\x12\x0e\n\x06nbytes\x18\x03 \x01(\x04*\x8e\x01\n\x0bHistoryKind\x12\x17\n\x13HISTORY_UNSPECIFIED\x10\x00\x12\x13\n\x0fPAGED_KEY_VALUE\x10\x01\x12\x10\n\x0cPAGED_LATENT\x10\x02\x12\x12\n\x0eSLIDING_WINDOW\x10\x03\x12\x13\n\x0fRECURRENT_STATE\x10\x04\x12\x16\n\x12\x43ONTINUATION_STATE\x10\x05*\xfa\x06\n\x0bValueSource\x12\x15\n\x11VALUE_UNSPECIFIED\x10\x00\x12\x0c\n\x08\x43ONSTANT\x10\x01\x12\x10\n\x0cREQUEST_SLOT\x10\x02\x12\x16\n\x12REQUEST_GENERATION\x10\x03\x12\r\n\tLIVE_ROWS\x10\x04\x12\x10\n\x0cQUERY_LENGTH\x10\x05\x12\x12\n\x0eSAMPLE_ENABLED\x10\x06\x12\x10\n\x0cPROMPT_TOKEN\x10\x07\x12\x11\n\rPENDING_TOKEN\x10\x08\x12\x1c\n\x18PROMPT_AND_PENDING_TOKEN\x10\t\x12\x13\n\x0fPROMPT_POSITION\x10\n\x12\x17\n\x13IMAGE_FEATURE_INDEX\x10\x0b\x12\x16\n\x12IMAGE_LEFT_CONTEXT\x10\x0c\x12\x17\n\x13IMAGE_RIGHT_CONTEXT\x10\r\x12\x0e\n\nPAGE_TABLE\x10\x0e\x12\x13\n\x0fLIVE_STATE_ROWS\x10\x0f\x12\x12\n\x0eLIVE_STATE_ROW\x10\x10\x12\x14\n\x10PREFIX_STATE_ROW\x10\x11\x12\x16\n\x12SNAPSHOT_STATE_ROW\x10\x12\x12\x13\n\x0f\x43ONSUMED_TOKENS\x10\x13\x12\x1c\n\x18SNAPSHOT_CONSUMED_TOKENS\x10\x14\x12\x0f\n\x0bPREFILL_ROW\x10\x15\x12\x11\n\rSAMPLING_SEED\x10\x16\x12\x17\n\x13SAMPLING_PARAMETERS\x10\x17\x12\x12\n\x0eSAMPLING_TOP_K\x10\x18\x12\x12\n\x0eMAX_NEW_TOKENS\x10\x19\x12\x11\n\rEOS_TOKEN_IDS\x10\x1a\x12\r\n\tEOS_COUNT\x10\x1b\x12\x14\n\x10PROMPT_HISTOGRAM\x10\x1c\x12\x10\n\x0cGRAMMAR_MASK\x10\x1d\x12\x13\n\x0fGRAMMAR_ENABLED\x10\x1e\x12\x12\n\x0e\x43OPY_TOKEN_IDS\x10\x1f\x12\x14\n\x10\x43OPY_TOKEN_COUNT\x10 \x12\x18\n\x14\x43OPY_CONSUMED_TOKENS\x10!\x12\x13\n\x0f\x43OPY_GENERATION\x10\"\x12\x15\n\x11\x43OPY_GRAMMAR_MASK\x10#\x12\x16\n\x12\x43OPY_GRAMMAR_COUNT\x10$\x12\x10\n\x0cIMAGE_TENSOR\x10%\x12\x16\n\x12IMAGE_FEATURE_SLOT\x10&\x12\x17\n\x13PROMPT_TARGET_TOKEN\x10\'\x12\x1c\n\x18SPECULATIVE_PREFIX_LIMIT\x10(*|\n\rInputIndexing\x12\x18\n\x14INDEXING_UNSPECIFIED\x10\x00\x12\n\n\x06SCALAR\x10\x01\x12\x12\n\x0eSUBMISSION_ROW\x10\x02\x12\x0f\n\x0bREQUEST_ROW\x10\x03\x12\x10\n\x0cPACKED_TOKEN\x10\x04\x12\x0e\n\nACTIVE_ROW\x10\x05*T\n\x0fStateAccessKind\x12\x1c\n\x18STATE_ACCESS_UNSPECIFIED\x10\x00\x12\x08\n\x04READ\x10\x01\x12\t\n\x05WRITE\x10\x02\x12\x0e\n\nREAD_WRITE\x10\x03*\xf5\x01\n\x0eGraphOperation\x12\x1f\n\x1bGRAPH_OPERATION_UNSPECIFIED\x10\x00\x12\x08\n\x04INIT\x10\x01\x12\x0b\n\x07RESTORE\x10\x02\x12\x0c\n\x08SNAPSHOT\x10\x03\x12\x0b\n\x07PREFILL\x10\x04\x12\n\n\x06\x44\x45\x43ODE\x10\x05\x12\n\n\x06RESUME\x10\x06\x12\n\n\x06VISION\x10\x07\x12\x0f\n\x0bSPECULATIVE\x10\x08\x12\t\n\x05MIXED\x10\t\x12\x12\n\x0eSCORED_PREFILL\x10\x0c\"\x04\x08\n\x10\n\"\x04\x08\x0b\x10\x0b\"\x04\x08\r\x10\r*\x08PROPOSAL*\x0cVERIFICATION*\x12PACKED_SPECULATIVE*\xa1\x01\n\x0eToolParserKind\x12\x1b\n\x17TOOL_PARSER_UNSPECIFIED\x10\x00\x12\x0e\n\nQWEN_CODER\x10\x01\x12\x15\n\x11GLM_ARGUMENT_TAGS\x10\x02\x12\x08\n\x04\x44SML\x10\x03\x12\x08\n\x04XTML\x10\x04\x12\x0f\n\x0bMINIMAX_XML\x10\x05\x12\x0e\n\nGEMMA_CALL\x10\x06\x12\x08\n\x04\x41TEM\x10\x07\x12\x0c\n\x08TML_JSON\x10\x08*\xa6\x01\n\x13ReasoningParserKind\x12 \n\x1cREASONING_PARSER_UNSPECIFIED\x10\x00\x12\x0e\n\nTHINK_TAGS\x10\x01\x12\x0e\n\nXTML_THINK\x10\x02\x12\x0c\n\x08MM_THINK\x10\x03\x12\x13\n\x0f\x43HANNEL_THOUGHT\x10\x04\x12\x15\n\x11RECIPIENT_HEADERS\x10\x05\x12\x13\n\x0f\x43ONTENT_HEADERS\x10\x06*\xf4\x02\n\x14ImagePreparationKind\x12!\n\x1dIMAGE_PREPARATION_UNSPECIFIED\x10\x00\x12\x1a\n\x16QWEN_IMAGE_PREPARATION\x10\x01\x12\x1a\n\x16MIMO_IMAGE_PREPARATION\x10\x02\x12\x1e\n\x1a\x44\x45\x45PSEEK_IMAGE_PREPARATION\x10\x03\x12\x1d\n\x19MOONVIT_IMAGE_PREPARATION\x10\x04\x12\x1a\n\x16STEP_IMAGE_PREPARATION\x10\x05\x12\x1b\n\x17RADIO_IMAGE_PREPARATION\x10\x06\x12(\n$DEEPSEEK_ROW_MAJOR_IMAGE_PREPARATION\x10\x07\x12\x19\n\x15GLM_IMAGE_PREPARATION\x10\x08\x12!\n\x1dPATCH_MERGE_IMAGE_PREPARATION\x10\t\x12!\n\x1dPADDED_GRID_IMAGE_PREPARATION\x10\n*\xb6\x02\n\x0fImageLayoutKind\x12\x1c\n\x18IMAGE_LAYOUT_UNSPECIFIED\x10\x00\x12\x15\n\x11QWEN_IMAGE_LAYOUT\x10\x01\x12\x15\n\x11MIMO_IMAGE_LAYOUT\x10\x02\x12\x19\n\x15\x44\x45\x45PSEEK_IMAGE_LAYOUT\x10\x03\x12\x18\n\x14MOONVIT_IMAGE_LAYOUT\x10\x04\x12\x15\n\x11STEP_IMAGE_LAYOUT\x10\x05\x12\x16\n\x12RADIO_IMAGE_LAYOUT\x10\x06\x12#\n\x1f\x44\x45\x45PSEEK_ROW_MAJOR_IMAGE_LAYOUT\x10\x07\x12\x14\n\x10GLM_IMAGE_LAYOUT\x10\x08\x12\x1a\n\x16\x42RACKETED_IMAGE_LAYOUT\x10\t\x12\x1c\n\x18\x43LASS_TOKEN_IMAGE_LAYOUT\x10\n*P\n\rPromptEncoder\x12\x10\n\x0cPROMPT_JINJA\x10\x00\x12\x0f\n\x0b\x44\x45\x45PSEEK_V4\x10\x01\x12\x10\n\x0c\x44\x45\x45PSEEK_V41\x10\x02\x12\n\n\x06TML_V0\x10\x03*B\n\x10TemplateEncoding\x12\x11\n\rTEMPLATE_TEXT\x10\x00\x12\x1b\n\x17TEMPLATE_TOKEN_SEGMENTS\x10\x01*/\n\x06Memory\x12\n\n\x06\x44\x45VICE\x10\x00\x12\x0f\n\x0bMAPPED_HOST\x10\x01\x12\x08\n\x04PEER\x10\x02*(\n\x08\x43\x61pacity\x12\x0e\n\nSTATE_ROWS\x10\x00\x12\x0c\n\x08KV_PAGES\x10\x01*-\n\x0eRelocationKind\x12\x0b\n\x07POINTER\x10\x00\x12\x0e\n\nTENSOR_MAP\x10\x01*4\n\x0b\x44raftSource\x12\x0c\n\x08NO_DRAFT\x10\x00\x12\x07\n\x03MTP\x10\x01\x12\x0e\n\nCOPY_DRAFT\x10\x02\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12runtime_core.proto\x12\rflare.runtime\"\x99\x01\n\x0eProtectedIndex\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x0f\n\x07\x63ore_id\x18\x02 \x01(\x0c\x12\x0e\n\x06signer\x18\x03 \x01(\x0c\x12*\n\x06groups\x18\x04 \x03(\x0b\x32\x1a.flare.runtime.SignedGroup\x12)\n\x05ranks\x18\x05 \x03(\x0b\x32\x1a.flare.runtime.SignedGroup\"<\n\x0bSignedGroup\x12\x0c\n\x04kind\x18\x01 \x01(\r\x12\x0c\n\x04\x62ody\x18\x02 \x01(\x0c\x12\x11\n\tsignature\x18\x03 \x01(\x0c\"\xcc\x01\n\x11PublisherMetadata\x12(\n\x05index\x18\x01 \x01(\x0b\x32\x19.flare.runtime.ModelIndex\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x17\n\x0fofficial_source\x18\x03 \x01(\t\x12\x14\n\x0cgroup_hashes\x18\x04 \x03(\x0c\x12\x13\n\x0b\x64\x65velopment\x18\x05 \x01(\x08\x12\r\n\x05ranks\x18\x06 \x01(\r\x12\'\n\x07license\x18\x07 \x01(\x0b\x32\x16.flare.runtime.License\"%\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04link\x18\x02 \x01(\t\"F\n\x0fResourceCatalog\x12\x33\n\tresources\x18\x01 \x03(\x0b\x32 .flare.runtime.ProtectedResource\"^\n\x11ProtectedResource\x12\x0e\n\x06offset\x18\x01 \x01(\x04\x12\x14\n\x0cstored_bytes\x18\x02 \x01(\x04\x12\x13\n\x0bplain_bytes\x18\x03 \x01(\x04\x12\x0e\n\x06sha256\x18\x04 \x03(\x0c\"`\n\nStoredData\x12\x0e\n\x06nbytes\x18\x01 \x01(\x04\x12\x0e\n\x06sha256\x18\x02 \x01(\x0c\x12\x0e\n\x06offset\x18\x03 \x01(\x04\x12\"\n\x05parts\x18\x04 \x03(\x0b\x32\x13.flare.runtime.Span\"\xd9\x01\n\x0eRankProtection\x12\x0c\n\x04rank\x18\x01 \x01(\r\x12*\n\x06\x62undle\x18\x02 \x01(\x0b\x32\x1a.flare.runtime.BundleIndex\x12\x33\n\x0b\x65xecutables\x18\x03 \x01(\x0b\x32\x1e.flare.runtime.ResourceCatalog\x12/\n\x07support\x18\x04 \x01(\x0b\x32\x1e.flare.runtime.ResourceCatalog\x12\'\n\x04\x64\x61ta\x18\x05 \x01(\x0b\x32\x19.flare.runtime.StoredData\"\xcf\x03\n\nModelIndex\x12\x0f\n\x07version\x18\x01 \x01(\r\x12)\n\x05ranks\x18\x02 \x03(\x0b\x32\x1a.flare.runtime.BundleIndex\x12.\n\x0bpeer_memory\x18\x03 \x03(\x0b\x32\x19.flare.runtime.PeerMemory\x12$\n\x06\x61ssets\x18\x05 \x03(\x0b\x32\x14.flare.runtime.Asset\x12%\n\x06\x65ngram\x18\x06 \x01(\x0b\x32\x15.flare.runtime.Engram\x12\x43\n\x11sampling_defaults\x18\x07 \x01(\x0b\x32(.flare.runtime.DefaultSamplingParameters\x12\x31\n\x08\x66rontend\x18\x08 \x01(\x0b\x32\x1f.flare.runtime.FrontendSettings\x12\'\n\x07\x63luster\x18\t \x01(\x0b\x32\x16.flare.runtime.Cluster\x12\x30\n\x08\x63ontract\x18\n \x01(\x0b\x32\x1e.flare.runtime.ServingContract\x12/\n\x07rebuild\x18\x0b \x01(\x0b\x32\x1e.flare.runtime.RebuildManifestJ\x04\x08\x04\x10\x05\"\xf2\x06\n\x0fRebuildManifest\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x0e\n\x06recipe\x18\x02 \x01(\t\x12\x17\n\x0fsource_identity\x18\x03 \x01(\t\x12\x1d\n\x15\x63heckpoint_repository\x18\x04 \x01(\t\x12\x1b\n\x13\x63heckpoint_revision\x18\x05 \x01(\t\x12\x17\n\x0ftensor_parallel\x18\x06 \x01(\r\x12\x17\n\x0fmax_concurrency\x18\x07 \x01(\x04\x12\x15\n\rprefill_chunk\x18\x08 \x01(\x04\x12\x19\n\x11image_concurrency\x18\t \x01(\x04\x12\x18\n\x10\x63ontext_parallel\x18\n \x01(\x08\x12\x16\n\x0estream_overlap\x18\x0b \x01(\x08\x12\x15\n\rmixed_prefill\x18\x0c \x01(\x08\x12%\n\x18\x65xpert_scale_compression\x18\r \x01(\tH\x00\x88\x01\x01\x12\x18\n\x10mla_compute_mode\x18\x0e \x01(\t\x12\x11\n\troce_ring\x18\x0f \x01(\x08\x12\x14\n\x0conline_mxfp8\x18\x10 \x01(\x08\x12\x1b\n\x13language_model_only\x18\x11 \x01(\x08\x12\x36\n\x0bspeculation\x18\x12 \x01(\x0b\x32!.flare.runtime.RebuildSpeculation\x12\x19\n\x11\x63ompiled_forwards\x18\x13 \x01(\x08\x12\x15\n\rreserve_bytes\x18\x14 \x01(\x04\x12\x18\n\x10local_checkpoint\x18\x15 \x01(\x08\x12\x12\n\nroce_chain\x18\x16 \x01(\x08\x12\x14\n\x0cjoint_tuning\x18\x17 \x01(\x08\x12\x1e\n\x16shard_hyperconnections\x18\x18 \x01(\x08\x12\x19\n\x11\x63ollective_tuning\x18\x19 \x01(\t\x12\x1e\n\x16\x63ollective_ladder_json\x18\x1a \x01(\t\x12\x1c\n\x14\x66orce_moe_decode_a16\x18\x1b \x01(\x08\x12\x19\n\x11row_shard_prefill\x18\x1c \x01(\x08\x12\x1b\n\x13row_shard_attention\x18\x1d \x01(\x08\x12\x1a\n\x12online_mxfp8_scope\x18\x1e \x01(\t\x12\x1d\n\x15\x63ontext_parallel_size\x18\x1f \x01(\rB\x1b\n\x19_expert_scale_compression\"\xfe\x03\n\x12RebuildSpeculation\x12\x14\n\x0c\x64raft_tokens\x18\x01 \x01(\r\x12\x10\n\x08\x61\x64\x61ptive\x18\x02 \x01(\x08\x12\x17\n\x0fproposal_source\x18\x03 \x01(\t\x12\x16\n\x0ehistory_drafts\x18\x04 \x01(\x08\x12\x16\n\x0etoken_capacity\x18\x05 \x01(\x04\x12\x1b\n\x13packed_verification\x18\x06 \x01(\x08\x12\x1e\n\x16\x64raft_prefix_selection\x18\x07 \x01(\t\x12$\n\x17\x64raft_head_quantization\x18\x08 \x01(\tH\x00\x88\x01\x01\x12%\n\x18markov_head_quantization\x18\t \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0cgraph_budget\x18\n \x01(\x04H\x02\x88\x01\x01\x12$\n\x1cvocabulary_parallel_sampling\x18\x0b \x01(\x08\x12\x0e\n\x06method\x18\x0c \x01(\t\x12\x10\n\x08mtp_only\x18\r \x01(\x08\x12$\n\x17\x64raft_body_quantization\x18\x0e \x01(\tH\x03\x88\x01\x01\x42\x1a\n\x18_draft_head_quantizationB\x1b\n\x19_markov_head_quantizationB\x0f\n\r_graph_budgetB\x1a\n\x18_draft_body_quantization\"e\n\x0eRebuildWeights\x12\x0f\n\x07version\x18\x01 \x01(\r\x12-\n\x07tensors\x18\x02 \x03(\x0b\x32\x1c.flare.runtime.RebuildTensor\x12\x13\n\x0bunavailable\x18\x03 \x03(\t\"\xa2\x01\n\rRebuildTensor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x64type\x18\x02 \x01(\t\x12\x15\n\relement_bytes\x18\x03 \x01(\r\x12\r\n\x05shape\x18\x04 \x03(\x04\x12\x0e\n\x06stride\x18\x05 \x03(\x04\x12\x13\n\x0blayout_type\x18\x06 \x01(\t\x12)\n\x07storage\x18\x07 \x01(\x0b\x32\x18.flare.runtime.Reference\"\xcb\x06\n\x0fServingContract\x12\x10\n\x08model_id\x18\x01 \x01(\t\x12\x10\n\x08identity\x18\x02 \x01(\t\x12\x34\n\x0emodel_contract\x18\x03 \x01(\x0b\x32\x1c.flare.runtime.ModelContract\x12\x1b\n\x13max_sequence_tokens\x18\x04 \x01(\x04\x12\x1a\n\x12prefill_chunk_size\x18\x05 \x01(\x04\x12\r\n\x05pages\x18\x06 \x01(\x04\x12\x12\n\nstate_rows\x18\x07 \x01(\x04\x12\x11\n\tsnapshots\x18\x08 \x01(\x04\x12\x13\n\x0bpage_tokens\x18\t \x01(\x04\x12\x1a\n\x12speculative_widths\x18\x0b \x03(\x04\x12$\n\x17speculative_fixed_width\x18\x0c \x01(\x04H\x00\x88\x01\x01\x12\x16\n\x0ehistory_drafts\x18\r \x01(\x08\x12#\n\x1bspeculative_proposal_source\x18\x0e \x01(\t\x12\x1b\n\x0emax_new_tokens\x18\x0f \x01(\x04H\x01\x88\x01\x01\x12\x17\n\nmax_active\x18\x10 \x01(\x04H\x02\x88\x01\x01\x12\x14\n\x0cserved_names\x18\x11 \x03(\t\x12\x1a\n\rreserve_bytes\x18\x12 \x01(\x04H\x03\x88\x01\x01\x12\x1b\n\x0equeue_capacity\x18\x13 \x01(\x04H\x04\x88\x01\x01\x12\x1d\n\x10\x66rontend_workers\x18\x14 \x01(\x04H\x05\x88\x01\x01\x12\x1b\n\x0emax_body_bytes\x18\x15 \x01(\x04H\x06\x88\x01\x01\x12\x14\n\x0c\x63ontext_hint\x18\x16 \x01(\t\x12\x15\n\rcapacity_hint\x18\x17 \x01(\t\x12\"\n\x1aspeculative_token_capacity\x18\x18 \x01(\x04\x12 \n\x13\x64\x65\x63ode_weight_bytes\x18\x19 \x01(\x04H\x07\x88\x01\x01\x42\x1a\n\x18_speculative_fixed_widthB\x11\n\x0f_max_new_tokensB\r\n\x0b_max_activeB\x10\n\x0e_reserve_bytesB\x11\n\x0f_queue_capacityB\x13\n\x11_frontend_workersB\x11\n\x0f_max_body_bytesB\x16\n\x14_decode_weight_bytesJ\x04\x08\n\x10\x0b\"\x8b\x02\n\rModelContract\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x12\n\nvocabulary\x18\x02 \x01(\x04\x12\x18\n\x10input_vocabulary\x18\x03 \x01(\x04\x12-\n\x06vision\x18\x04 \x01(\x0b\x32\x1d.flare.runtime.VisionCapacity\x12+\n\x05\x63\x61\x63he\x18\x05 \x01(\x0b\x32\x1c.flare.runtime.CacheContract\x12,\n\x06graphs\x18\x06 \x03(\x0b\x32\x1c.flare.runtime.GraphContract\x12\x31\n\x08\x66rontend\x18\x07 \x01(\x0b\x32\x1f.flare.runtime.FrontendContract\"\x90\x01\n\x0eVisionCapacity\x12\x15\n\rfeature_slots\x18\x05 \x01(\x04\x12\x1c\n\x14\x66\x65\x61tures_per_request\x18\x01 \x01(\x04\x12\x12\n\nmax_images\x18\x02 \x01(\x04\x12\x1b\n\x13patches_per_feature\x18\x03 \x01(\x04\x12\x18\n\x10patch_capacities\x18\x04 \x03(\x04\"\x86\x01\n\rCacheContract\x12\x13\n\x0bpage_tokens\x18\x01 \x01(\x04\x12-\n\x07regions\x18\x02 \x03(\x0b\x32\x1c.flare.runtime.HistoryRegion\x12\x31\n\x0bpage_tables\x18\x03 \x03(\x0b\x32\x1c.flare.runtime.PageTableView\"\x96\x01\n\rHistoryRegion\x12(\n\x04kind\x18\x01 \x01(\x0e\x32\x1a.flare.runtime.HistoryKind\x12)\n\x07storage\x18\x02 \x01(\x0b\x32\x18.flare.runtime.StateView\x12\x19\n\x11tokens_per_record\x18\x03 \x01(\x04\x12\x15\n\rwindow_tokens\x18\x04 \x01(\x04\"4\n\rPageTableView\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\t\x12\x13\n\x0bpage_offset\x18\x02 \x01(\x04\"\xb1\x01\n\x0cValueBinding\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\t\x12*\n\x06source\x18\x02 \x01(\x0e\x32\x1a.flare.runtime.ValueSource\x12.\n\x08indexing\x18\x03 \x01(\x0e\x32\x1c.flare.runtime.InputIndexing\x12\x14\n\x0cvalue_offset\x18\x04 \x01(\x12\x12\x0f\n\x07padding\x18\x05 \x01(\x12\x12\x0e\n\x06tensor\x18\x06 \x01(\t\"K\n\tStateView\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\t\x12\x0e\n\x06offset\x18\x02 \x01(\x04\x12\x0e\n\x06stride\x18\x03 \x01(\x04\x12\x0e\n\x06nbytes\x18\x04 \x01(\x04\"e\n\x0bStateAccess\x12&\n\x04view\x18\x01 \x01(\x0b\x32\x18.flare.runtime.StateView\x12.\n\x06\x61\x63\x63\x65ss\x18\x02 \x01(\x0e\x32\x1e.flare.runtime.StateAccessKind\"O\n\x10\x43ompletionCursor\x12\'\n\x05state\x18\x01 \x01(\x0b\x32\x18.flare.runtime.StateView\x12\x12\n\nlag_tokens\x18\x02 \x01(\x04\"4\n\x0cTunedBinding\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\t\x12\x14\n\x0ctuning_graph\x18\x02 \x01(\t\"\xfa\x02\n\rGraphContract\x12\r\n\x05graph\x18\x01 \x01(\t\x12+\n\x06inputs\x18\x02 \x03(\x0b\x32\x1b.flare.runtime.ValueBinding\x12)\n\x05state\x18\x03 \x03(\x0b\x32\x1a.flare.runtime.StateAccess\x12\x1e\n\x16state_rows_per_request\x18\x04 \x01(\x04\x12\x35\n\x0c\x64raft_cursor\x18\x05 \x01(\x0b\x32\x1f.flare.runtime.CompletionCursor\x12\x18\n\x10\x65mbedding_tables\x18\x06 \x03(\r\x12/\n\rpending_token\x18\x07 \x01(\x0b\x32\x18.flare.runtime.StateView\x12+\n\x04kind\x18\x08 \x01(\x0e\x32\x1d.flare.runtime.GraphOperation\x12\x33\n\x0etuned_bindings\x18\t \x03(\x0b\x32\x1b.flare.runtime.TunedBinding\"k\n\nToolParser\x12+\n\x04kind\x18\x01 \x01(\x0e\x32\x1d.flare.runtime.ToolParserKind\x12\x19\n\x11wrapping_newlines\x18\x02 \x01(\r\x12\x15\n\rtag_separator\x18\x03 \x01(\t\"\x95\x01\n\x0fReasoningParser\x12\x30\n\x04kind\x18\x01 \x01(\x0e\x32\".flare.runtime.ReasoningParserKind\x12\x0c\n\x04open\x18\x02 \x01(\t\x12\r\n\x05\x63lose\x18\x03 \x01(\t\x12\x19\n\x11wrapping_newlines\x18\x04 \x01(\r\x12\x18\n\x10terminal_markers\x18\x05 \x03(\t\"\xca\x01\n\x0ePrefixBoundary\x12\r\n\x05roles\x18\x01 \x03(\t\x12\x18\n\x0b\x61ppend_role\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x1e\n\x16\x63ontinue_final_message\x18\x03 \x01(\x08\x12\x18\n\x10minimum_messages\x18\x04 \x01(\x04\x12\x16\n\x0erequired_start\x18\x05 \x01(\t\x12\x14\n\x0crequired_end\x18\x06 \x01(\t\x12\x17\n\x0frequired_marker\x18\x07 \x01(\tB\x0e\n\x0c_append_role\"y\n\tImageSpan\x12\x13\n\x0bstart_token\x18\x01 \x01(\r\x12\x11\n\tend_token\x18\x02 \x01(\r\x12\x12\n\nmax_tokens\x18\x03 \x01(\x04\x12\x17\n\x0finterior_tokens\x18\x04 \x03(\r\x12\x17\n\x0funscoped_tokens\x18\x05 \x03(\r\"Q\n\x10ImageTokenLayout\x12\x15\n\rfeature_token\x18\x01 \x01(\r\x12&\n\x04span\x18\x02 \x01(\x0b\x32\x18.flare.runtime.ImageSpan\"\xf2\x03\n\x10\x46rontendContract\x12(\n\x05tools\x18\x01 \x01(\x0b\x32\x19.flare.runtime.ToolParser\x12\x31\n\treasoning\x18\x02 \x01(\x0b\x32\x1e.flare.runtime.ReasoningParser\x12\x43\n\x11image_preparation\x18\x03 \x01(\x0e\x32#.flare.runtime.ImagePreparationKindH\x00\x88\x01\x01\x12\x39\n\x0cimage_layout\x18\x04 \x01(\x0e\x32\x1e.flare.runtime.ImageLayoutKindH\x01\x88\x01\x01\x12\x17\n\x0frejected_tokens\x18\x05 \x03(\r\x12\x1b\n\x13position_dimensions\x18\x06 \x01(\r\x12\x18\n\x10\x61ssistant_marker\x18\x07 \x01(\t\x12\x1b\n\x13message_end_markers\x18\x08 \x03(\t\x12\x36\n\x0fprefix_boundary\x18\t \x01(\x0b\x32\x1d.flare.runtime.PrefixBoundary\x12\x35\n\x0cimage_tokens\x18\n \x01(\x0b\x32\x1f.flare.runtime.ImageTokenLayoutB\x14\n\x12_image_preparationB\x0f\n\r_image_layout\"1\n\x07\x43luster\x12\r\n\x05world\x18\x01 \x01(\r\x12\x11\n\texport_id\x18\x03 \x01(\tJ\x04\x08\x02\x10\x03\"\x8c\x03\n\x19\x44\x65\x66\x61ultSamplingParameters\x12\x11\n\x04seed\x18\x01 \x01(\x04H\x00\x88\x01\x01\x12\x18\n\x0btemperature\x18\x02 \x01(\x02H\x01\x88\x01\x01\x12\x12\n\x05top_k\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05top_p\x18\x04 \x01(\x02H\x03\x88\x01\x01\x12\x12\n\x05min_p\x18\x05 \x01(\x02H\x04\x88\x01\x01\x12\x1f\n\x12repetition_penalty\x18\x06 \x01(\x02H\x05\x88\x01\x01\x12\x1d\n\x10presence_penalty\x18\x07 \x01(\x02H\x06\x88\x01\x01\x12\x1e\n\x11\x66requency_penalty\x18\x08 \x01(\x02H\x07\x88\x01\x01\x12-\n\x06ranges\x18\t \x01(\x0b\x32\x1d.flare.runtime.SamplingRangesB\x07\n\x05_seedB\x0e\n\x0c_temperatureB\x08\n\x06_top_kB\x08\n\x06_top_pB\x08\n\x06_min_pB\x15\n\x13_repetition_penaltyB\x13\n\x11_presence_penaltyB\x14\n\x12_frequency_penalty\"H\n\x12SamplingFloatRange\x12\x10\n\x03min\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12\x10\n\x03max\x18\x02 \x01(\x02H\x01\x88\x01\x01\x42\x06\n\x04_minB\x06\n\x04_max\"F\n\x10SamplingIntRange\x12\x10\n\x03min\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x10\n\x03max\x18\x02 \x01(\x05H\x01\x88\x01\x01\x42\x06\n\x04_minB\x06\n\x04_max\"\x96\x03\n\x0eSamplingRanges\x12\x36\n\x0btemperature\x18\x01 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12.\n\x05top_k\x18\x02 \x01(\x0b\x32\x1f.flare.runtime.SamplingIntRange\x12\x30\n\x05top_p\x18\x03 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12\x30\n\x05min_p\x18\x04 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12=\n\x12repetition_penalty\x18\x05 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12;\n\x10presence_penalty\x18\x06 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\x12<\n\x11\x66requency_penalty\x18\x07 \x01(\x0b\x32!.flare.runtime.SamplingFloatRange\"\xc5\x02\n\x10\x46rontendSettings\x12\x33\n\ttokenizer\x18\x01 \x01(\x0b\x32 .flare.runtime.TokenizerSettings\x12+\n\x05image\x18\x02 \x01(\x0b\x32\x1c.flare.runtime.ImageSettings\x12\x15\n\reos_token_ids\x18\x03 \x03(\r\x12:\n\x11template_encoding\x18\x04 \x01(\x0e\x32\x1f.flare.runtime.TemplateEncoding\x12\x34\n\x0eprompt_encoder\x18\x05 \x01(\x0e\x32\x1c.flare.runtime.PromptEncoder\x12\'\n\x1aimage_placeholder_token_id\x18\x06 \x01(\rH\x00\x88\x01\x01\x42\x1d\n\x1b_image_placeholder_token_id\"\xdc\x02\n\x11TokenizerSettings\x12\x16\n\tbos_token\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\teos_token\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tunk_token\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsep_token\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x16\n\tpad_token\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x16\n\tcls_token\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\nmask_token\x18\x07 \x01(\tH\x06\x88\x01\x01\x12;\n\x19\x61\x64\x64itional_special_tokens\x18\x08 \x01(\x0b\x32\x18.flare.runtime.TokenListB\x0c\n\n_bos_tokenB\x0c\n\n_eos_tokenB\x0c\n\n_unk_tokenB\x0c\n\n_sep_tokenB\x0c\n\n_pad_tokenB\x0c\n\n_cls_tokenB\r\n\x0b_mask_token\"\x1b\n\tTokenList\x12\x0e\n\x06values\x18\x01 \x03(\t\"\xfc\x03\n\rImageSettings\x12\x30\n\x04qwen\x18\x01 \x01(\x0b\x32 .flare.runtime.QwenImageSettingsH\x00\x12\x30\n\x04mimo\x18\x02 \x01(\x0b\x32 .flare.runtime.MimoImageSettingsH\x00\x12)\n\tdeepseek4\x18\x03 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12\'\n\x07moonvit\x18\x04 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12$\n\x04step\x18\x05 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12%\n\x05radio\x18\x06 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12J\n\x12\x64\x65\x65pseek_row_major\x18\x07 \x01(\x0b\x32,.flare.runtime.DeepseekRowMajorImageSettingsH\x00\x12#\n\x03glm\x18\x08 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12=\n\x0bpatch_merge\x18\t \x01(\x0b\x32&.flare.runtime.PatchMergeImageSettingsH\x00\x12+\n\x0bpadded_grid\x18\n \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x42\t\n\x07profile\"\xb9\x01\n\x17PatchMergeImageSettings\x12\x12\n\npatch_size\x18\x01 \x01(\r\x12\x12\n\nmin_pixels\x18\x02 \x01(\x04\x12\x12\n\nmax_pixels\x18\x03 \x01(\x04\x12\x12\n\nimage_mean\x18\x04 \x03(\x02\x12\x11\n\timage_std\x18\x05 \x03(\x02\x12\x13\n\x0bimage_token\x18\x06 \x01(\t\x12\x13\n\x0bstart_token\x18\x07 \x01(\t\x12\x11\n\tend_token\x18\x08 \x01(\t\"\x87\x01\n\x1d\x44\x65\x65pseekRowMajorImageSettings\x12\x12\n\nmin_pixels\x18\x01 \x01(\x04\x12\x12\n\nmax_tokens\x18\x02 \x01(\x04\x12#\n\x16max_width_height_ratio\x18\x03 \x01(\x04H\x00\x88\x01\x01\x42\x19\n\x17_max_width_height_ratio\"\x82\x01\n\x11QwenImageSettings\x12\x12\n\nmin_pixels\x18\x01 \x01(\x04\x12\x12\n\nmax_pixels\x18\x02 \x01(\x04\x12\x15\n\rposition_side\x18\x03 \x01(\r\x12\x16\n\x0eimage_token_id\x18\x04 \x01(\r\x12\x16\n\x0evideo_token_id\x18\x05 \x01(\r\"b\n\x11MimoImageSettings\x12\x12\n\nmin_pixels\x18\x01 \x01(\x04\x12\x12\n\nmax_pixels\x18\x02 \x01(\x04\x12\x12\n\nimage_mean\x18\x03 \x03(\x02\x12\x11\n\timage_std\x18\x04 \x03(\x02\"|\n\x06\x45ngram\x12\x11\n\ttoken_map\x18\x01 \x03(\r\x12\x14\n\x0cpad_token_id\x18\x02 \x01(\r\x12\x1d\n\x15\x63ompressed_vocabulary\x18\x03 \x01(\r\x12*\n\x06tables\x18\x04 \x03(\x0b\x32\x1a.flare.runtime.EngramTable\"\x80\x04\n\x0b\x45ngramTable\x12\r\n\x05layer\x18\x01 \x01(\r\x12\x0c\n\x04rows\x18\x02 \x01(\x04\x12\x17\n\x0f\x65mbedding_width\x18\x03 \x01(\r\x12\x13\n\x0bscale_block\x18\x04 \x01(\r\x12$\n\x07weights\x18\x05 \x01(\x0b\x32\x13.flare.runtime.Span\x12#\n\x06scales\x18\x06 \x01(\x0b\x32\x13.flare.runtime.Span\x12\'\n\x04hash\x18\x07 \x01(\x0b\x32\x19.flare.runtime.EngramHash\x12\x15\n\rweight_buffer\x18\x08 \x01(\t\x12\x14\n\x0cscale_buffer\x18\t \x01(\t\x12\x13\n\x0bhash_buffer\x18\n \x01(\t\x12\x13\n\x0bready_event\x18\x0b \x01(\t\x12\x35\n\x08\x65ncoding\x18\x0c \x01(\x0e\x32#.flare.runtime.EngramTable.Encoding\x12\x16\n\x0erequest_buffer\x18\r \x01(\t\x12\x14\n\x0cglobal_scale\x18\x0e \x01(\x02\x12/\n\x06shards\x18\x0f \x03(\x0b\x32\x1f.flare.runtime.EngramTableShard\"E\n\x08\x45ncoding\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x11\n\rFP8_E4M3_E8M0\x10\x01\x12\x15\n\x11NVFP4_E4M3_GLOBAL\x10\x02\"\x8c\x01\n\x10\x45ngramTableShard\x12\x0c\n\x04rank\x18\x01 \x01(\r\x12\x11\n\tfirst_row\x18\x02 \x01(\x04\x12\x0c\n\x04rows\x18\x03 \x01(\x04\x12$\n\x07weights\x18\x04 \x01(\x0b\x32\x13.flare.runtime.Span\x12#\n\x06scales\x18\x05 \x01(\x0b\x32\x13.flare.runtime.Span\"\xc3\x01\n\nEngramHash\x12\x30\n\x06method\x18\x01 \x01(\x0e\x32 .flare.runtime.EngramHash.Method\x12\x0e\n\x06orders\x18\x02 \x03(\r\x12\x13\n\x0bmultipliers\x18\x03 \x03(\x04\x12\x0e\n\x06primes\x18\x04 \x03(\x04\x12\x0f\n\x07offsets\x18\x05 \x03(\x04\"=\n\x06Method\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nXOR_MODULO\x10\x01\x12\x12\n\x0e\x45OS_XOR_MODULO\x10\x02\"K\n\x05\x41sset\x12\x0c\n\x04name\x18\x01 \x01(\t\x12$\n\x07payload\x18\x02 \x01(\x0b\x32\x13.flare.runtime.Span\x12\x0e\n\x06sha256\x18\x03 \x01(\t\"\xd3\x04\n\x0b\x42undleIndex\x12\x14\n\x0c\x61rchitecture\x18\x01 \x03(\x05\x12.\n\x0b\x61llocations\x18\x02 \x03(\x0b\x32\x19.flare.runtime.Allocation\x12&\n\x07modules\x18\x03 \x03(\x0b\x32\x15.flare.runtime.Module\x12*\n\tfunctions\x18\x04 \x03(\x0b\x32\x17.flare.runtime.Function\x12)\n\x06graphs\x18\x05 \x03(\x0b\x32\x19.flare.runtime.GraphIndex\x12\x30\n\x0creservations\x18\x06 \x03(\x0b\x32\x1a.flare.runtime.Reservation\x12+\n\texecution\x18\x07 \x01(\x0b\x32\x18.flare.runtime.Execution\x12\'\n\x07\x63ontext\x18\x08 \x01(\x0b\x32\x16.flare.runtime.Context\x12<\n\tresources\x18\t \x03(\x0b\x32).flare.runtime.BundleIndex.ResourcesEntry\x12\x0e\n\x06\x65vents\x18\n \x03(\t\x12*\n\x04roce\x18\x0b \x01(\x0b\x32\x1c.flare.runtime.RoceTransport\x12\x36\n\x0frebuild_weights\x18\x0c \x01(\x0b\x32\x1d.flare.runtime.RebuildWeights\x1a\x45\n\x0eResourcesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.flare.runtime.Span:\x02\x38\x01\"\xf8\x01\n\rRoceTransport\x12\x11\n\tproxy_abi\x18\x01 \x01(\r\x12\r\n\x05world\x18\x02 \x01(\r\x12\x0c\n\x04rank\x18\x03 \x01(\r\x12\x16\n\x0eregion_address\x18\x04 \x01(\x04\x12\x14\n\x0cregion_bytes\x18\x05 \x01(\x04\x12\x18\n\x10\x63ounters_address\x18\x06 \x01(\x04\x12\x16\n\x0e\x63ounters_bytes\x18\x07 \x01(\x04\x12\x12\n\nslot_bytes\x18\x08 \x01(\x04\x12\x12\n\npipe_bytes\x18\t \x01(\x04\x12\x13\n\x0b\x63trl_offset\x18\n \x01(\x04\x12\x14\n\x0cvarlen_bytes\x18\x10 \x01(\x04J\x04\x08\x0b\x10\x10\"v\n\nGraphIndex\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x66lags\x18\x02 \x01(\x04\x12\x10\n\x08resource\x18\x03 \x01(\t\x12\x12\n\nnode_count\x18\x04 \x01(\x04\x12\x12\n\nedge_count\x18\x05 \x01(\x04\x12\x11\n\tfunctions\x18\x06 \x03(\x04\"M\n\nPeerMemory\x12\r\n\x05owner\x18\x01 \x01(\r\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\x04\x12\x0e\n\x06nbytes\x18\x03 \x01(\x04\x12\x0f\n\x07readers\x18\x04 \x03(\r\"M\n\x07\x43ontext\x12\r\n\x05\x66lags\x18\x01 \x01(\r\x12\r\n\x05\x63\x61\x63he\x18\x02 \x01(\x05\x12$\n\x06limits\x18\x03 \x03(\x0b\x32\x14.flare.runtime.Limit\"\"\n\x05Limit\x12\n\n\x02id\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\x04\".\n\x0bReservation\x12\x0f\n\x07\x61\x64\x64ress\x18\x01 \x01(\x04\x12\x0e\n\x06nbytes\x18\x02 \x01(\x04\"8\n\x04Span\x12\x0e\n\x06offset\x18\x01 \x01(\x04\x12\x0e\n\x06nbytes\x18\x02 \x01(\x04\x12\x10\n\x08\x66ilename\x18\x03 \x01(\t\"\xa2\x02\n\nAllocation\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06nbytes\x18\x02 \x01(\x04\x12\x11\n\talignment\x18\x03 \x01(\x04\x12\x11\n\x04\x66ile\x18\x04 \x01(\tH\x00\x88\x01\x01\x12\'\n\ndata_spans\x18\x05 \x03(\x0b\x32\x13.flare.runtime.Span\x12.\n\x0brelocations\x18\x06 \x03(\x0b\x32\x19.flare.runtime.Relocation\x12\x14\n\x07\x61\x64\x64ress\x18\x07 \x01(\x04H\x01\x88\x01\x01\x12%\n\x06memory\x18\x08 \x01(\x0e\x32\x15.flare.runtime.Memory\x12%\n\x06sizing\x18\t \x01(\x0b\x32\x15.flare.runtime.SizingB\x07\n\x05_fileB\n\n\x08_address\"X\n\x06Sizing\x12)\n\x08\x63\x61pacity\x18\x01 \x01(\x0e\x32\x17.flare.runtime.Capacity\x12\x12\n\nunit_bytes\x18\x02 \x01(\x04\x12\x0f\n\x07maximum\x18\x03 \x01(\x04\"L\n\x06Module\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x66ile\x18\x02 \x01(\t\x12&\n\x07globals\x18\x03 \x03(\x0b\x32\x15.flare.runtime.Global\"f\n\x06Global\x12\x0e\n\x06symbol\x18\x01 \x01(\t\x12\x0c\n\x04\x66ile\x18\x02 \x01(\t\x12\x0e\n\x06nbytes\x18\x03 \x01(\x04\x12.\n\x0brelocations\x18\x04 \x03(\x0b\x32\x19.flare.runtime.Relocation\"y\n\x08\x46unction\x12\x0e\n\x06module\x18\x01 \x01(\t\x12\x0e\n\x06symbol\x18\x02 \x01(\t\x12\x17\n\x0fparameter_sizes\x18\x03 \x03(\x04\x12\x34\n\nattributes\x18\x04 \x03(\x0b\x32 .flare.runtime.FunctionAttribute\".\n\x11\x46unctionAttribute\x12\n\n\x02id\x18\x01 \x01(\x05\x12\r\n\x05value\x18\x02 \x01(\x05\"/\n\tReference\x12\x12\n\nallocation\x18\x01 \x01(\t\x12\x0e\n\x06offset\x18\x02 \x01(\x04\"s\n\nRelocation\x12\x0e\n\x06offset\x18\x01 \x01(\x04\x12(\n\x06target\x18\x02 \x01(\x0b\x32\x18.flare.runtime.Reference\x12+\n\x04kind\x18\x03 \x01(\x0e\x32\x1d.flare.runtime.RelocationKind\"D\n\x04\x42lob\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12.\n\x0brelocations\x18\x02 \x03(\x0b\x32\x19.flare.runtime.Relocation\";\n\tAttribute\x12\n\n\x02id\x18\x01 \x01(\x05\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.flare.runtime.Blob\"\x9d\x01\n\x06Kernel\x12\x10\n\x08\x66unction\x18\x01 \x01(\x04\x12\x0c\n\x04grid\x18\x02 \x03(\r\x12\r\n\x05\x62lock\x18\x03 \x03(\r\x12\x0e\n\x06shared\x18\x04 \x01(\r\x12&\n\targuments\x18\x05 \x03(\x0b\x32\x13.flare.runtime.Blob\x12,\n\nattributes\x18\x06 \x03(\x0b\x32\x18.flare.runtime.Attribute\"\x8a\x01\n\x06Memset\x12-\n\x0b\x64\x65stination\x18\x01 \x01(\x0b\x32\x18.flare.runtime.Reference\x12\r\n\x05pitch\x18\x02 \x01(\x04\x12\r\n\x05value\x18\x03 \x01(\r\x12\x14\n\x0c\x65lement_size\x18\x04 \x01(\r\x12\r\n\x05width\x18\x05 \x01(\x04\x12\x0e\n\x06height\x18\x06 \x01(\x04\"o\n\x04\x43opy\x12(\n\x06source\x18\x01 \x01(\x0b\x32\x18.flare.runtime.Reference\x12-\n\x0b\x64\x65stination\x18\x02 \x01(\x0b\x32\x18.flare.runtime.Reference\x12\x0e\n\x06nbytes\x18\x03 \x01(\x04\"\x07\n\x05\x45mpty\"\x1a\n\tEventWait\x12\r\n\x05\x65vent\x18\x01 \x01(\t\"\xe1\x01\n\x04Node\x12\'\n\x06kernel\x18\x01 \x01(\x0b\x32\x15.flare.runtime.KernelH\x00\x12\'\n\x06memset\x18\x02 \x01(\x0b\x32\x15.flare.runtime.MemsetH\x00\x12#\n\x04\x63opy\x18\x03 \x01(\x0b\x32\x13.flare.runtime.CopyH\x00\x12%\n\x05\x65mpty\x18\x04 \x01(\x0b\x32\x14.flare.runtime.EmptyH\x00\x12.\n\nevent_wait\x18\x05 \x01(\x0b\x32\x18.flare.runtime.EventWaitH\x00\x42\x0b\n\toperation\"9\n\x04\x45\x64ge\x12\x0e\n\x06source\x18\x01 \x01(\x04\x12\x13\n\x0b\x64\x65stination\x18\x02 \x01(\x04\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"\xfc\x01\n\tExecution\x12\x13\n\x0b\x61\x62i_version\x18\x01 \x01(\r\x12\x12\n\nmax_active\x18\x02 \x01(\x04\x12\x19\n\x11input_frame_bytes\x18\x03 \x01(\x04\x12&\n\x07\x62uffers\x18\x04 \x03(\x0b\x32\x15.flare.runtime.Buffer\x12-\n\x06graphs\x18\x05 \x03(\x0b\x32\x1d.flare.runtime.ExecutionGraph\x12\x31\n\rtuning_graphs\x18\x06 \x03(\x0b\x32\x1a.flare.runtime.TuningGraph\x12!\n\x19\x63ompletion_token_capacity\x18\x07 \x01(\r\"C\n\x0bTuningGraph\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07outputs\x18\x02 \x03(\t\x12\x15\n\rstatus_buffer\x18\x03 \x01(\t\"\x82\x01\n\x06\x42uffer\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06nbytes\x18\x02 \x01(\x04\x12\r\n\x05\x64type\x18\x03 \x01(\t\x12\r\n\x05shape\x18\x04 \x03(\x04\x12\x0f\n\x07strides\x18\x05 \x03(\x04\x12+\n\treference\x18\x06 \x01(\x0b\x32\x18.flare.runtime.Reference\"\x95\x04\n\x0e\x45xecutionGraph\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04kind\x18\x02 \x01(\t\x12\x15\n\rsampling_mode\x18\x03 \x01(\t\x12\x16\n\x0e\x62\x61tch_capacity\x18\x04 \x01(\x04\x12\x16\n\x0etoken_capacity\x18\x05 \x01(\x04\x12\x33\n\x0einput_bindings\x18\x06 \x03(\x0b\x32\x1b.flare.runtime.InputBinding\x12\x19\n\x0c\x64raft_tokens\x18\x07 \x01(\rH\x00\x88\x01\x01\x12\x1a\n\rverify_tokens\x18\x08 \x01(\rH\x01\x88\x01\x01\x12\x30\n\x0c\x64raft_source\x18\t \x01(\x0e\x32\x1a.flare.runtime.DraftSource\x12 \n\x13speculation_enabled\x18\n \x01(\x08H\x02\x88\x01\x01\x12\x1d\n\x15\x65stimated_duration_ns\x18\x0b \x01(\x04\x12\x18\n\x10prefill_capacity\x18\x0c \x01(\x04\x12\x1d\n\x15packed_token_capacity\x18\r \x01(\x04\x12\x18\n\x10\x63onfidence_width\x18\x0e \x01(\r\x12\x33\n\x0etuned_bindings\x18\x0f \x03(\x0b\x32\x1b.flare.runtime.TunedBindingB\x0f\n\r_draft_tokensB\x10\n\x0e_verify_tokensB\x16\n\x14_speculation_enabled\"H\n\x0cInputBinding\x12\x13\n\x0b\x62uffer_name\x18\x01 \x01(\t\x12\x13\n\x0bhost_offset\x18\x02 \x01(\x04\x12\x0e\n\x06nbytes\x18\x03 \x01(\x04*\x8e\x01\n\x0bHistoryKind\x12\x17\n\x13HISTORY_UNSPECIFIED\x10\x00\x12\x13\n\x0fPAGED_KEY_VALUE\x10\x01\x12\x10\n\x0cPAGED_LATENT\x10\x02\x12\x12\n\x0eSLIDING_WINDOW\x10\x03\x12\x13\n\x0fRECURRENT_STATE\x10\x04\x12\x16\n\x12\x43ONTINUATION_STATE\x10\x05*\xfa\x06\n\x0bValueSource\x12\x15\n\x11VALUE_UNSPECIFIED\x10\x00\x12\x0c\n\x08\x43ONSTANT\x10\x01\x12\x10\n\x0cREQUEST_SLOT\x10\x02\x12\x16\n\x12REQUEST_GENERATION\x10\x03\x12\r\n\tLIVE_ROWS\x10\x04\x12\x10\n\x0cQUERY_LENGTH\x10\x05\x12\x12\n\x0eSAMPLE_ENABLED\x10\x06\x12\x10\n\x0cPROMPT_TOKEN\x10\x07\x12\x11\n\rPENDING_TOKEN\x10\x08\x12\x1c\n\x18PROMPT_AND_PENDING_TOKEN\x10\t\x12\x13\n\x0fPROMPT_POSITION\x10\n\x12\x17\n\x13IMAGE_FEATURE_INDEX\x10\x0b\x12\x16\n\x12IMAGE_LEFT_CONTEXT\x10\x0c\x12\x17\n\x13IMAGE_RIGHT_CONTEXT\x10\r\x12\x0e\n\nPAGE_TABLE\x10\x0e\x12\x13\n\x0fLIVE_STATE_ROWS\x10\x0f\x12\x12\n\x0eLIVE_STATE_ROW\x10\x10\x12\x14\n\x10PREFIX_STATE_ROW\x10\x11\x12\x16\n\x12SNAPSHOT_STATE_ROW\x10\x12\x12\x13\n\x0f\x43ONSUMED_TOKENS\x10\x13\x12\x1c\n\x18SNAPSHOT_CONSUMED_TOKENS\x10\x14\x12\x0f\n\x0bPREFILL_ROW\x10\x15\x12\x11\n\rSAMPLING_SEED\x10\x16\x12\x17\n\x13SAMPLING_PARAMETERS\x10\x17\x12\x12\n\x0eSAMPLING_TOP_K\x10\x18\x12\x12\n\x0eMAX_NEW_TOKENS\x10\x19\x12\x11\n\rEOS_TOKEN_IDS\x10\x1a\x12\r\n\tEOS_COUNT\x10\x1b\x12\x14\n\x10PROMPT_HISTOGRAM\x10\x1c\x12\x10\n\x0cGRAMMAR_MASK\x10\x1d\x12\x13\n\x0fGRAMMAR_ENABLED\x10\x1e\x12\x12\n\x0e\x43OPY_TOKEN_IDS\x10\x1f\x12\x14\n\x10\x43OPY_TOKEN_COUNT\x10 \x12\x18\n\x14\x43OPY_CONSUMED_TOKENS\x10!\x12\x13\n\x0f\x43OPY_GENERATION\x10\"\x12\x15\n\x11\x43OPY_GRAMMAR_MASK\x10#\x12\x16\n\x12\x43OPY_GRAMMAR_COUNT\x10$\x12\x10\n\x0cIMAGE_TENSOR\x10%\x12\x16\n\x12IMAGE_FEATURE_SLOT\x10&\x12\x17\n\x13PROMPT_TARGET_TOKEN\x10\'\x12\x1c\n\x18SPECULATIVE_PREFIX_LIMIT\x10(*|\n\rInputIndexing\x12\x18\n\x14INDEXING_UNSPECIFIED\x10\x00\x12\n\n\x06SCALAR\x10\x01\x12\x12\n\x0eSUBMISSION_ROW\x10\x02\x12\x0f\n\x0bREQUEST_ROW\x10\x03\x12\x10\n\x0cPACKED_TOKEN\x10\x04\x12\x0e\n\nACTIVE_ROW\x10\x05*T\n\x0fStateAccessKind\x12\x1c\n\x18STATE_ACCESS_UNSPECIFIED\x10\x00\x12\x08\n\x04READ\x10\x01\x12\t\n\x05WRITE\x10\x02\x12\x0e\n\nREAD_WRITE\x10\x03*\xf5\x01\n\x0eGraphOperation\x12\x1f\n\x1bGRAPH_OPERATION_UNSPECIFIED\x10\x00\x12\x08\n\x04INIT\x10\x01\x12\x0b\n\x07RESTORE\x10\x02\x12\x0c\n\x08SNAPSHOT\x10\x03\x12\x0b\n\x07PREFILL\x10\x04\x12\n\n\x06\x44\x45\x43ODE\x10\x05\x12\n\n\x06RESUME\x10\x06\x12\n\n\x06VISION\x10\x07\x12\x0f\n\x0bSPECULATIVE\x10\x08\x12\t\n\x05MIXED\x10\t\x12\x12\n\x0eSCORED_PREFILL\x10\x0c\"\x04\x08\n\x10\n\"\x04\x08\x0b\x10\x0b\"\x04\x08\r\x10\r*\x08PROPOSAL*\x0cVERIFICATION*\x12PACKED_SPECULATIVE*\xa1\x01\n\x0eToolParserKind\x12\x1b\n\x17TOOL_PARSER_UNSPECIFIED\x10\x00\x12\x0e\n\nQWEN_CODER\x10\x01\x12\x15\n\x11GLM_ARGUMENT_TAGS\x10\x02\x12\x08\n\x04\x44SML\x10\x03\x12\x08\n\x04XTML\x10\x04\x12\x0f\n\x0bMINIMAX_XML\x10\x05\x12\x0e\n\nGEMMA_CALL\x10\x06\x12\x08\n\x04\x41TEM\x10\x07\x12\x0c\n\x08TML_JSON\x10\x08*\xa6\x01\n\x13ReasoningParserKind\x12 \n\x1cREASONING_PARSER_UNSPECIFIED\x10\x00\x12\x0e\n\nTHINK_TAGS\x10\x01\x12\x0e\n\nXTML_THINK\x10\x02\x12\x0c\n\x08MM_THINK\x10\x03\x12\x13\n\x0f\x43HANNEL_THOUGHT\x10\x04\x12\x15\n\x11RECIPIENT_HEADERS\x10\x05\x12\x13\n\x0f\x43ONTENT_HEADERS\x10\x06*\xf4\x02\n\x14ImagePreparationKind\x12!\n\x1dIMAGE_PREPARATION_UNSPECIFIED\x10\x00\x12\x1a\n\x16QWEN_IMAGE_PREPARATION\x10\x01\x12\x1a\n\x16MIMO_IMAGE_PREPARATION\x10\x02\x12\x1e\n\x1a\x44\x45\x45PSEEK_IMAGE_PREPARATION\x10\x03\x12\x1d\n\x19MOONVIT_IMAGE_PREPARATION\x10\x04\x12\x1a\n\x16STEP_IMAGE_PREPARATION\x10\x05\x12\x1b\n\x17RADIO_IMAGE_PREPARATION\x10\x06\x12(\n$DEEPSEEK_ROW_MAJOR_IMAGE_PREPARATION\x10\x07\x12\x19\n\x15GLM_IMAGE_PREPARATION\x10\x08\x12!\n\x1dPATCH_MERGE_IMAGE_PREPARATION\x10\t\x12!\n\x1dPADDED_GRID_IMAGE_PREPARATION\x10\n*\xb6\x02\n\x0fImageLayoutKind\x12\x1c\n\x18IMAGE_LAYOUT_UNSPECIFIED\x10\x00\x12\x15\n\x11QWEN_IMAGE_LAYOUT\x10\x01\x12\x15\n\x11MIMO_IMAGE_LAYOUT\x10\x02\x12\x19\n\x15\x44\x45\x45PSEEK_IMAGE_LAYOUT\x10\x03\x12\x18\n\x14MOONVIT_IMAGE_LAYOUT\x10\x04\x12\x15\n\x11STEP_IMAGE_LAYOUT\x10\x05\x12\x16\n\x12RADIO_IMAGE_LAYOUT\x10\x06\x12#\n\x1f\x44\x45\x45PSEEK_ROW_MAJOR_IMAGE_LAYOUT\x10\x07\x12\x14\n\x10GLM_IMAGE_LAYOUT\x10\x08\x12\x1a\n\x16\x42RACKETED_IMAGE_LAYOUT\x10\t\x12\x1c\n\x18\x43LASS_TOKEN_IMAGE_LAYOUT\x10\n*P\n\rPromptEncoder\x12\x10\n\x0cPROMPT_JINJA\x10\x00\x12\x0f\n\x0b\x44\x45\x45PSEEK_V4\x10\x01\x12\x10\n\x0c\x44\x45\x45PSEEK_V41\x10\x02\x12\n\n\x06TML_V0\x10\x03*B\n\x10TemplateEncoding\x12\x11\n\rTEMPLATE_TEXT\x10\x00\x12\x1b\n\x17TEMPLATE_TOKEN_SEGMENTS\x10\x01*/\n\x06Memory\x12\n\n\x06\x44\x45VICE\x10\x00\x12\x0f\n\x0bMAPPED_HOST\x10\x01\x12\x08\n\x04PEER\x10\x02*(\n\x08\x43\x61pacity\x12\x0e\n\nSTATE_ROWS\x10\x00\x12\x0c\n\x08KV_PAGES\x10\x01*-\n\x0eRelocationKind\x12\x0b\n\x07POINTER\x10\x00\x12\x0e\n\nTENSOR_MAP\x10\x01*4\n\x0b\x44raftSource\x12\x0c\n\x08NO_DRAFT\x10\x00\x12\x07\n\x03MTP\x10\x01\x12\x0e\n\nCOPY_DRAFT\x10\x02\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -33,36 +33,36 @@ DESCRIPTOR._loaded_options = None _globals['_BUNDLEINDEX_RESOURCESENTRY']._loaded_options = None _globals['_BUNDLEINDEX_RESOURCESENTRY']._serialized_options = b'8\001' - _globals['_HISTORYKIND']._serialized_start=14696 - _globals['_HISTORYKIND']._serialized_end=14838 - _globals['_VALUESOURCE']._serialized_start=14841 - _globals['_VALUESOURCE']._serialized_end=15731 - _globals['_INPUTINDEXING']._serialized_start=15733 - _globals['_INPUTINDEXING']._serialized_end=15857 - _globals['_STATEACCESSKIND']._serialized_start=15859 - _globals['_STATEACCESSKIND']._serialized_end=15943 - _globals['_GRAPHOPERATION']._serialized_start=15946 - _globals['_GRAPHOPERATION']._serialized_end=16191 - _globals['_TOOLPARSERKIND']._serialized_start=16194 - _globals['_TOOLPARSERKIND']._serialized_end=16355 - _globals['_REASONINGPARSERKIND']._serialized_start=16358 - _globals['_REASONINGPARSERKIND']._serialized_end=16524 - _globals['_IMAGEPREPARATIONKIND']._serialized_start=16527 - _globals['_IMAGEPREPARATIONKIND']._serialized_end=16899 - _globals['_IMAGELAYOUTKIND']._serialized_start=16902 - _globals['_IMAGELAYOUTKIND']._serialized_end=17212 - _globals['_PROMPTENCODER']._serialized_start=17214 - _globals['_PROMPTENCODER']._serialized_end=17294 - _globals['_TEMPLATEENCODING']._serialized_start=17296 - _globals['_TEMPLATEENCODING']._serialized_end=17362 - _globals['_MEMORY']._serialized_start=17364 - _globals['_MEMORY']._serialized_end=17411 - _globals['_CAPACITY']._serialized_start=17413 - _globals['_CAPACITY']._serialized_end=17453 - _globals['_RELOCATIONKIND']._serialized_start=17455 - _globals['_RELOCATIONKIND']._serialized_end=17500 - _globals['_DRAFTSOURCE']._serialized_start=17502 - _globals['_DRAFTSOURCE']._serialized_end=17554 + _globals['_HISTORYKIND']._serialized_start=14725 + _globals['_HISTORYKIND']._serialized_end=14867 + _globals['_VALUESOURCE']._serialized_start=14870 + _globals['_VALUESOURCE']._serialized_end=15760 + _globals['_INPUTINDEXING']._serialized_start=15762 + _globals['_INPUTINDEXING']._serialized_end=15886 + _globals['_STATEACCESSKIND']._serialized_start=15888 + _globals['_STATEACCESSKIND']._serialized_end=15972 + _globals['_GRAPHOPERATION']._serialized_start=15975 + _globals['_GRAPHOPERATION']._serialized_end=16220 + _globals['_TOOLPARSERKIND']._serialized_start=16223 + _globals['_TOOLPARSERKIND']._serialized_end=16384 + _globals['_REASONINGPARSERKIND']._serialized_start=16387 + _globals['_REASONINGPARSERKIND']._serialized_end=16553 + _globals['_IMAGEPREPARATIONKIND']._serialized_start=16556 + _globals['_IMAGEPREPARATIONKIND']._serialized_end=16928 + _globals['_IMAGELAYOUTKIND']._serialized_start=16931 + _globals['_IMAGELAYOUTKIND']._serialized_end=17241 + _globals['_PROMPTENCODER']._serialized_start=17243 + _globals['_PROMPTENCODER']._serialized_end=17323 + _globals['_TEMPLATEENCODING']._serialized_start=17325 + _globals['_TEMPLATEENCODING']._serialized_end=17391 + _globals['_MEMORY']._serialized_start=17393 + _globals['_MEMORY']._serialized_end=17440 + _globals['_CAPACITY']._serialized_start=17442 + _globals['_CAPACITY']._serialized_end=17482 + _globals['_RELOCATIONKIND']._serialized_start=17484 + _globals['_RELOCATIONKIND']._serialized_end=17529 + _globals['_DRAFTSOURCE']._serialized_start=17531 + _globals['_DRAFTSOURCE']._serialized_end=17583 _globals['_PROTECTEDINDEX']._serialized_start=38 _globals['_PROTECTEDINDEX']._serialized_end=191 _globals['_SIGNEDGROUP']._serialized_start=193 @@ -82,149 +82,149 @@ _globals['_MODELINDEX']._serialized_start=988 _globals['_MODELINDEX']._serialized_end=1451 _globals['_REBUILDMANIFEST']._serialized_start=1454 - _globals['_REBUILDMANIFEST']._serialized_end=2307 - _globals['_REBUILDSPECULATION']._serialized_start=2310 - _globals['_REBUILDSPECULATION']._serialized_end=2820 - _globals['_REBUILDWEIGHTS']._serialized_start=2822 - _globals['_REBUILDWEIGHTS']._serialized_end=2923 - _globals['_REBUILDTENSOR']._serialized_start=2926 - _globals['_REBUILDTENSOR']._serialized_end=3088 - _globals['_SERVINGCONTRACT']._serialized_start=3091 - _globals['_SERVINGCONTRACT']._serialized_end=3934 - _globals['_MODELCONTRACT']._serialized_start=3937 - _globals['_MODELCONTRACT']._serialized_end=4204 - _globals['_VISIONCAPACITY']._serialized_start=4207 - _globals['_VISIONCAPACITY']._serialized_end=4351 - _globals['_CACHECONTRACT']._serialized_start=4354 - _globals['_CACHECONTRACT']._serialized_end=4488 - _globals['_HISTORYREGION']._serialized_start=4491 - _globals['_HISTORYREGION']._serialized_end=4641 - _globals['_PAGETABLEVIEW']._serialized_start=4643 - _globals['_PAGETABLEVIEW']._serialized_end=4695 - _globals['_VALUEBINDING']._serialized_start=4698 - _globals['_VALUEBINDING']._serialized_end=4875 - _globals['_STATEVIEW']._serialized_start=4877 - _globals['_STATEVIEW']._serialized_end=4952 - _globals['_STATEACCESS']._serialized_start=4954 - _globals['_STATEACCESS']._serialized_end=5055 - _globals['_COMPLETIONCURSOR']._serialized_start=5057 - _globals['_COMPLETIONCURSOR']._serialized_end=5136 - _globals['_TUNEDBINDING']._serialized_start=5138 - _globals['_TUNEDBINDING']._serialized_end=5190 - _globals['_GRAPHCONTRACT']._serialized_start=5193 - _globals['_GRAPHCONTRACT']._serialized_end=5571 - _globals['_TOOLPARSER']._serialized_start=5573 - _globals['_TOOLPARSER']._serialized_end=5680 - _globals['_REASONINGPARSER']._serialized_start=5683 - _globals['_REASONINGPARSER']._serialized_end=5832 - _globals['_PREFIXBOUNDARY']._serialized_start=5835 - _globals['_PREFIXBOUNDARY']._serialized_end=6037 - _globals['_IMAGESPAN']._serialized_start=6039 - _globals['_IMAGESPAN']._serialized_end=6160 - _globals['_IMAGETOKENLAYOUT']._serialized_start=6162 - _globals['_IMAGETOKENLAYOUT']._serialized_end=6243 - _globals['_FRONTENDCONTRACT']._serialized_start=6246 - _globals['_FRONTENDCONTRACT']._serialized_end=6744 - _globals['_CLUSTER']._serialized_start=6746 - _globals['_CLUSTER']._serialized_end=6795 - _globals['_DEFAULTSAMPLINGPARAMETERS']._serialized_start=6798 - _globals['_DEFAULTSAMPLINGPARAMETERS']._serialized_end=7194 - _globals['_SAMPLINGFLOATRANGE']._serialized_start=7196 - _globals['_SAMPLINGFLOATRANGE']._serialized_end=7268 - _globals['_SAMPLINGINTRANGE']._serialized_start=7270 - _globals['_SAMPLINGINTRANGE']._serialized_end=7340 - _globals['_SAMPLINGRANGES']._serialized_start=7343 - _globals['_SAMPLINGRANGES']._serialized_end=7749 - _globals['_FRONTENDSETTINGS']._serialized_start=7752 - _globals['_FRONTENDSETTINGS']._serialized_end=8077 - _globals['_TOKENIZERSETTINGS']._serialized_start=8080 - _globals['_TOKENIZERSETTINGS']._serialized_end=8428 - _globals['_TOKENLIST']._serialized_start=8430 - _globals['_TOKENLIST']._serialized_end=8457 - _globals['_IMAGESETTINGS']._serialized_start=8460 - _globals['_IMAGESETTINGS']._serialized_end=8968 - _globals['_PATCHMERGEIMAGESETTINGS']._serialized_start=8971 - _globals['_PATCHMERGEIMAGESETTINGS']._serialized_end=9156 - _globals['_DEEPSEEKROWMAJORIMAGESETTINGS']._serialized_start=9159 - _globals['_DEEPSEEKROWMAJORIMAGESETTINGS']._serialized_end=9294 - _globals['_QWENIMAGESETTINGS']._serialized_start=9297 - _globals['_QWENIMAGESETTINGS']._serialized_end=9427 - _globals['_MIMOIMAGESETTINGS']._serialized_start=9429 - _globals['_MIMOIMAGESETTINGS']._serialized_end=9527 - _globals['_ENGRAM']._serialized_start=9529 - _globals['_ENGRAM']._serialized_end=9653 - _globals['_ENGRAMTABLE']._serialized_start=9656 - _globals['_ENGRAMTABLE']._serialized_end=10168 - _globals['_ENGRAMTABLE_ENCODING']._serialized_start=10099 - _globals['_ENGRAMTABLE_ENCODING']._serialized_end=10168 - _globals['_ENGRAMTABLESHARD']._serialized_start=10171 - _globals['_ENGRAMTABLESHARD']._serialized_end=10311 - _globals['_ENGRAMHASH']._serialized_start=10314 - _globals['_ENGRAMHASH']._serialized_end=10509 - _globals['_ENGRAMHASH_METHOD']._serialized_start=10448 - _globals['_ENGRAMHASH_METHOD']._serialized_end=10509 - _globals['_ASSET']._serialized_start=10511 - _globals['_ASSET']._serialized_end=10586 - _globals['_BUNDLEINDEX']._serialized_start=10589 - _globals['_BUNDLEINDEX']._serialized_end=11184 - _globals['_BUNDLEINDEX_RESOURCESENTRY']._serialized_start=11115 - _globals['_BUNDLEINDEX_RESOURCESENTRY']._serialized_end=11184 - _globals['_ROCETRANSPORT']._serialized_start=11187 - _globals['_ROCETRANSPORT']._serialized_end=11435 - _globals['_GRAPHINDEX']._serialized_start=11437 - _globals['_GRAPHINDEX']._serialized_end=11555 - _globals['_PEERMEMORY']._serialized_start=11557 - _globals['_PEERMEMORY']._serialized_end=11634 - _globals['_CONTEXT']._serialized_start=11636 - _globals['_CONTEXT']._serialized_end=11713 - _globals['_LIMIT']._serialized_start=11715 - _globals['_LIMIT']._serialized_end=11749 - _globals['_RESERVATION']._serialized_start=11751 - _globals['_RESERVATION']._serialized_end=11797 - _globals['_SPAN']._serialized_start=11799 - _globals['_SPAN']._serialized_end=11855 - _globals['_ALLOCATION']._serialized_start=11858 - _globals['_ALLOCATION']._serialized_end=12148 - _globals['_SIZING']._serialized_start=12150 - _globals['_SIZING']._serialized_end=12238 - _globals['_MODULE']._serialized_start=12240 - _globals['_MODULE']._serialized_end=12316 - _globals['_GLOBAL']._serialized_start=12318 - _globals['_GLOBAL']._serialized_end=12420 - _globals['_FUNCTION']._serialized_start=12422 - _globals['_FUNCTION']._serialized_end=12543 - _globals['_FUNCTIONATTRIBUTE']._serialized_start=12545 - _globals['_FUNCTIONATTRIBUTE']._serialized_end=12591 - _globals['_REFERENCE']._serialized_start=12593 - _globals['_REFERENCE']._serialized_end=12640 - _globals['_RELOCATION']._serialized_start=12642 - _globals['_RELOCATION']._serialized_end=12757 - _globals['_BLOB']._serialized_start=12759 - _globals['_BLOB']._serialized_end=12827 - _globals['_ATTRIBUTE']._serialized_start=12829 - _globals['_ATTRIBUTE']._serialized_end=12888 - _globals['_KERNEL']._serialized_start=12891 - _globals['_KERNEL']._serialized_end=13048 - _globals['_MEMSET']._serialized_start=13051 - _globals['_MEMSET']._serialized_end=13189 - _globals['_COPY']._serialized_start=13191 - _globals['_COPY']._serialized_end=13302 - _globals['_EMPTY']._serialized_start=13304 - _globals['_EMPTY']._serialized_end=13311 - _globals['_EVENTWAIT']._serialized_start=13313 - _globals['_EVENTWAIT']._serialized_end=13339 - _globals['_NODE']._serialized_start=13342 - _globals['_NODE']._serialized_end=13567 - _globals['_EDGE']._serialized_start=13569 - _globals['_EDGE']._serialized_end=13626 - _globals['_EXECUTION']._serialized_start=13629 - _globals['_EXECUTION']._serialized_end=13881 - _globals['_TUNINGGRAPH']._serialized_start=13883 - _globals['_TUNINGGRAPH']._serialized_end=13950 - _globals['_BUFFER']._serialized_start=13953 - _globals['_BUFFER']._serialized_end=14083 - _globals['_EXECUTIONGRAPH']._serialized_start=14086 - _globals['_EXECUTIONGRAPH']._serialized_end=14619 - _globals['_INPUTBINDING']._serialized_start=14621 - _globals['_INPUTBINDING']._serialized_end=14693 + _globals['_REBUILDMANIFEST']._serialized_end=2336 + _globals['_REBUILDSPECULATION']._serialized_start=2339 + _globals['_REBUILDSPECULATION']._serialized_end=2849 + _globals['_REBUILDWEIGHTS']._serialized_start=2851 + _globals['_REBUILDWEIGHTS']._serialized_end=2952 + _globals['_REBUILDTENSOR']._serialized_start=2955 + _globals['_REBUILDTENSOR']._serialized_end=3117 + _globals['_SERVINGCONTRACT']._serialized_start=3120 + _globals['_SERVINGCONTRACT']._serialized_end=3963 + _globals['_MODELCONTRACT']._serialized_start=3966 + _globals['_MODELCONTRACT']._serialized_end=4233 + _globals['_VISIONCAPACITY']._serialized_start=4236 + _globals['_VISIONCAPACITY']._serialized_end=4380 + _globals['_CACHECONTRACT']._serialized_start=4383 + _globals['_CACHECONTRACT']._serialized_end=4517 + _globals['_HISTORYREGION']._serialized_start=4520 + _globals['_HISTORYREGION']._serialized_end=4670 + _globals['_PAGETABLEVIEW']._serialized_start=4672 + _globals['_PAGETABLEVIEW']._serialized_end=4724 + _globals['_VALUEBINDING']._serialized_start=4727 + _globals['_VALUEBINDING']._serialized_end=4904 + _globals['_STATEVIEW']._serialized_start=4906 + _globals['_STATEVIEW']._serialized_end=4981 + _globals['_STATEACCESS']._serialized_start=4983 + _globals['_STATEACCESS']._serialized_end=5084 + _globals['_COMPLETIONCURSOR']._serialized_start=5086 + _globals['_COMPLETIONCURSOR']._serialized_end=5165 + _globals['_TUNEDBINDING']._serialized_start=5167 + _globals['_TUNEDBINDING']._serialized_end=5219 + _globals['_GRAPHCONTRACT']._serialized_start=5222 + _globals['_GRAPHCONTRACT']._serialized_end=5600 + _globals['_TOOLPARSER']._serialized_start=5602 + _globals['_TOOLPARSER']._serialized_end=5709 + _globals['_REASONINGPARSER']._serialized_start=5712 + _globals['_REASONINGPARSER']._serialized_end=5861 + _globals['_PREFIXBOUNDARY']._serialized_start=5864 + _globals['_PREFIXBOUNDARY']._serialized_end=6066 + _globals['_IMAGESPAN']._serialized_start=6068 + _globals['_IMAGESPAN']._serialized_end=6189 + _globals['_IMAGETOKENLAYOUT']._serialized_start=6191 + _globals['_IMAGETOKENLAYOUT']._serialized_end=6272 + _globals['_FRONTENDCONTRACT']._serialized_start=6275 + _globals['_FRONTENDCONTRACT']._serialized_end=6773 + _globals['_CLUSTER']._serialized_start=6775 + _globals['_CLUSTER']._serialized_end=6824 + _globals['_DEFAULTSAMPLINGPARAMETERS']._serialized_start=6827 + _globals['_DEFAULTSAMPLINGPARAMETERS']._serialized_end=7223 + _globals['_SAMPLINGFLOATRANGE']._serialized_start=7225 + _globals['_SAMPLINGFLOATRANGE']._serialized_end=7297 + _globals['_SAMPLINGINTRANGE']._serialized_start=7299 + _globals['_SAMPLINGINTRANGE']._serialized_end=7369 + _globals['_SAMPLINGRANGES']._serialized_start=7372 + _globals['_SAMPLINGRANGES']._serialized_end=7778 + _globals['_FRONTENDSETTINGS']._serialized_start=7781 + _globals['_FRONTENDSETTINGS']._serialized_end=8106 + _globals['_TOKENIZERSETTINGS']._serialized_start=8109 + _globals['_TOKENIZERSETTINGS']._serialized_end=8457 + _globals['_TOKENLIST']._serialized_start=8459 + _globals['_TOKENLIST']._serialized_end=8486 + _globals['_IMAGESETTINGS']._serialized_start=8489 + _globals['_IMAGESETTINGS']._serialized_end=8997 + _globals['_PATCHMERGEIMAGESETTINGS']._serialized_start=9000 + _globals['_PATCHMERGEIMAGESETTINGS']._serialized_end=9185 + _globals['_DEEPSEEKROWMAJORIMAGESETTINGS']._serialized_start=9188 + _globals['_DEEPSEEKROWMAJORIMAGESETTINGS']._serialized_end=9323 + _globals['_QWENIMAGESETTINGS']._serialized_start=9326 + _globals['_QWENIMAGESETTINGS']._serialized_end=9456 + _globals['_MIMOIMAGESETTINGS']._serialized_start=9458 + _globals['_MIMOIMAGESETTINGS']._serialized_end=9556 + _globals['_ENGRAM']._serialized_start=9558 + _globals['_ENGRAM']._serialized_end=9682 + _globals['_ENGRAMTABLE']._serialized_start=9685 + _globals['_ENGRAMTABLE']._serialized_end=10197 + _globals['_ENGRAMTABLE_ENCODING']._serialized_start=10128 + _globals['_ENGRAMTABLE_ENCODING']._serialized_end=10197 + _globals['_ENGRAMTABLESHARD']._serialized_start=10200 + _globals['_ENGRAMTABLESHARD']._serialized_end=10340 + _globals['_ENGRAMHASH']._serialized_start=10343 + _globals['_ENGRAMHASH']._serialized_end=10538 + _globals['_ENGRAMHASH_METHOD']._serialized_start=10477 + _globals['_ENGRAMHASH_METHOD']._serialized_end=10538 + _globals['_ASSET']._serialized_start=10540 + _globals['_ASSET']._serialized_end=10615 + _globals['_BUNDLEINDEX']._serialized_start=10618 + _globals['_BUNDLEINDEX']._serialized_end=11213 + _globals['_BUNDLEINDEX_RESOURCESENTRY']._serialized_start=11144 + _globals['_BUNDLEINDEX_RESOURCESENTRY']._serialized_end=11213 + _globals['_ROCETRANSPORT']._serialized_start=11216 + _globals['_ROCETRANSPORT']._serialized_end=11464 + _globals['_GRAPHINDEX']._serialized_start=11466 + _globals['_GRAPHINDEX']._serialized_end=11584 + _globals['_PEERMEMORY']._serialized_start=11586 + _globals['_PEERMEMORY']._serialized_end=11663 + _globals['_CONTEXT']._serialized_start=11665 + _globals['_CONTEXT']._serialized_end=11742 + _globals['_LIMIT']._serialized_start=11744 + _globals['_LIMIT']._serialized_end=11778 + _globals['_RESERVATION']._serialized_start=11780 + _globals['_RESERVATION']._serialized_end=11826 + _globals['_SPAN']._serialized_start=11828 + _globals['_SPAN']._serialized_end=11884 + _globals['_ALLOCATION']._serialized_start=11887 + _globals['_ALLOCATION']._serialized_end=12177 + _globals['_SIZING']._serialized_start=12179 + _globals['_SIZING']._serialized_end=12267 + _globals['_MODULE']._serialized_start=12269 + _globals['_MODULE']._serialized_end=12345 + _globals['_GLOBAL']._serialized_start=12347 + _globals['_GLOBAL']._serialized_end=12449 + _globals['_FUNCTION']._serialized_start=12451 + _globals['_FUNCTION']._serialized_end=12572 + _globals['_FUNCTIONATTRIBUTE']._serialized_start=12574 + _globals['_FUNCTIONATTRIBUTE']._serialized_end=12620 + _globals['_REFERENCE']._serialized_start=12622 + _globals['_REFERENCE']._serialized_end=12669 + _globals['_RELOCATION']._serialized_start=12671 + _globals['_RELOCATION']._serialized_end=12786 + _globals['_BLOB']._serialized_start=12788 + _globals['_BLOB']._serialized_end=12856 + _globals['_ATTRIBUTE']._serialized_start=12858 + _globals['_ATTRIBUTE']._serialized_end=12917 + _globals['_KERNEL']._serialized_start=12920 + _globals['_KERNEL']._serialized_end=13077 + _globals['_MEMSET']._serialized_start=13080 + _globals['_MEMSET']._serialized_end=13218 + _globals['_COPY']._serialized_start=13220 + _globals['_COPY']._serialized_end=13331 + _globals['_EMPTY']._serialized_start=13333 + _globals['_EMPTY']._serialized_end=13340 + _globals['_EVENTWAIT']._serialized_start=13342 + _globals['_EVENTWAIT']._serialized_end=13368 + _globals['_NODE']._serialized_start=13371 + _globals['_NODE']._serialized_end=13596 + _globals['_EDGE']._serialized_start=13598 + _globals['_EDGE']._serialized_end=13655 + _globals['_EXECUTION']._serialized_start=13658 + _globals['_EXECUTION']._serialized_end=13910 + _globals['_TUNINGGRAPH']._serialized_start=13912 + _globals['_TUNINGGRAPH']._serialized_end=13979 + _globals['_BUFFER']._serialized_start=13982 + _globals['_BUFFER']._serialized_end=14112 + _globals['_EXECUTIONGRAPH']._serialized_start=14115 + _globals['_EXECUTIONGRAPH']._serialized_end=14648 + _globals['_INPUTBINDING']._serialized_start=14650 + _globals['_INPUTBINDING']._serialized_end=14722 # @@protoc_insertion_point(module_scope) diff --git a/ignition/recipes/declaration.py b/ignition/recipes/declaration.py index c86523f24..6ac81cb65 100644 --- a/ignition/recipes/declaration.py +++ b/ignition/recipes/declaration.py @@ -222,6 +222,10 @@ class Recipe: namespace. Export writes locally; it does not upload or select a signing identity. `b12x_environment` holds the b12x settings the recipe prepares with. b12x reads them only from its environment, so Ignition sets them for the preparing process. + `replicated_attention_bytes(world_size, prompt_logprobs, mtp_only)` gives the device + weights a row-sharded attention plan loads beside each rank's partition, from the + plan's prefill kinds, which `weights_bytes` adds to the estimate so capacity sizing + counts them. """ name: str @@ -260,6 +264,9 @@ class Recipe: draft_head_quantizations: tuple[str, ...] = () draft_body_quantizations: tuple[str, ...] = () markov_head_quantizations: tuple[str, ...] = () + # Device weights a row-sharded attention plan replicates per rank, from the checkpoint + # geometry, the tensor-parallel size, and the plan's scored and draft-only prefill kinds. + replicated_attention_bytes: Callable[..., int] | None = None def mixed_shapes(self, concurrency, chunk, speculation): from ignition.components.capacity import mixed_shapes, serving_modes @@ -317,8 +324,13 @@ def costs(self, world_size: int, speculation: Speculation | None, checkpoint: Ch return ranks[world_size] if method is None else method.ranks[world_size] def weights_bytes(self, checkpoint: Checkpoint, world_size: int, speculation: Speculation | None, - expert_scale_compression: str | None = None) -> int: - """Estimate one rank's device weights from the checkpoint's stored tensors and their expansion.""" + expert_scale_compression: str | None = None, *, row_shard_attention: bool = False, + prompt_logprobs: bool = False, mtp_only: bool = False) -> int: + """Estimate one rank's device weights from the checkpoint's stored tensors and their expansion. + + A row-sharded attention plan holds every replicated attention extent beside the partition + the estimate already charges, so `replicated_attention_bytes` adds those copies. + """ learned = speculation is not None device_bytes = (checkpoint.device_bytes if expert_scale_compression is None else checkpoint.scale_compressed_device_bytes[expert_scale_compression]) @@ -326,8 +338,11 @@ def weights_bytes(self, checkpoint: Checkpoint, world_size: int, speculation: Sp method = None if speculation is None else self.speculative_methods.get(speculation.method) if method is not None and method.draft_bytes is not None: draft_bytes = method.draft_bytes + replicated = (self.replicated_attention_bytes(world_size, prompt_logprobs=prompt_logprobs, + mtp_only=mtp_only) + if row_shard_attention and self.replicated_attention_bytes is not None else 0) return -(-(device_bytes + checkpoint.expansion_bytes - checkpoint.draft_bytes - + (draft_bytes if learned else 0)) // world_size) + + (draft_bytes if learned else 0)) // world_size) + replicated def live_rows(self, concurrency: int, speculation: Speculation | None) -> int: """Permanently backed null, live, and speculative execution rows.""" diff --git a/ignition/recipes/deepseek41/attention.py b/ignition/recipes/deepseek41/attention.py index 6ee651778..49c57e931 100644 --- a/ignition/recipes/deepseek41/attention.py +++ b/ignition/recipes/deepseek41/attention.py @@ -14,6 +14,7 @@ from b12x.preparation import PreparedCall from ignition.components import sm103_projection +from ignition.components.collectives import PreparedGatherShards from ignition.components.compressed_mla import PreparedCompressedCacheWriter, PreparedCompressedMLA from ignition.components.attention_output import PreparedGroupedOutputProjection from ignition.components.linear import QuantizedProjection, UnquantizedProjection @@ -121,7 +122,13 @@ class AttentionScratch: normalized_keys: torch.Tensor @classmethod - def allocate(cls, tokens, world, device, *, candidate_tokens=None): + def allocate(cls, tokens, world, device, *, candidate_tokens=None, shard_rows=None): + """Attention buffers for full rows, or for one rank's shard of a row-sharded layer. + + A shard keeps every head, so its query side is shard-sized and holds all 64 heads; the two + index-key planes stay full-row because the compressor still runs over every row, and the + gathered KV latent then lands in the caller's full-size window buffer. + """ candidate_tokens = tokens if candidate_tokens is None else candidate_tokens if type(candidate_tokens) is not int or not 0 < candidate_tokens <= tokens: raise ValueError("candidate row capacity must fit the attention token capacity") @@ -129,44 +136,77 @@ def zeros(shape, dtype=torch.bfloat16): return torch.zeros(shape, dtype=dtype, device=device) first, last = TensorParallelShard(0, world).padded_partition(8) heads = (last - first) * 8 - return cls(zeros((tokens, 1792)), - zeros((tokens, 1280)), zeros((tokens, heads, 512)), - zeros((tokens, 512)), zeros((tokens, heads, 512)), - zeros((tokens, 5120)), zeros((tokens, 512), torch.int32).fill_(-1), - zeros((candidate_tokens, 16384), torch.int32).fill_(-1), zeros(candidate_tokens, torch.int32), - zeros((tokens, 32, 128)), zeros((tokens, 32)), zeros((tokens, 128)), zeros((tokens, 128))) + if shard_rows is not None: + # Every rank's shard covers the whole head axis, which is the eight head blocks of the + # checkpoint layout rather than this rank's partition of them. + rows, candidates, heads = shard_rows, min(candidate_tokens, shard_rows), 8 * (last - first) * world + else: + rows, candidates = tokens, candidate_tokens + return cls(zeros((rows, 1792)), + zeros((rows, 1280)), zeros((rows, heads, 512)), + zeros((rows, 512)), zeros((rows, heads, 512)), + zeros((rows, 5120)), zeros((rows, 512), torch.int32).fill_(-1), + zeros((candidates, 16384), torch.int32).fill_(-1), zeros(candidates, torch.int32), + zeros((rows, 32, 128)), zeros((rows, 32)), zeros((tokens, 128)), zeros((tokens, 128))) + + def rows(self, count, tokens): + """This scratch narrowed to `count` query rows, keeping index keys at `tokens` rows.""" + if not 0 < count <= self.projected.shape[0]: + raise ValueError("attention scratch rows must fit the allocated query rows") + if not 0 < tokens <= self.index_keys.shape[0]: + raise ValueError("attention scratch tokens must fit the allocated index keys") + return type(self)(self.projected[:count], self.query_low[:count], self.query[:count], + self.window_values[:count], self.attended[:count], self.output[:count], self.selected[:count], + self.candidates[:count], self.candidate_counts[:count], self.index_query[:count], + self.index_weights[:count], self.index_keys[:tokens], self.normalized_keys[:tokens]) class PreparedCompressor: - def __init__(self, layer, weights, source, metadata, pool, slots, *, session, name, deferred=False): - self.pool, self.layer = pool, layer - self.weight = weights["norm"] + def __init__(self, layer, weights, source, metadata, pool, slots, *, session, name, deferred=False, + row_shard=None, runtime=None, coordinator=None): + self.pool, self.metadata, self.layer = pool, metadata, layer self.deferred = deferred + # The kernel call reads the norm directly; `self.arguments` is the declaration the preparation + # binds, and a forward does not read it. + self.weight = weights["norm"] ratio = pool.config.text["compress_ratios"][layer] self.ratio = ratio + # The kernel still compresses every row, so its operands stay full-row; only the two input + # projections run on the shard, and a gather returns their products to all rows. tokens, rows = source.shape[0], metadata.starts.numel() + total = tokens if row_shard is None else metadata.positions.numel() + self.total = total device = source.device + dtype = torch.float32 if ratio == 2 else torch.bfloat16 def zeros(shape, dtype): return torch.zeros(shape, device=device, dtype=dtype) - self.values = zeros((tokens, 512), torch.float32 if ratio == 2 else torch.bfloat16) - self.project_values = UnquantizedProjection(source, weights["values"], self.values, + self.values = zeros((total, 512), dtype) + self.local_values = self.values if row_shard is None else zeros((tokens, 512), dtype) + self.values_gather = None if row_shard is None else PreparedGatherShards( + self.local_values.view(torch.bfloat16), self.values.view(torch.bfloat16), runtime=runtime, + session=session, coordinator=coordinator, name=f"{name}.values_gather") + self.project_values = UnquantizedProjection(source, weights["values"], self.local_values, session=session, name=f"{name}.values").functional(session=session) - self.output = zeros((tokens, 512), torch.bfloat16) - self.emitted = zeros(tokens, torch.bool) - self.slots = zeros(tokens, torch.int64).fill_(-1) + self.output = zeros((total, 512), torch.bfloat16) + self.emitted = zeros(total, torch.bool) + self.slots = zeros(total, torch.int64).fill_(-1) self.state_ids = torch.arange(rows, dtype=torch.int64, device=device) self.arguments = dict(values=self.values, weight=weights["norm"], query_start_loc=metadata.query_start_loc, positions=metadata.starts, state_ids=self.state_ids, destination_slots=slots, live_counts=metadata.live_counts, out=self.output, emitted=self.emitted, emitted_slots=self.slots) if ratio == 2: self.gates = zeros(self.values.shape, self.values.dtype) - self.project_gates = UnquantizedProjection(source, weights["gates"], self.gates, + self.local_gates = self.gates if row_shard is None else zeros(self.local_values.shape, dtype) + self.gates_gather = None if row_shard is None else PreparedGatherShards( + self.local_gates.view(torch.bfloat16), self.gates.view(torch.bfloat16), runtime=runtime, + session=session, coordinator=coordinator, name=f"{name}.gates_gather") + self.project_gates = UnquantizedProjection(source, weights["gates"], self.local_gates, session=session, name=f"{name}.gates").functional(session=session) self.pending = {key: zeros((rows, 512) if key != "pending_position" else (rows,), torch.float32 if key != "pending_position" else torch.int64) for key in ("pending_values", "pending_gates", "pending_position")} self.arguments.update(gates=self.gates, **self.pending) - self.plan = mla_compress.plan(mla_compress.Caps(device=device, max_tokens=tokens, + self.plan = mla_compress.plan(mla_compress.Caps(device=device, max_tokens=total, max_requests=rows, max_states=rows, ratio=ratio)) declaration = self.arguments @@ -174,9 +214,9 @@ def zeros(shape, dtype): def prepare(state): arguments = {key: value if key in ("weight", "state_ids") else torch.zeros_like(value) for key, value in declaration.items()} - arguments["query_start_loc"].copy_(torch.arange(rows + 1, device=device) * tokens // rows) - arguments["live_counts"].copy_(torch.tensor([tokens, rows], dtype=torch.int32, device=device)) - arguments["destination_slots"].copy_(torch.arange(tokens, device=device)) + arguments["query_start_loc"].copy_(torch.arange(rows + 1, device=device) * total // rows) + arguments["live_counts"].copy_(torch.tensor([total, rows], dtype=torch.int32, device=device)) + arguments["destination_slots"].copy_(torch.arange(total, device=device)) if ratio == 2: arguments["pending_position"].fill_(-1) binding = state.bind(**arguments) @@ -185,20 +225,26 @@ def prepare(state): declare(session, (self.plan.request(name=name, prepare_call=prepare),)) def functional(self, *, session): register_finalizer(session, self.release_examples) + if self.values_gather is not None: + self.values_gather.functional(session=session) + if self.ratio == 2 and self.gates_gather is not None: + self.gates_gather.functional(session=session) return self def release_examples(self): self.values = self.output = self.emitted = self.slots = self.state_ids = self.arguments = None + self.local_values = None if self.ratio == 2: - self.gates = self.pending = None + self.gates = self.pending = self.local_gates = None def __call__(self, source, metadata, slots): from b12x.attention.mla_compress._impl import _run_ratio1, _run_ratio2 m = metadata - output = torch.zeros((source.shape[0], 512), dtype=torch.bfloat16, device=source.device) - emitted = torch.zeros(source.shape[0], dtype=torch.bool, device=source.device) - emitted_slots = torch.full((source.shape[0],), -1, dtype=torch.int64, device=source.device) + total = self.total + output = torch.zeros((total, 512), dtype=torch.bfloat16, device=source.device) + emitted = torch.zeros(total, dtype=torch.bool, device=source.device) + emitted_slots = torch.full((total,), -1, dtype=torch.int64, device=source.device) state_ids = torch.arange(m.starts.numel(), dtype=torch.int64, device=source.device) gates = None if self.ratio == 2: @@ -208,7 +254,11 @@ def __call__(self, source, metadata, slots): gather_rows_three(tuple(self.pool.region(f"layer{self.layer}.{key}") for key in pending), m.state_rows, tuple(pending.values())) gates = self.project_gates(source) + if self.gates_gather is not None: + gates = self.gates_gather(gates.view(torch.bfloat16)).view(torch.float32) values = self.project_values(source) + if self.values_gather is not None: + values = self.values_gather(values.view(torch.bfloat16)).view(torch.float32) _run_ratio2(self.plan.handle, values, gates, self.weight, m.query_start_loc, m.starts, state_ids, slots, m.live_counts, *pending.values(), output, emitted, emitted_slots) if not self.deferred: @@ -217,6 +267,8 @@ def __call__(self, source, metadata, slots): m.state_rows, m.lengths > 0) else: values = self.project_values(source) + if self.values_gather is not None: + values = self.values_gather(values.view(torch.bfloat16)) _run_ratio1(self.plan.handle, values, self.weight, m.query_start_loc, m.starts, state_ids, slots, m.live_counts, output, emitted, emitted_slots) return output, emitted_slots, (values, gates) @@ -226,7 +278,7 @@ def commit(self, counts, records, metadata): return m = metadata values, gates = records - last = (m.query_start_loc[:-1] + counts - 1).long().clamp(0, values.shape[0] - 1) + last = (m.query_start_loc[:-1] + counts - 1).long().clamp(0, self.total - 1) position = m.starts + counts - 1 pending = (counts > 0) & (position % 2 == 0) for key, source in (("pending_values", values), ("pending_gates", gates)): @@ -236,17 +288,23 @@ def commit(self, counts, records, metadata): class PreparedGlobalKV: - """Project and store every source row before decoder queries are packed.""" + """Project and store every source row before decoder queries are packed. + + A row-sharded layer computes the two compressor projections on its own rows and gathers them, so + the compression kernel, the index keys and the page writes stay full-row and hold every row. + """ def __init__(self, layer, weights, source, scratch, metadata, pool, rotary, *, - session, name, indices, deferred=False): + session, name, indices, deferred=False, row_shard=None, runtime=None, coordinator=None): self.rotary = rotary self.ratio = pool.config.text["compress_ratios"][layer] - if indices.metadata is not metadata or indices.position.shape != (source.shape[0],): + total = source.shape[0] if row_shard is None else metadata.positions.numel() + if indices.metadata is not metadata or indices.position.shape != (total,): raise ValueError("global KV indices must describe every source row") slots = indices.compressed(self.ratio).slots self.compressor = PreparedCompressor(layer, weights["compressor"], source, metadata, - pool, slots, session=session, name=f"{name}.compress", deferred=deferred) + pool, slots, session=session, name=f"{name}.compress", deferred=deferred, row_shard=row_shard, + runtime=runtime, coordinator=coordinator) self.index_key = UnquantizedProjection(self.compressor.output, weights["index_k"], scratch.index_keys, session=session, name=f"{name}.index_key").functional(session=session) self.index_norm = PreparedRMSNorm(scratch.index_keys, weights["index_norm"], scratch.normalized_keys, @@ -283,7 +341,8 @@ def _commit_window(Staged, Window, State, Starts, Lengths, STRIDE: tl.constexpr, class PreparedAttention: def __init__(self, layer, weights, source, scratch, metadata, pool, rotary, *, mode, world, session, name, workspace, replay_start=None, window_staging=None, - mla_compute_mode="auto", window_metadata=None, examples=None): + mla_compute_mode="auto", window_metadata=None, examples=None, row_shard=None, + window_target=None, runtime=None, coordinator=None): self.layer, self.weights, self.scratch, self.metadata = layer, weights, scratch, metadata self.pool, self.rotary, self.mode = pool, rotary, mode self.replay_start = replay_start @@ -292,7 +351,11 @@ def __init__(self, layer, weights, source, scratch, metadata, pool, rotary, *, m self.ratio = 0 if draft else config.text["compress_ratios"][layer] self.source = None if draft else config.source(layer) self.eps = config.text["rms_norm_eps"] + # `source` holds this rank's query rows: every row, or one shard when attention is sharded. tokens, device = source.shape[0], source.device + total = tokens if row_shard is None else metadata.positions.numel() + self.rows_index = None if row_shard is None else row_shard.rows_index + self.total = total rows = metadata.starts.numel() def zeros(shape, dtype): return torch.zeros(shape, dtype=dtype, device=device) @@ -315,35 +378,46 @@ def zeros(shape, dtype): ratios=(self.ratio,) if self.ratio else ()) elif (window_metadata.metadata is not metadata or window_metadata.mode != mode or window_metadata.window_page_size != self.window_page_size - or window_metadata.window_indices.shape != (tokens, window_width) + or window_metadata.window_indices.shape != (total, window_width) or window_metadata.replay_start is not replay_start or window_metadata.visible is not None or not isinstance(window_metadata, AttentionIndices)): raise ValueError("shared window indices must match the attention metadata and cache geometry") self.window_metadata = window_metadata + # Slots and the cache itself span every row; everything a query row reads is this rank's slice. self.window_slots = window_metadata.window_slots - self.window_indices = window_metadata.window_indices - self.window_lengths = window_metadata.window_lengths + def rank_rows(full): + return full if row_shard is None else full[:tokens] + self.window_indices = rank_rows(window_metadata.window_indices) + self.window_lengths = rank_rows(window_metadata.window_lengths) self.token_offsets = torch.arange(tokens, device=device) geometry = window_metadata.compressed(self.ratio) if self.ratio else None - self.lengths = zeros(tokens, torch.int32) if geometry is None else geometry.lengths + self.lengths = zeros(tokens, torch.int32) if geometry is None else rank_rows(geometry.lengths) self.active_width = zeros(1, torch.int32) if geometry is None else geometry.active_width - self.index_lengths = zeros(tokens, torch.int32) if geometry is None else geometry.index_lengths - self.page_table = metadata.token_page_table + self.index_lengths = zeros(tokens, torch.int32) if geometry is None else rank_rows(geometry.index_lengths) + self.page_table = rank_rows(metadata.token_page_table) self.projections = {} + # A shard owns every attention head and every output group, so it reads the replicated + # projections while a full-row layer keeps the tensor-parallel partition. + sharded = row_shard is not None + if sharded and not {"q_b_full", "output_full", "sink_full"} <= set(weights): + raise ValueError("row-sharded attention requires replicated attention weights") + q_b_key, output_key, sink_key = (("q_b_full", "output_full", "sink_full") if sharded + else ("q_b", "output", "sink")) use_b12x_mxfp8 = not sm103_projection.is_sm103(device) paired_mxfp8 = use_b12x_mxfp8 and all(isinstance(getattr(weights[key], "weight", None), mxfp8.MXFP8Rows) - for key in ("qkv_a", "q_b")) + for key in ("qkv_a", q_b_key)) self.norm_quant = (PreparedMLANormQuant(scratch.projected, (weights["q_norm"], weights["kv_norm"]), scratch.query_low, scratch.window_values, eps=self.eps, session=session, name=name + ".qkv_norm", release_dependents=tokens <= 8, examples=examples) if paired_mxfp8 and self.eps == 1e-20 and mla_norm_mxfp8.is_supported(device) else None) - for key, values, out in ( - ("qkv_a", source, scratch.projected), ("q_b", scratch.query_low, scratch.query.view(tokens, -1)), + for key, name_key, values, out in ( + ("qkv_a", "qkv_a", source, scratch.projected), + ("q_b", q_b_key, scratch.query_low, scratch.query.view(tokens, -1)), ): packed_mxfp8 = use_b12x_mxfp8 and isinstance( - getattr(weights[key], "weight", None), mxfp8.MXFP8Rows) - self.projections[key] = QuantizedProjection(values, weights[key], out, + getattr(weights[name_key], "weight", None), mxfp8.MXFP8Rows) + self.projections[key] = QuantizedProjection(values, weights[name_key], out, session=session, name=f"{name}.{key}", workspace=workspace, activation_mode="auto", mxfp8_min_amax=(1e-4 if packed_mxfp8 and (key != "q_b" or self.norm_quant is None) else None), @@ -355,9 +429,23 @@ def zeros(shape, dtype): eps=self.eps, session=session, name=name + ".q_norm") if self.norm_quant is None else None) self.window_norm = (PreparedRMSNorm(scratch.projected[:, 1280:], weights["kv_norm"], scratch.window_values, eps=self.eps, session=session, name=name + ".kv_norm") if self.norm_quant is None else None) + # The output projection sums all of this rank's groups in one kernel, so a shard needs no + # cross-rank reduction of the attention output. self.output_projection = PreparedGroupedOutputProjection(scratch.attended.view(tokens, -1, 4096), - weights["output"], scratch.output, session=session, name=name + ".output", workspace=workspace) - self.window_writer = PreparedCompressedCacheWriter(scratch.window_values, self.window_cache, + weights[output_key], scratch.output, session=session, name=name + ".output", workspace=workspace) + # Every rank's caches must hold every row, so a shard gathers the KV latent it computed with + # this rank's global slots before the write, leaving the cache bytes full-row. + window_values = scratch.window_values if row_shard is None else window_target + if row_shard is None: + if window_target is not None: + raise ValueError("a full-row attention writes its own KV latent") + self.window_gather = None + else: + if window_target is None or window_target.shape != (total, 512): + raise ValueError("row-sharded attention gathers its KV latent into a full-row buffer") + self.window_gather = PreparedGatherShards(scratch.window_values, window_target, + runtime=runtime, session=session, coordinator=coordinator, name=name + ".window_gather") + self.window_writer = PreparedCompressedCacheWriter(window_values, self.window_cache, self.window_slots, page_size=self.window_page_size, kind="swa", session=session, name=f"{name}.window_write", history=self.window if mode in ("verify", "draft") else None, @@ -367,7 +455,8 @@ def zeros(shape, dtype): self.selectors = [] if "compressor" in weights: self.global_kv = PreparedGlobalKV(layer, weights, source, scratch, metadata, pool, rotary, - session=session, name=name, indices=window_metadata, deferred=mode == "verify") + session=session, name=name, indices=window_metadata, deferred=mode == "verify", + row_shard=row_shard, runtime=runtime, coordinator=coordinator) if "index_q" in weights: self.index_query = QuantizedProjection(scratch.query_low, weights["index_q"], scratch.index_query.view(tokens, -1), session=session, name=f"{name}.index_query", @@ -397,7 +486,7 @@ def zeros(shape, dtype): page_table=self.page_table, indexed_page_size=128 // self.ratio) self.attention = PreparedCompressedMLA(scratch.query, self.window_cache, self.window_indices, - self.window_lengths, weights["sink"], scratch.attended, + self.window_lengths, weights[sink_key], scratch.attended, window_page_size=self.window_page_size, # Speculative rows use explicit indices into staged caches. Their # prepared row bound belongs to the decode contract, including @@ -438,6 +527,8 @@ def zeros(shape, dtype): self.query_norm.functional() self.window_norm.functional() self.output_projection.functional(session=session) + if self.window_gather is not None: + self.window_gather.functional(session=session) self.window_writer.functional(session=session, cache=None) self.attention.functional(session=session, window=None) if self.selectors: @@ -463,7 +554,15 @@ def _run(self, source, metadata, selection, *, indices=None, replay_start=None, m = metadata indices = (self.window_metadata(m, replay_start=replay_start) if self.owns_window_metadata else indices) - position = indices.position + # A shard computes the geometry for every row, because its caches hold every row, and reads + # this rank's slice of the row-indexed operands through the row numbers the graph has. + rows_index = self.rows_index + def rank_rows(full): + return full if rows_index is None else full.index_select(0, rows_index) + position = rank_rows(indices.position) + window_indices = rank_rows(indices.window_indices) + window_lengths = rank_rows(indices.window_lengths) + page_table = rank_rows(m.token_page_table) if window_cache is None: window_cache = self.window if self.mode == "decode" else torch.zeros( (m.starts.numel(), self.window_page_size * 528), dtype=torch.uint8, device=source.device) @@ -487,6 +586,8 @@ def _run(self, source, metadata, selection, *, indices=None, replay_start=None, query = self.projections["q_b"](query_low, encoded_bases=encoded_bases).view(self.query_shape) self.rotary.apply(query, position, compressed=bool(self.ratio)) self.rotary.apply(window_values, position, compressed=bool(self.ratio)) + if self.window_gather is not None: + window_values = self.window_gather(window_values) self.window_writer(window_values, indices.window_slots, history_rows=m.state_rows if self.mode in ("verify", "draft") else None, cache=window_cache) dependency = window_cache @@ -494,6 +595,8 @@ def _run(self, source, metadata, selection, *, indices=None, replay_start=None, records = None if self.ratio: geometry = indices.compressed(self.ratio) + lengths = rank_rows(geometry.lengths) + index_lengths = rank_rows(geometry.index_lengths) if self.global_kv is not None: records = self.global_kv(source, self.selectors[0], m, geometry) if self.selectors: @@ -506,7 +609,7 @@ def _run(self, source, metadata, selection, *, indices=None, replay_start=None, first = ordinal * INDEXER_QUERY_ROWS last = min(source.shape[0], first + INDEXER_QUERY_ROWS) result = selector(index_query[first:last], index_weights[first:last], - m.token_page_table[first:last], geometry.lengths[first:last], geometry.active_width, + page_table[first:last], lengths[first:last], geometry.active_width, candidates=None if self.layer <= self.candidate_source else candidates[first:last], counts=None if self.layer <= self.candidate_source else candidate_counts[first:last]) if self.layer == self.candidate_source: @@ -522,10 +625,10 @@ def _run(self, source, metadata, selection, *, indices=None, replay_start=None, if self.concurrent_query is not None: query = self.concurrent_query.finish(pending, dependency).view(self.query_shape) self.rotary.apply(query, position, compressed=bool(self.ratio)) - attended = self.attention(query, indices.window_indices, indices.window_lengths, + attended = self.attention(query, window_indices, window_lengths, indexed_indices=selected if self.ratio else None, - indexed_lengths=geometry.index_lengths if self.ratio else None, - page_table=m.token_page_table if self.ratio else None, window=window_cache) + indexed_lengths=index_lengths if self.ratio else None, + page_table=page_table if self.ratio else None, window=window_cache) self.rotary.apply(attended, position, compressed=bool(self.ratio), inverse=True) output = self.output_projection(attended.view(source.shape[0], -1, 4096)) record = window_cache, records diff --git a/ignition/recipes/deepseek41/bootstrap.py b/ignition/recipes/deepseek41/bootstrap.py index a95f30305..9ff539ced 100644 --- a/ignition/recipes/deepseek41/bootstrap.py +++ b/ignition/recipes/deepseek41/bootstrap.py @@ -66,10 +66,11 @@ def _prepare_draft(*args, draft_vocabulary_runtime, vocabulary_runtime=None, **k return PreparedDraft(*args, vocabulary_runtime=runtime, **kwargs) -def _forward_factories(mla_compute_mode, *, vision, +def _forward_factories(mla_compute_mode, *, vision, row_shard_attention=False, draft_vocabulary_runtime=None, examples=None): """Bind attention arithmetic and the proposal head's BF16 gather channel.""" - return (partial(PreparedDecoder, vision=vision, mla_compute_mode=mla_compute_mode, examples=examples), + return (partial(PreparedDecoder, vision=vision, mla_compute_mode=mla_compute_mode, examples=examples, + row_shard_attention=row_shard_attention), partial(_prepare_draft, draft_vocabulary_runtime=draft_vocabulary_runtime, mla_compute_mode=mla_compute_mode, examples=examples)) @@ -104,6 +105,9 @@ class PreparationConfig: # Prefill reductions keep each rank's contiguous row shard and gather the branch input that # every row's consumers read. row_shard_prefill: bool = False + # Row-sharded prefill also runs attention on the shard, with every head and output group + # replicated on every rank. + row_shard_attention: bool = False devices: tuple[int, ...] = (0, 1, 2, 3) # Device ordinals are host-local when each rank runs on its own Spark. hosts: tuple[str, ...] | None = None @@ -349,7 +353,9 @@ def prepare_rank(snapshot, config, *, rank, group, progress=None, core_storage=N expert_scale_compression=config.expert_scale_compression, draft_head_quantization=config.draft_head_quantization, markov_head_quantization=config.markov_head_quantization, - vocabulary_parallel_sampling=config.vocabulary_parallel_sampling) + vocabulary_parallel_sampling=config.vocabulary_parallel_sampling, + row_shard_attention=config.row_shard_attention, + prompt_logprobs=config.prompt_logprobs, mtp_only=config.mtp_only) release_unused() if progress is not None: progress("Target head: BF16 checkpoint weights with FP32 accumulation and logits") @@ -418,9 +424,11 @@ def prepare_rank(snapshot, config, *, rank, group, progress=None, core_storage=N candidate_tokens = min(candidate_tokens, 128) row_shard = shard if config.row_shard_prefill else None prefill_storage = Scratch.allocate(config.prefill_chunk_size, 1, shard.world_size, device, - candidate_tokens=candidate_tokens, row_shard=row_shard) + candidate_tokens=candidate_tokens, row_shard=row_shard, + attention_shard=config.row_shard_attention) scored_storage = (Scratch.allocate(config.prefill_chunk_size, 1, shard.world_size, device, - candidate_tokens=config.prefill_chunk_size, row_shard=row_shard) + candidate_tokens=config.prefill_chunk_size, row_shard=row_shard, + attention_shard=config.row_shard_attention) if config.prompt_logprobs else None) decoder_storage = Scratch.allocate(config.decode_tokens, verification_samples, shard.world_size, device) vision_state = vision_inputs = vision_prefill = vision_storage = feature_pool = None @@ -436,7 +444,7 @@ def prepare_rank(snapshot, config, *, rank, group, progress=None, core_storage=N feature_type=1, image_slots=config.image_concurrency, arena=feature_arena) vision_storage = {} decoder_factory, draft_factory = _forward_factories(config.mla_compute_mode, - vision=vision_state, + vision=vision_state, row_shard_attention=config.row_shard_attention, draft_vocabulary_runtime=draft_vocabulary_collective, examples=examples) buffers = dict(inputs.buffers(), **tables.buffers(), **commands.buffers(), **{ name: getattr(admission, name) for name in admission.__dataclass_fields__}, @@ -514,6 +522,10 @@ def add(name, forward, kind, rows, tokens, fields, *, enabled=None, packed_token if (config.row_shard_prefill and mode == "prefill" and decoder.row_shard is None and progress is not None): progress(f"{name}: row-sharded prefill falls back to full-row prefill for this capacity") + elif (config.row_shard_attention and mode == "prefill" and not decoder.attention_shard + and progress is not None): + progress(f"{name}: row-sharded attention falls back to full-row attention;" + " the replicated attention weights stay resident") forward = PreparedGeneration(decoder, batch, completion, frame, sequence, runtime=collective, coordinator=coordinator, session=session, name=name + ".generation", workspace=arena, context_factory=PreparedContext if kind == "scored_prefill" else _CONTEXT) diff --git a/ignition/recipes/deepseek41/config.py b/ignition/recipes/deepseek41/config.py index 081031452..a67a80979 100644 --- a/ignition/recipes/deepseek41/config.py +++ b/ignition/recipes/deepseek41/config.py @@ -7,6 +7,9 @@ import json from dataclasses import dataclass from pathlib import Path +from types import SimpleNamespace + +from ignition.components.parallel import TensorParallelShard @dataclass(frozen=True) @@ -48,6 +51,81 @@ def from_text(cls, text): return result +# The geometry the DeepSeek V4.1 Flash checkpoint must declare. Weight-layout and memory tests read +# these values instead of repeating them, so a geometry change fails the config check first. +TEXT_GEOMETRY = { + "hidden_size": 5120, "num_hidden_layers": 40, "num_attention_heads": 64, + "num_key_value_heads": 1, "head_dim": 512, "qk_rope_head_dim": 64, + "q_lora_rank": 1280, "o_lora_rank": 1024, "o_groups": 8, + "vocab_size": 129280, "moe_intermediate_size": 2304, "n_routed_experts": 384, + "n_shared_experts": 1, "num_experts_per_tok": 6, "scoring_func": "sqrtsoftplus", + "index_n_heads": 32, "index_head_dim": 128, "index_topk": 512, + "candidate_topk_blocks": 2048, "candidate_block_size": 8, "hc_mult": 4, + "sliding_window": 128, "engram_max_ngram_size": 4, "engram_n_heads": 8, + "engram_head_dim": 256, +} + + +def attention_weight_extents(text, shard): + """Checkpoint row extents for the attention projections, per rank and replicated. + + A tensor-parallel rank packs its share of the eight head blocks and eight output groups. A + row-sharded prefill rank instead owns every head and every output group, so `all_heads`, + `all_groups`, and `all_sink` name the wholesale extents it replicates beside the partitioned ones. + """ + first, last = shard.padded_partition(8) + return SimpleNamespace( + heads=(first * 8 * text["head_dim"], last * 8 * text["head_dim"]), + groups=(first * text["o_lora_rank"], last * text["o_lora_rank"]), + sink=(first * 8, last * 8), + all_heads=(0, text["num_attention_heads"] * text["head_dim"]), + all_groups=(0, text["o_groups"] * text["o_lora_rank"]), + all_sink=(0, text["num_attention_heads"])) + + +# The backbone schedule that decides which layers a prepared prefill can put on a row shard: +# two uncompressed layers, eighteen at ratio 2, and twenty at ratio 1 whose KV arrives from layer +# 20, the two Engram layers, and the three DSpark target layers. Planning sizes the replicated +# attention charge before a checkpoint is downloaded, so it reads them from this declaration, and +# a row-sharded attention load refuses a checkpoint whose decoder boundary or layer sets differ. +QUALIFIED_SCHEDULE = { + "compress_ratios": [0, 0] + [2] * 18 + [1] * 20, + "kv_source_layer_ids": [2, 8, 14, 20], + "engram_layer_ids": [1, 14], + "dspark_target_layer_ids": [37, 38, 39], +} + + +def replicated_layer_bytes(world_size): + """Device bytes one layer's replicated attention projections add per rank. + + The layer loads all `num_attention_heads` head rows of `wq_b` and all `o_groups` output groups + of `wo_a` and `wo_b` at MXFP8 packing, with one value byte per weight and two retained + scale representations of one byte per 32 values each. The FP32 attention sinks are also + replicated. The declared projection dimensions need no scale padding. These copies + coexist with the partition every rank keeps for decode. + """ + text, extent = TEXT_GEOMETRY, attention_weight_extents(TEXT_GEOMETRY, TensorParallelShard(0, world_size)) + group_width = text["num_attention_heads"] * text["head_dim"] // text["o_groups"] + values = ((extent.all_heads[1] - extent.all_heads[0]) * text["q_lora_rank"] + + (extent.all_groups[1] - extent.all_groups[0]) * (group_width + text["hidden_size"])) + sinks = extent.all_sink[1] - extent.all_sink[0] + return values + 2 * (-(-values // 32)) + sinks * 4 + + +def replicated_attention_bytes(world_size, *, prompt_logprobs=False, mtp_only=False): + """Device bytes per rank added by the complete replicated attention copies. + + Only the layers a prepared prefill can put on a row shard load the replicated projections. + A draft target layer shards only in a prefill graph that omits the draft, and `--mtp-only` + prepares no such graph, so its plan replicates three fewer layers than a scored plan does. + """ + config = ModelConfig({**TEXT_GEOMETRY, **QUALIFIED_SCHEDULE}, {}) + aux = QUALIFIED_SCHEDULE["dspark_target_layer_ids"] if mtp_only else () + return len(config.shardable_attention_layers(config.prefill_stop(prompt_logprobs), aux)) * \ + replicated_layer_bytes(world_size) + + @dataclass(frozen=True) class ModelConfig: text: dict @@ -64,17 +142,7 @@ def from_dict(cls, config): or config.get("architectures") != ["DeepseekV41ForCausalLM"]): raise ValueError("DeepSeek V4.1 requires DeepseekV41ForCausalLM") text, quant = config["text_config"], config["quantization_config"] - for name, value in { - "hidden_size": 5120, "num_hidden_layers": 40, "num_attention_heads": 64, - "num_key_value_heads": 1, "head_dim": 512, "qk_rope_head_dim": 64, - "q_lora_rank": 1280, "o_lora_rank": 1024, "o_groups": 8, - "vocab_size": 129280, "moe_intermediate_size": 2304, "n_routed_experts": 384, - "n_shared_experts": 1, "num_experts_per_tok": 6, "scoring_func": "sqrtsoftplus", - "index_n_heads": 32, "index_head_dim": 128, "index_topk": 512, - "candidate_topk_blocks": 2048, "candidate_block_size": 8, "hc_mult": 4, - "sliding_window": 128, "engram_max_ngram_size": 4, "engram_n_heads": 8, - "engram_head_dim": 256, - }.items(): + for name, value in TEXT_GEOMETRY.items(): if text.get(name) != value: raise ValueError(f"DeepSeek V4.1 Flash {name} requires {value}, got {text.get(name)}") if (quant.get("quant_method") != "fp8" or quant.get("weight_block_size") != [32, 32] @@ -134,6 +202,43 @@ def decoder_start(self): return next((source.layer for source in self.sources if source.ratio == 1 and any(ratio > 1 for ratio in ratios[:source.layer])), None) + def prefill_stop(self, scored): + """First backbone layer a prepared prefill leaves on full rows. + + Packed prefill stops at the decoder boundary, whose suffix a replay restores from full + rows. Scored prefill replays no suffix, so only the last layer stays full-row. A prefill of + 128 rows or fewer never shards, because a row shard needs at least 512 rows per rank on at + least two ranks, so the stop only names the layer where sharding would end. + """ + return (self.layers if scored or self.decoder_start is None else self.decoder_start) - 1 + + def shardable_attention_layers(self, stop, aux_layers=()): + """Backbone layers whose attention a row-sharded prefill can run on this rank's rows. + + A prefill leaves the layers from `stop` on full rows: the packed prefill stops at the + decoder boundary whose suffix a replay restores from full rows, and scored prefill, which + replays no suffix, stops below the last layer. An Engram layer reads and writes full rows, + and a layer in `aux_layers` writes a full-row draft feedback column, so neither can shard. + """ + engram, aux = set(self.text["engram_layer_ids"]), set(aux_layers) + return tuple(index for index in range(min(stop, self.layers)) + if index not in engram and index not in aux) + + def require_planned_attention_schedule(self): + """Refuse a checkpoint whose schedule the row-sharded attention charge does not describe. + + `replicated_attention_bytes` sizes the charge from `QUALIFIED_SCHEDULE` before a checkpoint + is read, and the loader replicates the layers this config names, so another decoder boundary, + Engram set, or draft target set would load weights the plan did not charge. A plan without + the option reads none of these layer sets, so it accepts the checkpoint as it stands. + """ + qualified = ModelConfig({**TEXT_GEOMETRY, **QUALIFIED_SCHEDULE}, {}) + if (self.decoder_start != qualified.decoder_start + or self.text["engram_layer_ids"] != qualified.text["engram_layer_ids"] + or tuple(self.text.get("dspark_target_layer_ids", ())) != tuple(qualified.text["dspark_target_layer_ids"])): + raise ValueError("row-sharded attention requires the qualified decoder boundary," + " Engram layers, and DSpark target layers") + def source(self, layer): if not self.text["compress_ratios"][layer]: return None diff --git a/ignition/recipes/deepseek41/decoder.py b/ignition/recipes/deepseek41/decoder.py index 9ececf991..996aa1007 100644 --- a/ignition/recipes/deepseek41/decoder.py +++ b/ignition/recipes/deepseek41/decoder.py @@ -54,6 +54,11 @@ class RowShard: attn_mix: torch.Tensor attention_hc: MHCScratch ffn_hc: MHCScratch + # Row-sharded attention: the shard's own attention scratch, the full-row KV latent the window + # writer stores, and the shard's router logits before they are gathered to every row. + attention: AttentionScratch | None = None + window_values: torch.Tensor | None = None + router_logits: torch.Tensor | None = None # Every graph's `rows_index` is a view of this table, one segment per shard row count (see # `_row_table_layout`). One compiled forward binds every rank, and binding compares storage # offsets, so a segment sits at the same offset on every rank and only its contents, this rank's @@ -61,7 +66,8 @@ class RowShard: rows_table: torch.Tensor | None = None @classmethod - def allocate(cls, tokens, world, rank, device): + def allocate(cls, tokens, world, rank, device, *, candidate_tokens=None, attention=False, + experts=384): """This rank's shard declarations for a decoder of `tokens` rows, or None when rows do not divide.""" if world < 2 or tokens % world or tokens // world % MHC_CALL_ROWS: return None @@ -80,7 +86,12 @@ def scratch(): first = layout[-1][1] return cls(world, rank, rows, start, rows_table[first:first + rows], zeros(rows, 5120), zeros(rows, 5120), zeros(rows, 4, 5120), zeros(rows, 4, dtype=torch.float32), - zeros(rows, 4, dtype=torch.float32), scratch(), scratch(), rows_table) + zeros(rows, 4, dtype=torch.float32), scratch(), scratch(), + (AttentionScratch.allocate(tokens, world, device, candidate_tokens=candidate_tokens, + shard_rows=rows) if attention else None), + zeros(tokens, 512) if attention else None, + zeros(rows, experts, dtype=torch.float32) if attention else None, + rows_table) def graph(self, tokens): """The shard a decoder of `tokens` rows uses, or None when its rows do not divide.""" @@ -97,7 +108,10 @@ def graph(self, tokens): rows_index=self.rows_table[first:first + rows], attention_output=self.attention_output[:rows], mlp_output=self.mlp_output[:rows], residual=self.residual[:rows], pre_mix=self.pre_mix[:rows], attn_mix=self.attn_mix[:rows], - attention_hc=self.attention_hc.rows(rows), ffn_hc=self.ffn_hc.rows(rows)) + attention_hc=self.attention_hc.rows(rows), ffn_hc=self.ffn_hc.rows(rows), + attention=None if self.attention is None else self.attention.rows(rows, tokens), + window_values=None if self.window_values is None else self.window_values[:tokens], + router_logits=None if self.router_logits is None else self.router_logits[:rows]) def _row_table_layout(rows): @@ -126,29 +140,131 @@ def row_shard_ready(scratch, shard, runtime): and PreparedChunkedGatherShards.fits(runtime, shard.residual, scratch.residual)) -def shard_schedule(layers, stop, sharded, aux_layers=()): +def attention_shard_ready(scratch, shard, runtime): + """True when a runtime serves the gathers row-sharded attention adds to the same scratch. + + The KV latent, the router logits and the carried selection are gathered whole. A selected row is + the widest shard-row operand a shard gathers, so proving it proves the compressor's two input + projections, which are gathered whole at the same row width and no wider. + """ + if shard is None or shard.attention is None: + return False + return (PreparedGatherShards.fits(runtime, shard.attention.window_values, shard.window_values) + and PreparedGatherShards.fits(runtime, shard.router_logits.view(torch.bfloat16), + scratch.router_logits.view(torch.bfloat16)) + and PreparedGatherShards.fits(runtime, shard.attention.selected.view(torch.bfloat16), + scratch.attention.selected.view(torch.bfloat16))) + + +def shard_schedule(config, stop, sharded, aux_layers=()): """Per-layer `(shard, defer, reseed)` for a prefill decoder whose shard exists. - Layers below `stop` that are not Engram layers keep this rank's rows. An Engram layer reads and - writes full rows, so it runs full-row, and the layers on either side of it change row dimension: - a layer that lands on the far side of such a change runs its own post, so the full-row residual - its consumer reads is complete, and the first sharded layer after a full-row run reseeds the - shard from the rows it owns. + The layers `config.shardable_attention_layers` names keep this rank's rows. An Engram layer + reads and writes full rows, so it runs full-row, and the layers on either side of it change row + dimension: a layer that lands on the far side of such a change runs its own post, so the + full-row residual its consumer reads is complete, and the first sharded layer after a full-row + run reseeds the shard from the rows it owns. """ + shardable = set(config.shardable_attention_layers(stop, aux_layers)) + engram = set(config.text["engram_layer_ids"]) def sharded_at(index): - return (sharded is not None and index < stop and index < len(layers) - and index not in aux_layers and layers[index].engram is None) + return sharded is not None and index in shardable schedule = [] - for index in range(len(layers)): + for index in range(config.layers): shard = sharded_at(index) # Deferred post hands this layer's four streams to the next layer's attention mHC, which # reads them only when both layers hold the same rows. - defer = (index + 1 < len(layers) and layers[index + 1].engram is None + defer = (index + 1 < config.layers and index + 1 not in engram and index + 1 not in aux_layers and sharded_at(index + 1) == shard) schedule.append((shard, defer, shard and index > 0 and not sharded_at(index - 1))) return schedule +@dataclass(frozen=True) +class SelectionMove: + """One attention layer's move of the selection state it reads into the rows it runs in. + + `selected` and `candidates` name a direction each, because the sparse indices and the candidate + shortlist are written by different layers and so carry different row representations. + """ + + source: int + selected: str | None = None + candidates: str | None = None + + +def selection_moves(config, schedule, *, attention_shard): + """Per-layer moves of the selection state an attention-sharded prefill decoder carries. + + Attention reads the sparse indices an index source wrote against that source's latent pages, so + its consumer must hold that source's selection in the rows it runs in. The decoder carries one + selection value from layer to layer, so the value holds the rows of the layer that last wrote + it: a layer that runs in the other representation moves it before its attention reads it. The + candidate shortlist has its own producer and its own consumers, so it is tracked apart. + """ + if not attention_shard: + return (None,) * len(schedule) + candidate = config.text["candidate_source_layer_id"] + rows = ["shard" if sharded else "full" for sharded, _, _ in schedule] + def direction(held, index): + if held == rows[index]: + return None + return "full_to_shard" if held == "full" else "shard_to_full" + moves, held, shortlist = [], rows[0], None + for index in range(len(rows)): + source = config.index_source(index) + selected = candidates = None + if source == index: + # An index source writes its own selection, so it reads no carried one. + held = rows[index] + elif source is not None: + selected, held = direction(held, index), rows[index] + if index == candidate: + shortlist = rows[index] + elif source == index and index > candidate: + candidates, shortlist = direction(shortlist, index), rows[index] + moves.append(None if selected is None and candidates is None + else SelectionMove(source if source is not None else candidate, selected, candidates)) + return tuple(moves) + + +class PreparedSelectionRefill: + """Move the carried selection state into the rows the next attention layer reads. + + A full-to-shard move copies the rows this rank owns out of the replicated full state, and the + candidate shortlist and its counts move with the same rows. A shard-to-full move gathers every + rank's selected rows through a BF16 view of the same words. Both run before the layer's selectors + or attention read the state. + """ + + def __init__(self, move, *, full, shard, rows_index, runtime, session, coordinator, name): + self.move, self.rows_index, self.gather = move, rows_index, None + if move.candidates == "shard_to_full": + # The shard gather moves whole 16-byte rows; a count row is four bytes. + raise ValueError(f"layer {move.source} reads index source {move.source}'s candidate" + " shortlist, which only a shard produced; its counts cannot be gathered" + " with the shard-row gather") + if move.selected == "shard_to_full": + self.gather = PreparedGatherShards(shard.selected.view(torch.bfloat16), + full.selected.view(torch.bfloat16), runtime=runtime, session=session, + coordinator=coordinator, name=name + ".selected") + + def functional(self, *, session): + if self.gather is not None: + self.gather.functional(session=session) + + def __call__(self, selection): + selected, candidates, counts = selection + if self.move.selected == "full_to_shard": + selected = selected.index_select(0, self.rows_index) + elif self.move.selected == "shard_to_full": + selected = self.gather(selected.view(torch.bfloat16)).view(torch.int32) + if self.move.candidates == "full_to_shard": + candidates = None if candidates is None else candidates.index_select(0, self.rows_index) + counts = None if counts is None else counts.index_select(0, self.rows_index) + return selected, candidates, counts + + @dataclass(frozen=True) class Scratch: """BF16 decoder activations and FP32 target vocabulary logits.""" @@ -192,10 +308,11 @@ def field_value(name): @classmethod def allocate(cls, tokens, samples, world, device, *, experts=384, top_k=6, - candidate_tokens=None, row_shard=None): + candidate_tokens=None, row_shard=None, attention_shard=False): def zeros(*shape, dtype=torch.bfloat16): return torch.zeros(shape, dtype=dtype, device=device) - shard = None if row_shard is None else RowShard.allocate(tokens, world, row_shard.rank, device) + shard = (None if row_shard is None else RowShard.allocate(tokens, world, row_shard.rank, device, + candidate_tokens=candidate_tokens, attention=attention_shard, experts=experts)) residual, engram_residual = zeros(tokens, 4, 5120), zeros(tokens, 4, 5120) pre_mix, attn_mix = zeros(tokens, 4, dtype=torch.float32), zeros(tokens, 4, dtype=torch.float32) attention_hc = MHCScratch.allocate(tokens, 5120, device) @@ -259,9 +376,11 @@ class PreparedLayer: def __init__(self, index, weights, tables, scratch, metadata, pool, rotary, *, shard, mode, runtime, coordinator, session, name, workspace, carry, defer_post, boundary=None, replay_start=None, window_staging=None, mla_compute_mode="auto", - window_metadata=None, examples=None, row_shard=None, reseed=False): + window_metadata=None, examples=None, row_shard=None, attention_shard=False, + selection_move=None, reseed=False): s, text = scratch, pool.config.text self.scratch, self.carry, self.row_shard = s, carry, row_shard + self.attention_shard = attention_shard # A full-row layer owns the rows since this shard last ran, so the shard is refilled here. self.reseed = reseed and row_shard is not None self.metadata = metadata @@ -290,15 +409,35 @@ def __init__(self, index, weights, tables, scratch, metadata, pool, rotary, *, s self.attention_hc = None if boundary is not None else PreparedMHC( incoming, _mhc_weights(weights, "attn", "attn_norm"), attention_scratch, session=session, name=name + ".attn_hc", workspace=workspace, pre_mix=pre_mix, pre_out=attn_mix, **options) - self.attention = PreparedAttention(index, weights.attention, s.attention_hc.normalized, s.attention, + # Row-sharded attention reads and writes only this rank's rows, computes every head and + # output group, and therefore needs neither the branch-input gather nor a reduction. + attention_source = attention_scratch.normalized if attention_shard else s.attention_hc.normalized + attention_buffer = row_shard.attention if attention_shard else s.attention + self.attention = PreparedAttention(index, weights.attention, attention_source, attention_buffer, metadata, pool, rotary, mode=mode, world=shard.world_size, session=session, name=name + ".attention", workspace=workspace, replay_start=replay_start, window_staging=window_staging, mla_compute_mode=mla_compute_mode, - window_metadata=window_metadata, examples=examples) - self.attention_gather = None if row_shard is None else PreparedGatherShards( + window_metadata=window_metadata, examples=examples, runtime=runtime, coordinator=coordinator, + row_shard=row_shard if attention_shard else None, + window_target=row_shard.window_values if attention_shard else None) + # A layer on one side of a row transition reads a selection the other side wrote, so its + # forward moves that state into the rows its attention reads before it runs. + self.selection_move = None + if selection_move is not None: + # The layer that crosses back to full rows runs full-row, so it holds no shard of its + # own and reads the carried selection through the decoder's shard instead. + carried = row_shard if row_shard is not None else s.row_shard + if carried is None or carried.attention is None: + raise ValueError("a selection move requires the row shard's attention scratch") + self.selection_move = PreparedSelectionRefill(selection_move, full=s.attention, + shard=carried.attention, rows_index=carried.rows_index, runtime=runtime, + session=session, coordinator=coordinator, name=name + ".selection") + self.attention_gather = None if row_shard is None or attention_shard else PreparedGatherShards( attention_scratch.normalized, s.attention_hc.normalized, runtime=runtime, coordinator=coordinator, session=session, name=name + ".attention_gather") - if row_shard is None: + if attention_shard: + self.attention_reduce = None + elif row_shard is None: self.attention_reduce = PreparedAllReduce(s.attention.output, s.attention_output, runtime=runtime, coordinator=coordinator, session=session, name=name + ".attention_reduce") else: @@ -310,8 +449,15 @@ def __init__(self, index, weights, tables, scratch, metadata, pool, rotary, *, s self.ffn_gather = None if row_shard is None else PreparedGatherShards( ffn_scratch.normalized, s.ffn_hc.normalized, runtime=runtime, coordinator=coordinator, session=session, name=name + ".ffn_gather") - self.router_projection = UnquantizedProjection(s.ffn_hc.normalized, weights.experts.router, s.router_logits, + # Top-k is row-local, so a shard projects the logits it owns and one gather hands the + # router every row. The routed experts keep all rows. + router_source = ffn_scratch.normalized if attention_shard else s.ffn_hc.normalized + router_logits = s.router_logits if not attention_shard else row_shard.router_logits + self.router_projection = UnquantizedProjection(router_source, weights.experts.router, router_logits, session=session, name=name + ".router") + self.router_gather = None if not attention_shard else PreparedGatherShards( + router_logits.view(torch.bfloat16), s.router_logits.view(torch.bfloat16), runtime=runtime, + coordinator=coordinator, session=session, name=name + ".router_gather") self.router = PreparedRouter(s.router_logits, s.router, session=session, name=name + ".topk", score_func=text["scoring_func"], renormalize=text["norm_topk_prob"], routed_scaling_factor=text["routed_scaling_factor"], correction_bias=weights.experts.correction, @@ -350,13 +496,18 @@ def __init__(self, index, weights, tables, scratch, metadata, pool, rotary, *, s if self.attention_hc is not None: self.attention_hc.functional() + if self.selection_move is not None: + self.selection_move.functional(session=session) if self.attention_gather is not None: self.attention_gather.functional(session=session) - self.attention_reduce.functional(session=session) + if self.attention_reduce is not None: + self.attention_reduce.functional(session=session) self.ffn_hc.functional() if self.ffn_gather is not None: self.ffn_gather.functional(session=session) self.router_projection.functional(session=session) + if self.router_gather is not None: + self.router_gather.functional(session=session) self.router.functional(session=session) self.experts.functional(session=session) if self.concurrent_shared is None: @@ -376,7 +527,8 @@ def release_examples(self): self.scratch = self.metadata = self.image_mask = None if self.row_shard is not None: self.row_shard = replace(self.row_shard, attention_output=None, mlp_output=None, - residual=None, pre_mix=None, attn_mix=None, attention_hc=None, ffn_hc=None) + residual=None, pre_mix=None, attn_mix=None, attention_hc=None, ffn_hc=None, + attention=None, window_values=None, router_logits=None) if self.concurrent_shared is not None: self.shared = None @@ -402,17 +554,24 @@ def _run(self, incoming, pre_mix, metadata, indices, selection, *, boundary=None if self.attention_gather is not None: normalized = self.attention_gather(normalized) attention_normalized = normalized + if self.selection_move is not None: + selection = self.selection_move(selection) attended, records, selection, observations = self.attention._run(normalized, metadata, selection, indices=indices, window_cache=window_cache) - attention_output = self.attention_reduce(attended) + # A shard's own rows are complete once its output projection has summed all groups. + attention_output = attended if self.attention_reduce is None else self.attention_reduce(attended) next_mix = torch.zeros_like(attn_mix) residual, post, comb, normalized = self.ffn_hc(attention_output, residual, post, comb, pre_mix=attn_mix, pre_out=next_mix) + router_source = normalized if self.ffn_gather is not None: normalized = self.ffn_gather(normalized) pending = None if self.concurrent_shared is None else self.concurrent_shared.start(normalized) image_mask = metadata.token_ids >= 129280 if self.has_image_mask else None - router_logits = self.router_projection(normalized) + router_logits = self.router_projection(router_source if self.attention_shard else normalized) + if self.router_gather is not None: + # The gather's surface moves BF16 packs of the FP32 logits it was prepared over. + router_logits = self.router_gather(router_logits.view(torch.bfloat16)).view(torch.float32) route = self.router(router_logits, image_mask=image_mask) routed = self.experts(normalized, route.ids, route.weights, router_logits=router_logits, selected_logits=route.selected_logits, image_mask=image_mask) @@ -438,7 +597,8 @@ def _run(self, incoming, pre_mix, metadata, indices, selection, *, boundary=None class PreparedDecoder: def __init__(self, weights, tables, scratch, metadata, pool, *, mode, runtime, vocabulary_runtime, coordinator, session, name, workspace, - vision=None, mla_compute_mode="auto", scoring=None, vocabulary_parallel=False, examples=None): + vision=None, mla_compute_mode="auto", scoring=None, vocabulary_parallel=False, examples=None, + row_shard_attention=False): if scratch.local_logits.dtype != torch.float32 or scratch.logits.dtype != torch.float32: raise ValueError("DeepSeek target vocabulary logits require FP32 storage") self.weights, self.scratch, self.metadata, self.pool = weights, scratch, metadata, pool @@ -458,14 +618,18 @@ def __init__(self, weights, tables, scratch, metadata, pool, *, mode, runtime, self.aux_layers = () if weights.draft is None else weights.config.draft.target_layers if self.packing is not None and any(layer <= decoder_start for layer in self.aux_layers): raise ValueError("bounded decoder feedback requires draft target layers after its global-KV boundary") - # Layers below `stop` run row-sharded; the layer at `stop - 1` materializes every row again - # so the decoder suffix keeps its full-row boundary and replay gather. - stop = (len(weights.layers) if self.packing is None else decoder_start) - 1 + # Layers below the prefill's stop run row-sharded; the layer at `stop - 1` materializes + # every row again so the decoder suffix keeps its full-row boundary and replay gather. + stop = weights.config.prefill_stop(self.packing is None) self.row_shard = scratch.row_shard if (self.row_shard is None or mode != "prefill" or stop < 1 or not row_shard_ready(scratch, self.row_shard, runtime)): self.row_shard = None - schedule = shard_schedule(weights.layers, stop, self.row_shard, self.aux_layers) + # Attention joins the shard only when the bootstrap asked and its gathers are served too. + self.attention_shard = (self.row_shard is not None and mode == "prefill" and row_shard_attention + and attention_shard_ready(scratch, self.row_shard, runtime)) + schedule = shard_schedule(weights.config, stop, self.row_shard, self.aux_layers) + moves = selection_moves(weights.config, schedule, attention_shard=self.attention_shard) self.auxiliary_shape = (self.output_metadata.token_ids.numel(), len(self.aux_layers) * 5120) self.auxiliary = torch.zeros(self.auxiliary_shape, dtype=torch.bfloat16, device=metadata.positions.device) self.safe_tokens = torch.zeros_like(metadata.token_ids) if self.vision is not None else metadata.token_ids @@ -514,7 +678,8 @@ def __init__(self, weights, tables, scratch, metadata, pool, *, mode, runtime, window_staging=None if self.window_commit is None else self.window_commit.staging[index], mla_compute_mode=mla_compute_mode, window_metadata=(self.window_metadata if layer_metadata is metadata else self.suffix_window_metadata), - examples=examples, row_shard=sharded, reseed=reseed)) + examples=examples, row_shard=sharded, attention_shard=self.attention_shard and sharded is not None, + selection_move=moves[index], reseed=reseed)) carry = defer self.scoring = None if scoring is None else scoring.prepare(scratch.embedding, weights.head, vocabulary=129280, session=session, coordinator=coordinator, name=name + ".prompt", @@ -551,7 +716,8 @@ def release_outputs(self): self.scratch = self.metadata = self.output_scratch = self.output_metadata = None if self.row_shard is not None: self.row_shard = replace(self.row_shard, attention_output=None, mlp_output=None, - residual=None, pre_mix=None, attn_mix=None, attention_hc=None, ffn_hc=None) + residual=None, pre_mix=None, attn_mix=None, attention_hc=None, ffn_hc=None, + attention=None, window_values=None, router_logits=None) self.auxiliary = self.safe_tokens = self.head_hidden = self.local_logits = None def __call__(self, metadata): @@ -571,7 +737,10 @@ def __call__(self, metadata): incoming = (residual.index_select(0, self.row_shard.rows_index),) pre_mix = pre_mix.index_select(0, self.row_shard.rows_index) indices = self.window_metadata(metadata) - selection = (torch.full((embedding.shape[0], 512), -1, dtype=torch.int32, device=embedding.device), None, None) + # The carried selection holds this rank's rows from the first layer on; every later change + # of row representation moves it in the layer that crosses over. + selection_rows = self.row_shard.rows if self.attention_shard else embedding.shape[0] + selection = (torch.full((selection_rows, 512), -1, dtype=torch.int32, device=embedding.device), None, None) staging = (None if self.staging_shape is None else torch.zeros( self.staging_shape, dtype=torch.uint8, device=embedding.device)) auxiliary = torch.zeros(self.auxiliary_shape, dtype=torch.bfloat16, device=embedding.device) diff --git a/ignition/recipes/deepseek41/recipe.py b/ignition/recipes/deepseek41/recipe.py index 917dad35a..27e4c69eb 100644 --- a/ignition/recipes/deepseek41/recipe.py +++ b/ignition/recipes/deepseek41/recipe.py @@ -8,6 +8,7 @@ from ignition.planning import HEADROOM_BYTES from ignition.recipes.declaration import Checkpoint, RankCosts, RankUsage, Recipe, SamplingDefaults, SamplingRange, Speculation +from .config import replicated_attention_bytes def prepare(snapshot, plan, *, compile_forwards, kernel_cache, progress, retain_graphs=False, @@ -30,6 +31,7 @@ def prepare(snapshot, plan, *, compile_forwards, kernel_cache, progress, retain_ compile_workers=plan.compile_workers, autotune_effort=plan.autotune_effort, tuning_replica=plan.tuning_replica, native_dense_activations=plan.native_dense_activations, joint_tuning=plan.joint_tuning, row_shard_prefill=plan.row_shard_prefill, + row_shard_attention=plan.row_shard_attention, lock_gpu_clocks=plan.lock_gpu_clocks and plan.autotune_effort != "off", image_concurrency=plan.image_concurrency, devices=tuple(range(plan.world_size)) if ranks is None else (0,) * plan.world_size, @@ -70,4 +72,5 @@ def usage(prepared): stream_overlap=True, ranks=MappingProxyType({world: RankCosts(3_043_840, 113_920, 8 << 30) for world in (3, 4, 6)}), plain_ranks=MappingProxyType({world: RankCosts(2_841_088, 113_920, 8 << 30) for world in (3, 4, 6)}), + replicated_attention_bytes=replicated_attention_bytes, ) diff --git a/ignition/recipes/deepseek41/weights.py b/ignition/recipes/deepseek41/weights.py index 7b7303542..8280a24d2 100644 --- a/ignition/recipes/deepseek41/weights.py +++ b/ignition/recipes/deepseek41/weights.py @@ -14,7 +14,7 @@ from ignition.components.markov_sampling import fp8_markov_head from ignition.preparation.persistent import persistent_copy, allocated_tensor_bytes -from .config import ModelConfig +from .config import ModelConfig, attention_weight_extents @dataclass(frozen=True) @@ -78,11 +78,14 @@ class ModelWeights: @classmethod def load(cls, manifest, device, *, shard, progress=None, dspark=False, confidence=False, expert_scale_compression=None, draft_head_quantization=None, markov_head_quantization=None, - vocabulary_parallel_sampling=False): + vocabulary_parallel_sampling=False, row_shard_attention=False, prompt_logprobs=False, + mtp_only=False): config = ModelConfig.load(manifest.metadata_path) before = allocated_tensor_bytes(device) with DeepSeekWeightLoader(manifest, device, shard=shard, progress=progress, dspark=dspark, - expert_scale_compression=expert_scale_compression) as loader: + expert_scale_compression=expert_scale_compression, + row_shard_attention=row_shard_attention, + prompt_logprobs=prompt_logprobs, mtp_only=mtp_only) as loader: embedding, norm = loader.load_tensors(("embed.weight", "norm.weight")) head = loader.load_padded_tensor("head.weight", rows=shard.padded_partition(129280, alignment=64)) layers = tuple(loader.load_layer(i) for i in range(config.layers)) @@ -107,16 +110,35 @@ def load(cls, manifest, device, *, shard, progress=None, dspark=False, confidenc class DeepSeekWeightLoader(BlockFP8WeightLoader): def __init__(self, manifest, device, *, shard, progress=None, prefixes=None, dspark=False, - expert_scale_compression=None): + expert_scale_compression=None, row_shard_attention=False, prompt_logprobs=False, + mtp_only=False): if expert_scale_compression not in (None, "csf"): raise ValueError("DeepSeek V4.1 expert scale compression must be None or 'csf'") self.dspark = dspark self.expert_scale_compression = expert_scale_compression + # A row-sharded prefill computes every head and every output group on each rank, so the + # layer also loads the replicated projections; the partitioned ones still serve decode, + # the suffix, and every layer no prepared prefill runs on the shard. + self.row_shard_attention = row_shard_attention + self.mtp_only = mtp_only super().__init__(manifest, device, progress=progress, prefixes=prefixes) self.config = ModelConfig.load(manifest.metadata_path) + if row_shard_attention: + self.config.require_planned_attention_schedule() self.shard = shard + self.shardable_layers = frozenset(self.config.shardable_attention_layers( + self.config.prefill_stop(prompt_logprobs), self._aux_layers())) self._expert_scale_scratch = {} + def _aux_layers(self): + """Draft target layers, which a prepared prefill leaves full-row when it holds the draft. + + A plan that loads the draft also prepares every prefill graph without it, and those graphs + put the target layers on the shard, so the loader keeps them unless the plan builds draft + graphs only. + """ + return self.config.draft.target_layers if self.dspark and self.mtp_only else () + def load_experts(self, prefixes, interval): text = self.config.text return self.load_mxfp4_experts(prefixes, hidden_size=text["hidden_size"], @@ -135,17 +157,20 @@ def load_layer(self, layer, *, draft=False): prefix = f"mtp.{layer}" if draft else f"layers.{layer}" text, shard = self.config.text, self.shard attn = prefix + ".attn" - first, last = shard.padded_partition(8) - heads = first * 8 * 512, last * 8 * 512 - groups = first * 1024, last * 1024 + extents = attention_weight_extents(text, shard) result = { "qkv_a": self.load_linear_group((attn + ".wq_a", attn + ".wkv")), - "q_b": self.load_linear_group((attn + ".wq_b",), row_ranges=(heads,), pad=True), - "output": self.load_attention_output(attn, groups, pad=True), + "q_b": self.load_linear_group((attn + ".wq_b",), row_ranges=(extents.heads,), pad=True), + "output": self.load_attention_output(attn, extents.groups, pad=True), "q_norm": self.load_tensor(attn + ".q_norm.weight"), "kv_norm": self.load_tensor(attn + ".kv_norm.weight"), - "sink": self.load_padded_tensor(attn + ".attn_sink", rows=(first * 8, last * 8)), + "sink": self.load_padded_tensor(attn + ".attn_sink", rows=extents.sink), } + if self.row_shard_attention and not draft and layer in self.shardable_layers: + result["q_b_full"] = self.load_linear_group((attn + ".wq_b",), pad=True, + row_ranges=(extents.all_heads,)) + result["output_full"] = self.load_attention_output(attn, extents.all_groups, pad=True) + result["sink_full"] = self.load_padded_tensor(attn + ".attn_sink", rows=extents.all_sink) if not draft and layer in text["kv_source_layer_ids"]: ratio = text["compress_ratios"][layer] result["compressor"] = { diff --git a/tests/cli/test_planning.py b/tests/cli/test_planning.py index 6e8b3e397..56e34a1a4 100644 --- a/tests/cli/test_planning.py +++ b/tests/cli/test_planning.py @@ -237,6 +237,119 @@ def test_row_shard_prefill_requires_deepseek_tp_and_survives_rebuild_serializati row_shard_prefill=True) +def test_row_shard_attention_requires_row_shard_prefill_and_survives_rebuild_serialization(monkeypatch): + """Both row-shard options travel through `plan` and `fit` beside an explicit MXFP8 scope.""" + from types import SimpleNamespace + from google.protobuf import json_format + from ignition.preparation import rebuild_manifest, runtime_core_pb2 as proto + + monkeypatch.setattr("ignition.preparation.source_identity.source_identity", lambda: "test-source") + model = replace(SOLO, name="deepseek41") + gpus = (gpu(0, 95), gpu(1, 95)) + settings = dict(selected=(0, 1), concurrency=4, reserve_bytes=0, speculation=DRAFTING, + online_mxfp8_scope="all", row_shard_prefill=True) + sharded = planning.plan(model, model.checkpoints[0], gpus, **settings) + attention = planning.plan(model, model.checkpoints[0], gpus, row_shard_attention=True, **settings) + assert not sharded.row_shard_attention + assert attention.row_shard_attention and attention.row_shard_prefill + assert attention.online_mxfp8_scope == "all" + assert attention.to_dict()["row_shard_attention"] is True + # Disabling attention sharding preserves the plan with only prefill row sharding enabled. + assert replace(attention, row_shard_attention=False) == sharded + fitted = planning.fit(model, model.checkpoints[0], gpus, concurrency=4, reserve_bytes=0, + speculation=DRAFTING, online_mxfp8_scope="all", + row_shard_prefill=True, row_shard_attention=True) + assert fitted.row_shard_attention and fitted.row_shard_prefill + assert fitted.online_mxfp8_scope == "all" + for plan in (sharded, attention, fitted): + manifest = rebuild_manifest.from_plan(SimpleNamespace(directory=None), plan, compiled_forwards=True) + encoded = json_format.ParseDict(manifest, proto.RebuildManifest()) + assert proto.RebuildManifest.FromString(encoded.SerializeToString()).row_shard_attention \ + == plan.row_shard_attention + # Attention cannot join a shard that prefill never asked for. + with pytest.raises(UsageError, match="row-shard-prefill"): + planning.plan(model, model.checkpoints[0], gpus, row_shard_attention=True, + **{name: value for name, value in settings.items() if name != "row_shard_prefill"}) + # Attention sharding inherits prefill's three-rank and SM103 eligibility gates. + three = recipe("deepseek41", 60, (3, 4), speculation=DRAFTING) + with pytest.raises(UsageError, match="TP3"): + planning.plan(three, three.checkpoints[0], (gpu(0, 95), gpu(1, 95), gpu(2, 95)), + selected=(0, 1, 2), concurrency=4, reserve_bytes=0, speculation=DRAFTING, + row_shard_prefill=True, row_shard_attention=True) + with pytest.raises(UsageError, match="SM103"): + planning.plan(model, model.checkpoints[0], (gpu(0, 95, (10, 3)), gpu(1, 95, (10, 3))), + selected=(0, 1), concurrency=4, reserve_bytes=0, speculation=DRAFTING, + row_shard_prefill=True, row_shard_attention=True) + + +def test_row_shard_attention_charges_only_the_layers_a_prefill_can_shard(): + """The plan counts the replicated attention layers each prefill mode shards, not all 40.""" + from ignition.recipes import resolve + from ignition.recipes.deepseek41.config import replicated_layer_bytes + + model = resolve("deepseek-v4.1-flash") + speculation = model.recipe.speculation + gpus = tuple(gpu(index, 95) for index in range(4)) + settings = dict(selected=(0, 1, 2, 3), concurrency=4, reserve_bytes=0, speculation=speculation, + row_shard_prefill=True) + + def plan(**modes): + return planning.plan(model.recipe, model.checkpoint, gpus, **{**settings, **modes}) + + plain = plan() + # A replicated layer holds 124.78 MB at TP4: the whole extent, not the part beyond the + # partition, because the rank keeps its own partition beside it. + per_layer = 124_780_800 + assert replicated_layer_bytes(4) == per_layer + # The packed prefill stops at the decoder boundary, so 17 layers replicate: 0, 2-13, 15-18. + # Scored prefill replays no suffix, so --prompt-logprobs reaches 37. A draft-only plan builds no + # prefill graph without the draft, and those graphs are what shard a draft target layer, so its + # scored prefill replicates 35. + draft_only = replace(speculation, mtp_only=True) + for layers, modes in ((17, {}), (37, {"prompt_logprobs": True}), + (17, {"speculation": draft_only}), + (35, {"speculation": draft_only, "prompt_logprobs": True})): + assert (plan(row_shard_attention=True, **modes).weights_bytes - plain.weights_bytes + == layers * per_layer), modes + # Pages are sized from what the weights leave, so the charge removes the pages they occupy. + assert plan(row_shard_attention=True).pages < plain.pages + + +def test_the_export_command_plans_the_replicated_attention_charge(tmp_path): + """The export path's own plan counts the replicated weights, not only a direct `planning.plan`.""" + from ignition.cli import preparation + from ignition.cli.config import Configuration + from ignition.recipes import resolve + from ignition.recipes.deepseek41.config import replicated_attention_bytes, replicated_layer_bytes + + path = tmp_path / "flare.toml" + path.write_text("") + configuration = Configuration.load(path) + model = resolve("deepseek-v4.1-flash") + # Devote 256 GiB per GPU so the plan is sized to the supply, not to this host's free memory. + devices = tuple(replace(gpu(index, 95), total_bytes=256 * GIB, free_bytes=255 * GIB) for index in range(4)) + settings = dict(gpus="0,1,2,3", speculation=model.recipe.speculation, row_shard_prefill=True, + concurrency=configuration.max_concurrency(model.checkpoint.alias, model.recipe.concurrency, None)) + host = preparation.Host(devices, None) + sharded = preparation.derive(configuration, model, host, **settings) + attention = preparation.derive(configuration, model, host, row_shard_attention=True, **settings) + assert sharded.row_shard_prefill and not sharded.row_shard_attention + assert attention.row_shard_prefill and attention.row_shard_attention + # The 17 layers a packed prefill can shard replicate 124.78 MB each at TP4, and the rank keeps + # its partition beside them, so the estimate grows by the whole replicated extent the loader + # holds. + assert replicated_attention_bytes(4) == 17 * replicated_layer_bytes(4) == 2_121_273_600 + assert attention.weights_bytes - sharded.weights_bytes == replicated_attention_bytes(4) + # Pages are sized from what the weights leave, so the charge takes back the pages they occupy, + # less the partial page each of the two divisions rounds down on its own. + occupied = replicated_attention_bytes(4) / sharded.costs.page_bytes + assert abs(sharded.pages - attention.pages - occupied) < 1 + # The export path passes both flags to `plan`, which rejects attention without prefill. + with pytest.raises(UsageError, match="row-shard-prefill"): + preparation.derive(configuration, model, host, **{**settings, "row_shard_prefill": False}, + row_shard_attention=True) + + def test_image_concurrency_defaults_and_explicit_capacity(): vision = replace(SOLO, max_images=4) settings = dict(concurrency=32, reserve_bytes=0, speculation=None, selected=None) diff --git a/tests/components/test_attention_output_rank_sum_cuda.py b/tests/components/test_attention_output_rank_sum_cuda.py new file mode 100644 index 000000000..e9368b03f --- /dev/null +++ b/tests/components/test_attention_output_rank_sum_cuda.py @@ -0,0 +1,224 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Compare row-sharded and tensor-parallel attention output projections against a BF16 rounding budget. + +A row-sharded prefill rank projects every output group with one prepared grouped projection, +accumulating the group sum in FP32 before rounding once. The TP4 reference projects two groups per +rank into BF16 partials and accumulates those partials in FP32 in rank order. The cases use shared +quantized operands to test the output difference against the partial- and final-rounding budget. +They do not exercise a multi-rank collective or establish full-model equivalence. +""" + +import os + +import pytest +import torch +from b12x.gemm import wo_projection +from b12x.gemm._shared.wo_mxfp8 import dequantize_mxfp8_rows_torch +from b12x.preparation import PreparationSession + +from ignition.components.attention_output import PreparedGroupedOutputProjection + +pytestmark = pytest.mark.skipif(os.environ.get("FLARE_RUN_CUDA_TESTS") != "1", + reason="requires an explicitly assigned CUDA device") + +# DeepSeek V4.1 Flash attention output geometry: eight output groups of 4096 columns over 64 heads of +# 512 dimensions, a 1024-wide group rank, and a 5120-wide hidden axis, over four ranks of two groups. +GROUPS, GROUP_WIDTH, RANK, HIDDEN, WORLD = 8, 4096, 1024, 5120, 4 +GROUPS_PER_RANK = GROUPS // WORLD +BLOCK = 32 +# Weight scale exponents in this band span 2^-12..2^-8, narrow enough that no block dominates the +# group sum while the four ranks still accumulate operands of different magnitudes. +SCALE_BAND = (115, 119) +REFERENCE_ROWS = 2048 +HEADER = (f"{'rows':>6} {'input':9} {'A!=B':5} {'ulpAB':5} {'|A-B|/rms':10} " + f"{'A/exact diff':12} {'ulp':5} {'rms':7} {'B/exact diff':12} {'ulp':5} {'rms':7} " + f"{'budget':6} intermediate") +REPORTED = [] + + +def _bf16_ulp(values): + """BF16 spacing above the magnitude of each value, in FP32.""" + magnitude = values.abs().bfloat16() + above = torch.nextafter(magnitude, torch.full_like(magnitude, float("inf"))) + return (above.float() - magnitude.float()).abs() + + +def _ulp_distance(result, reference, significant): + """Largest BF16 ulp distance between two values, measured at their larger magnitude. + + A rank-order sum cancels to zero where four partials offset exactly, and the spacing at zero + says nothing about the error, so each distance uses the wider of the two spacings. + """ + spacing = torch.maximum(_bf16_ulp(result), _bf16_ulp(reference)) + return float(((result.float() - reference.float()).abs() / spacing)[significant].max()) + + +def _weight_blocks(device, generator): + """Random MXFP8 operands whose block scales come from the narrow exponent band.""" + def block(shape): + values = (torch.randn(shape, device=device, generator=generator) * 16).to(torch.float8_e4m3fn) + scales = torch.randint(SCALE_BAND[0], SCALE_BAND[1] + 1, (shape[0] // BLOCK, shape[1] // BLOCK), + dtype=torch.uint8, device=device, generator=generator) + return values, scales.view(torch.float8_e8m0fnu) + return (*block((GROUPS * RANK, GROUP_WIDTH)), *block((HIDDEN, GROUPS * RANK))) + + +def _pack(wo_a, wo_a_scale, wo_b, wo_b_scale, groups): + return wo_projection.pack_weights(wo_a, wo_a_scale, wo_b, wo_b_scale, groups=groups, + group_width=GROUP_WIDTH, rank=RANK, hidden=HIDDEN, block_size=(BLOCK, BLOCK)) + + +def _rank_pack(wo_a, wo_a_scale, wo_b, wo_b_scale, rank): + """Pack one rank's contiguous output groups, the way the checkpoint partitions them.""" + blocks = slice(rank * GROUPS_PER_RANK * RANK // BLOCK, (rank + 1) * GROUPS_PER_RANK * RANK // BLOCK) + columns = slice(rank * GROUPS_PER_RANK * RANK, (rank + 1) * GROUPS_PER_RANK * RANK) + return _pack(wo_a[columns], wo_a_scale[blocks], wo_b[:, columns], wo_b_scale[:, blocks], + GROUPS_PER_RANK) + + +def _reduce(partials): + """Accumulate BF16 partials in FP32 in rank order and round the sum once. + + The PCIe one-shot all-reduce keeps a float32 accumulator per element and adds the payloads in + rank order starting at this rank before storing BF16 once, and the RoCE two-shot carries the + same arithmetic, so rank zero's order ``[0, 1, 2, 3]`` is the reduction this models. The + copy-engine ring that serves single-host prefill reductions folds its payloads one peer at a + time in the payload dtype instead, which rounds three times per element, so the reduction + modeled here is the most accurate tensor-parallel result the recipes produce. + """ + total = partials[0].float() + for partial in partials[1:]: + total = total + partial.float() + return total.bfloat16() + + +def _exact(source, plan, packed, device): + """Dequantized FP64 result of the shared quantized operands, rounded once to BF16. + + The source quantization, the BF16 intermediate, and its MXFP8 quantization are the ones the + prepared projection performs, so only the two GEMMs are evaluated in FP64. The projection keeps + the intermediate as dense [groups, rows, rank] storage and its quantizer reads that layout. + """ + rows = source.shape[0] + x_q = wo_projection.quantize_input(source, plan=plan) + a_deq = dequantize_mxfp8_rows_torch(packed.wo_a.values, packed.wo_a.scale_rows).double() + intermediate = torch.empty((GROUPS, rows, RANK), dtype=torch.bfloat16, device=device) + for start in range(0, rows, REFERENCE_ROWS): + block = slice(start, min(start + REFERENCE_ROWS, rows)) + x_deq = dequantize_mxfp8_rows_torch(x_q.values[block].contiguous(), + x_q.scale_rows[:, block].contiguous()) + intermediate[:, block] = torch.einsum("tdg,rdg->trg", x_deq.double(), a_deq + ).bfloat16().permute(2, 0, 1) + del x_deq + b_q = wo_projection.quantize_input_b(intermediate.permute(1, 2, 0), plan=plan) + del intermediate + values = dequantize_mxfp8_rows_torch(b_q.values, b_q.scale_rows) + del b_q + b_deq = dequantize_mxfp8_rows_torch(packed.wo_b.values, packed.wo_b.scale_rows).double() + exact = torch.empty((rows, HIDDEN), dtype=torch.bfloat16, device=device) + for start in range(0, rows, REFERENCE_ROWS): + block = slice(start, min(start + REFERENCE_ROWS, rows)) + exact[block] = (values[block].double() @ b_deq.T).bfloat16() + return exact + + +def _row(rows, label, out_a, out_b, partials, exact, identical): + """Print and return one case's measurements, its budget, and both RMS errors against exact. + + The ulp columns ignore elements within 1% of the output RMS, where a spacing carries no + information about the group sum. + """ + rms = float(out_a.float().square().mean().sqrt()) + difference = (out_a.float() - out_b.float()).abs() + significant = out_a.abs() > 0.01 * rms + # Every rounding in either path contributes at most half a BF16 ulp of its own result: one for + # each BF16 partial, one for each path's output. The reduction adds four BF16 partials in FP32, + # so its four additions contribute at most 4 * 2^-24 of their running magnitude, bounded here by + # the sum of the partial magnitudes. + budget = 0.5 * _bf16_ulp(out_a) + 0.5 * _bf16_ulp(out_b) + extent = torch.zeros_like(out_a, dtype=torch.float32) + for partial in partials: + budget = budget + 0.5 * _bf16_ulp(partial) + extent = extent + partial.float().abs() + budget = budget + (WORLD * 2.0 ** -24) * extent + del extent + fields = [f"{rows:6d} {label:9s} {float((out_a != out_b).float().mean()):5.3f}" + f" {_ulp_distance(out_a, out_b, significant):5.1f}" + f" {float(difference.max()) / rms:10.4f}"] + errors = [] + for out in (out_a, out_b): + errors.append(float((out.float() - exact.float()).square().mean().sqrt())) + fields.append(f" {float((out != exact).float().mean()):12.4f}" + f" {_ulp_distance(out, exact, significant):5.1f}" + f" {errors[-1] / float(exact.float().square().mean().sqrt()):7.2e}") + fields.append(f" {float((difference / budget).max()):6.2f} {identical}") + if not REPORTED: + REPORTED.append(True) + print(HEADER, flush=True) + print("".join(fields), flush=True) + return difference, budget, errors + + +# A row-sharded prefill projection sees at least 512 rows (ignition/recipes/deepseek41/decoder.py:72). +@pytest.mark.parametrize("rows", [128, 4096, 16384]) +@pytest.mark.parametrize("gained", [False, True], ids=["unit", "lognormal"]) +@torch.no_grad() +def test_row_shard_group_sum_matches_partitioned_reduce(rows, gained): + device = torch.device("cuda:0") + generator = torch.Generator(device=device).manual_seed(4127 + 2 * rows + int(gained)) + wo_a, wo_a_scale, wo_b, wo_b_scale = _weight_blocks(device, generator) + full = _pack(wo_a, wo_a_scale, wo_b, wo_b_scale, GROUPS) + partitioned = [_rank_pack(wo_a, wo_a_scale, wo_b, wo_b_scale, rank) for rank in range(WORLD)] + source = torch.randn((rows, GROUPS, GROUP_WIDTH), dtype=torch.bfloat16, device=device, + generator=generator) * .125 + if gained: + # A per-row lognormal gain gives tokens magnitudes that differ by orders of magnitude. + gain = torch.randn(rows, device=device, generator=generator).exp().bfloat16() + source = source * gain[:, None, None] + shards = [source[:, rank * GROUPS_PER_RANK:(rank + 1) * GROUPS_PER_RANK].contiguous() + for rank in range(WORLD)] + outputs = [torch.empty((rows, HIDDEN), dtype=torch.bfloat16, device=device) for _ in range(WORLD + 1)] + with PreparationSession(device=device, autotune=False, compile_workers=2) as session: + wholesale = PreparedGroupedOutputProjection(source, full, outputs[0], + session=session, name=f"rows{rows}.groups{GROUPS}", workspace=None) + ranked = [PreparedGroupedOutputProjection(shards[rank], partitioned[rank], outputs[rank + 1], + session=session, name=f"rows{rows}.groups{GROUPS_PER_RANK}.rank{rank}", workspace=None) + for rank in range(WORLD)] + row_shard = wholesale(source).clone() + partials = [operation(shards[rank]).clone() for rank, operation in enumerate(ranked)] + whole_binding = wo_projection.bind(wholesale.plan, scratch=wholesale.scratch, + source_tgd=source, weights=full, out=outputs[0]) + rank_bindings = [wo_projection.bind(operation.plan, scratch=operation.scratch, + source_tgd=shards[rank], weights=partitioned[rank], out=outputs[rank + 1]) + for rank, operation in enumerate(ranked)] + # The quantized intermediate of each rank's groups is the wholesale projection's slice. + identical = [] + for rank, binding in enumerate(rank_bindings): + columns = slice(rank * GROUPS_PER_RANK * RANK, (rank + 1) * GROUPS_PER_RANK * RANK) + blocks = slice(columns.start // BLOCK, columns.stop // BLOCK) + identical.append(torch.equal( + whole_binding.tmp_q.values.contiguous().view(torch.uint8)[:, columns], + binding.tmp_q.values.contiguous().view(torch.uint8)) and torch.equal( + whole_binding.tmp_q.scale_rows.reshape(-1, GROUPS * RANK // BLOCK)[:, blocks], + binding.tmp_q.scale_rows.reshape(-1, GROUPS_PER_RANK * RANK // BLOCK))) + identical = "identical" if all(identical) else "differs" + reduced = _reduce(partials) + exact = _exact(source, wholesale.plan, full, device) + difference, budget, errors = _row(rows, "lognormal" if gained else "unit", row_shard, + reduced, partials, exact, identical) + assert identical == "identical", f"rows={rows}: the quantized intermediate differs between paths" + broken = difference > budget + if bool(broken.any()): + worst = int((difference / budget).flatten().argmax()) + row, column = divmod(worst, HIDDEN) + pytest.fail(f"rows={rows} gained={gained}: {int(broken.sum())} of {broken.numel()} elements" + f" exceed the rounding budget, worst {float((difference / budget).max()):.3f}x at" + f" row {row} column {column}: |A-B|={float(difference[row, column]):.6f}" + f" budget={float(budget[row, column]):.6f}" + f" A={float(row_shard[row, column]):.6f}" + f" B={float(reduced[row, column]):.6f}" + f" partials={[float(partial[row, column]) for partial in partials]}") + assert errors[0] <= errors[1], (f"rows={rows}: the group sum is less accurate than the" + f" reduced partials ({errors[0]:.3e} > {errors[1]:.3e})") diff --git a/tests/recipes/test_deepseek41_dspark_cuda.py b/tests/recipes/test_deepseek41_dspark_cuda.py index 5011493ac..415bfefdd 100644 --- a/tests/recipes/test_deepseek41_dspark_cuda.py +++ b/tests/recipes/test_deepseek41_dspark_cuda.py @@ -133,6 +133,7 @@ def test_ratio_two_pending_state_selects_the_last_accepted_target_token(compiled operator.pool, operator.metadata, operator.layer, operator.ratio = pool, metadata, 2, 2 operator.values = torch.arange(12 * 512, dtype=torch.float32, device="cuda:0").view(12, 512) operator.gates = -operator.values + operator.total = operator.values.shape[0] counts = torch.tensor([2, 2, 0], dtype=torch.int32, device="cuda:0") for name in ("pending_values", "pending_gates", "pending_position"): pool.region("layer2." + name).fill_(42) diff --git a/tests/recipes/test_deepseek41_expert_overlap.py b/tests/recipes/test_deepseek41_expert_overlap.py index ac1ae42b7..d6e1c18bf 100644 --- a/tests/recipes/test_deepseek41_expert_overlap.py +++ b/tests/recipes/test_deepseek41_expert_overlap.py @@ -64,8 +64,10 @@ def test_layer_consumes_standalone_router_results_with_runtime_image_mask(): layer.has_image_mask = True # Built without __init__, so give the layer an unsharded layer's shard state. layer.row_shard, layer.reseed = None, False + layer.attention_shard = False layer.attention_gather = layer.ffn_gather = None layer.residual_gather = layer.mix_gather = None + layer.selection_move = layer.router_gather = None residual = torch.arange(12, dtype=torch.float32).reshape(3, 4) metadata = SimpleNamespace(token_ids=torch.tensor([7, 129280, 9])) selection = (torch.zeros(3, 1, dtype=torch.int32), None, None) diff --git a/tests/recipes/test_deepseek41_preparation_lifetimes.py b/tests/recipes/test_deepseek41_preparation_lifetimes.py new file mode 100644 index 000000000..9d382d05c --- /dev/null +++ b/tests/recipes/test_deepseek41_preparation_lifetimes.py @@ -0,0 +1,124 @@ +"""CPU checks of decoder operand rows and declaration-example ownership.""" + +import gc +from types import SimpleNamespace +import weakref + +import pytest +import torch + +from ignition.recipes.deepseek41 import attention, decoder +from tests.recipes.test_deepseek41_row_shard_attention import _attention_layer, _Recorder + + +@pytest.mark.parametrize("policy", ["prefill-only", "gather-refused", "attention-sharded"]) +def test_router_input_matches_attention_sharding(monkeypatch, policy): + scratch = decoder.Scratch.allocate(1024, 8, 2, "cpu", row_shard=SimpleNamespace(rank=1), + attention_shard=True) + shard = scratch.row_shard + layer, _ = _attention_layer(scratch) + monkeypatch.setattr(decoder.PreparedGatherShards, "fits", + staticmethod(lambda *args: policy != "gather-refused")) + layer.attention_shard = (policy != "prefill-only" + and decoder.attention_shard_ready(scratch, shard, None)) + if not layer.attention_shard: + layer.attention_gather = _Recorder(scratch.attention_hc.normalized) + layer.attention = _Recorder((scratch.attention.output, {}, None, {})) + layer.attention_reduce = _Recorder(shard.attention_output) + layer.router_projection = _Recorder(scratch.router_logits) + layer.router_gather = None + layer.route_gather = None + layer.router = _Recorder(scratch.router) + layer._run((shard.residual,), shard.pre_mix, + SimpleNamespace(token_ids=torch.zeros(1024, dtype=torch.int64)), None, + (scratch.attention.selected, None, None)) + source, = layer.router_projection.calls[0] + expected = shard.ffn_hc.normalized if layer.attention_shard else scratch.ffn_hc.normalized + assert source is expected + assert source.shape == (512 if layer.attention_shard else 1024, 5120) + assert layer.experts.calls[0][0] is scratch.ffn_hc.normalized + + +def _component(*args, **kwargs): + # GPU plan declarations are replaced, not the owning constructors or finalizers. + return SimpleNamespace(plan=object(), functional=lambda **options: None) + + +def _prepare_layer(monkeypatch, scratch, metadata, sharded): + for name in ("PreparedMHC", "PreparedAttention", "PreparedAllReduce", "PreparedReduceScatter", + "PreparedChunkedGatherShards", "PreparedGatherShards", "UnquantizedProjection", + "PreparedRoutedExperts", "PreparedMLP", "PreparedMHCPost", "PreparedRouter"): + monkeypatch.setattr(decoder, name, _component) + monkeypatch.setattr(decoder, "_mhc_weights", lambda *args: None) + text = dict(rms_norm_eps=1e-6, hc_eps=1e-6, hc_sinkhorn_iters=20, scoring_func="sigmoid", + norm_topk_prob=True, routed_scaling_factor=1., swiglu_limit=10.) + weights = SimpleNamespace(engram=None, attention=None, experts=SimpleNamespace( + vision_correction=None, router=None, correction=None, experts=None, shared=None)) + return decoder.PreparedLayer(0, weights, None, scratch, metadata, + SimpleNamespace(config=SimpleNamespace(text=text)), None, + shard=SimpleNamespace(world_size=2), mode="prefill", runtime=None, coordinator=None, + session=object(), name="layer", workspace=SimpleNamespace(overlap=False), carry=False, + defer_post=False, row_shard=scratch.row_shard if sharded else None, attention_shard=sharded) + + +@pytest.mark.parametrize("sharded", [False, True]) +def test_layer_and_decoder_release_attention_examples(monkeypatch, sharded): + scratch = decoder.Scratch.allocate(1024, 8, 2, "cpu", + row_shard=SimpleNamespace(rank=1) if sharded else None, attention_shard=sharded) + examples = [weakref.ref(scratch.ffn_hc.normalized)] + if sharded: + examples += [weakref.ref(scratch.row_shard.window_values), + weakref.ref(scratch.row_shard.attention.window_values), + weakref.ref(scratch.row_shard.router_logits)] + metadata = SimpleNamespace(token_ids=torch.zeros(1024, dtype=torch.int64)) + layer = _prepare_layer(monkeypatch, scratch, metadata, sharded) + prepared = object.__new__(decoder.PreparedDecoder) + prepared.row_shard = scratch.row_shard + prepared.scratch = prepared.output_scratch = scratch + prepared.layers = [layer] + layer.release_examples() + prepared.release_outputs() + del scratch + gc.collect() + assert all(example() is None for example in examples) + if sharded: + assert torch.equal(layer.row_shard.rows_index, torch.arange(512, 1024)) + assert layer.row_shard.rows_table is prepared.row_shard.rows_table + + +@pytest.mark.parametrize("sharded", [False, True]) +def test_attention_releases_window_examples(monkeypatch, sharded): + for name in ("QuantizedProjection", "PreparedRMSNorm", "PreparedGroupedOutputProjection", + "PreparedCompressedCacheWriter", "PreparedCompressedMLA", "PreparedGatherShards"): + monkeypatch.setattr(attention, name, _component) + monkeypatch.setattr(attention.sm103_projection, "is_sm103", lambda device: False) + tokens, rows = 1024, 512 if sharded else 1024 + scratch = attention.AttentionScratch.allocate(tokens, 2, "cpu", candidate_tokens=1, + shard_rows=rows if sharded else None) + target = torch.zeros(tokens, 512) if sharded else None + examples = [weakref.ref(scratch.window_values)] + if target is not None: + examples.append(weakref.ref(target)) + metadata = SimpleNamespace(positions=torch.arange(tokens), starts=torch.zeros(1, dtype=torch.int64), + token_page_table=torch.zeros(tokens, 1, dtype=torch.int32), max_query_tokens=tokens) + monkeypatch.setattr(attention, "AttentionIndices", lambda *args, **kwargs: SimpleNamespace( + window_slots=torch.zeros(tokens, dtype=torch.int64), + window_indices=torch.zeros(tokens, 128, dtype=torch.int32), + window_lengths=torch.zeros(tokens, dtype=torch.int32), functional=lambda: None)) + config = SimpleNamespace(text={"compress_ratios": [0], "rms_norm_eps": 1e-6, + "candidate_source_layer_id": 0}, source=lambda layer: None) + pool = SimpleNamespace(config=config, window=lambda *args, **kwargs: torch.zeros(1, 128 * 528, + dtype=torch.uint8)) + weights = dict.fromkeys(("qkv_a", "q_b", "q_b_full", "output", "output_full", "sink", "sink_full", + "q_norm", "kv_norm")) + shard = SimpleNamespace(rows_index=torch.arange(512, 1024), attention=scratch, + window_values=target) if sharded else None + prepared = attention.PreparedAttention(0, weights, torch.zeros(rows, 5120), scratch, metadata, + pool, None, mode="prefill", world=2, session=object(), name="attention", workspace=None, + row_shard=shard, window_target=target) + prepared.release_examples() + del scratch, target, shard + gc.collect() + assert all(example() is None for example in examples) + if sharded: + assert torch.equal(prepared.rows_index, torch.arange(512, 1024)) diff --git a/tests/recipes/test_deepseek41_row_shard.py b/tests/recipes/test_deepseek41_row_shard.py index 61dca28e8..b25071b68 100644 --- a/tests/recipes/test_deepseek41_row_shard.py +++ b/tests/recipes/test_deepseek41_row_shard.py @@ -99,6 +99,7 @@ def _recording_layer(scratch, *, reseed=False): shard = scratch.row_shard layer = object.__new__(decoder.PreparedLayer) layer.scratch, layer.row_shard, layer.reseed = scratch, shard, reseed + layer.attention_shard, layer.selection_move = False, None layer.engram = layer.concurrent_shared = None layer.image_mask, layer.has_image_mask = None, False attended = shard.attention_output if shard is not None else scratch.attention_output @@ -113,6 +114,8 @@ def _recording_layer(scratch, *, reseed=False): layer.attention_reduce = _Recorder(attended) layer.mlp_reduce = _Recorder() layer.router_projection = _Recorder(scratch.router_logits) + layer.router_source = (scratch.ffn_hc.normalized if shard is None else shard.ffn_hc.normalized) + layer.router_gather = None layer.router = _Recorder(SimpleNamespace(ids=torch.zeros(1, dtype=torch.int32), weights=torch.zeros(1), selected_logits=None)) layer.experts = _Recorder(scratch.routed) diff --git a/tests/recipes/test_deepseek41_row_shard_attention.py b/tests/recipes/test_deepseek41_row_shard_attention.py new file mode 100644 index 000000000..89b891693 --- /dev/null +++ b/tests/recipes/test_deepseek41_row_shard_attention.py @@ -0,0 +1,654 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Flare community contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Row-sharded prefill attention replicates every head and keeps each cache complete.""" + +from types import SimpleNamespace + +import pytest +import torch + + +def test_attention_weight_extents_partition_per_rank_and_replicate_every_group(): + from ignition.components.parallel import TensorParallelShard + from ignition.recipes.deepseek41.config import TEXT_GEOMETRY, attention_weight_extents + + text = TEXT_GEOMETRY + extents = [attention_weight_extents(text, TensorParallelShard(rank, 4)) for rank in range(4)] + # The tensor-parallel partition keeps 16 heads and 2 output groups per rank: disjoint and whole. + assert [extent.heads for extent in extents] == [ + (0, 8192), (8192, 16384), (16384, 24576), (24576, 32768)] + assert [extent.groups for extent in extents] == [ + (0, 2048), (2048, 4096), (4096, 6144), (6144, 8192)] + assert [extent.sink for extent in extents] == [(0, 16), (16, 32), (32, 48), (48, 64)] + # A row-sharded rank replicates all 64 heads, all 8 output groups, and all 64 sinks. + assert all(extent.all_heads == (0, 32768) for extent in extents) + assert all(extent.all_groups == (0, 8192) for extent in extents) + assert all(extent.all_sink == (0, 64) for extent in extents) + + +def _checkpoint(**changes): + """The checkpoint's text and quantization blocks, with two Engram layers and three targets.""" + from ignition.recipes.deepseek41.config import TEXT_GEOMETRY + + return {"model_type": "deepseek_v41", "architectures": ["DeepseekV41ForCausalLM"], + "text_config": {**TEXT_GEOMETRY, "compress_ratios": [0, 0] + [2] * 18 + [1] * 20 + [0] * 3, + "kv_source_layer_ids": [2, 8, 14, 20], "engram_layer_ids": [1, 14], + "index_source_layer_ids": [2, 8, 14, 20, 24, 28, 32, 36], + "candidate_source_layer_id": 20, "num_nextn_predict_layers": 3, + "dspark_block_size": 5, "dspark_noise_token_id": 128799, + "dspark_target_layer_ids": [37, 38, 39], "dspark_markov_rank": 256, + "dspark_n_routed_experts": 128, "dspark_num_experts_per_tok": 3, **changes}, + "quantization_config": {"quant_method": "fp8", "weight_block_size": [32, 32], + "scale_fmt": "ue8m0", "expert_dtype": "fp4", "activation_scheme": "dynamic"}} + + +def _model_config(**changes): + """The checkpoint config as the loader parses it, with `changes` replacing text entries.""" + from ignition.recipes.deepseek41.config import ModelConfig + + return ModelConfig.from_dict(_checkpoint(**changes)) + + +@pytest.mark.parametrize("world_size", [2, 4, 8]) +def test_replicated_attention_weights_charge_retained_tensor_layouts(world_size): + from ignition.recipes.deepseek41.config import replicated_attention_bytes, replicated_layer_bytes + + # Retained q_b_full, output_full.wo_a, output_full.wo_b MXFP8 layouts, followed by + # sink_full. The MMA scale layout is physically separate from the compact row scales. + layouts = ( + ((32768, 1280), torch.float8_e4m3fn), + ((1, 32768, 40), torch.uint8), + ((32, 4, 256, 4, 10, 1), torch.uint8), + ((1024, 4096, 8), torch.float8_e4m3fn), + ((8, 1024, 128), torch.uint8), + ((32, 4, 8, 4, 32, 8), torch.uint8), + ((5120, 8192), torch.float8_e4m3fn), + ((1, 5120, 256), torch.uint8), + ((32, 4, 40, 4, 64, 1), torch.uint8), + ((64,), torch.float32), + ) + tensors = (torch.empty(shape, dtype=dtype, device="meta") for shape, dtype in layouts) + retained = sum(tensor.numel() * tensor.element_size() for tensor in tensors) + assert retained == 124_780_800 + assert replicated_layer_bytes(world_size) == retained + config = _model_config() + packed = len(config.shardable_attention_layers(config.prefill_stop(False))) + scored = len(config.shardable_attention_layers(config.prefill_stop(True))) + draft_only = len(config.shardable_attention_layers(config.prefill_stop(True), config.draft.target_layers)) + assert (packed, scored, draft_only) == (17, 37, 35) + for layers, modes in ((packed, {}), (scored, {"prompt_logprobs": True}), + (packed, {"mtp_only": True}), + (draft_only, {"mtp_only": True, "prompt_logprobs": True})): + assert replicated_attention_bytes(world_size, **modes) == layers * retained, modes + + +def test_only_layers_a_prepared_prefill_can_shard_load_the_replicated_projection(monkeypatch): + """A layer no prepared prefill runs on the shard keeps only its partition. + + The packed prefill stops at the decoder boundary, whose suffix a replay restores from full + rows, so 17 of the 40 backbone layers replicate. Scored prefill replays no suffix and stops + below the last layer, so 37 do. A draft-target layer runs full-row only in the graphs that load + the draft, and every plan that loads the draft prepares those graphs beside the ones without + it, so its target layers stay in the shardable set unless the plan builds draft graphs only. + """ + from ignition.components import block_fp8_weights + from ignition.components.parallel import TensorParallelShard + from ignition.recipes.deepseek41 import weights + from ignition.recipes.deepseek41.config import ModelConfig + from ignition.recipes.deepseek41.decoder import shard_schedule + + config, targets = _model_config(), (37, 38, 39) + # The loader reads the checkpoint's config and hands manifest reads to its base class, whose + # tensor transfers this case replaces; construction and the load decisions stay the real ones. + monkeypatch.setattr(weights, "persistent_copy", lambda tensor: tensor) + monkeypatch.setattr(ModelConfig, "load", classmethod(lambda cls, snapshot: config)) + monkeypatch.setattr(block_fp8_weights.BlockFP8WeightLoader, "__init__", lambda self, *a, **k: None) + + class RecordingLoader(weights.DeepSeekWeightLoader): + """Substitute tensor reads with placeholders, keeping the load decisions.""" + + def _record(self, *request): + return torch.zeros((), dtype=torch.float32) + + def load_linear_group(self, prefixes, **_): + return self._record(prefixes) + + def load_linear(self, prefix, **_): + return self._record(prefix) + + def load_linear_columns(self, prefix, column_ranges): + return self._record(prefix, column_ranges) + + def load_tensor(self, name, **_): + return self._record(name) + + def load_padded_tensor(self, name, **_): + return self._record(name) + + def load_attention_output(self, prefix, groups, **_): + return self._record(prefix, groups) + + def load_experts(self, prefixes, interval): + return self._record(prefixes) + + def replicated(prompt_logprobs=False, *, dspark=True, mtp_only=False): + loader = RecordingLoader(SimpleNamespace(metadata_path=None), None, shard=TensorParallelShard(0, 4), + dspark=dspark, row_shard_attention=True, prompt_logprobs=prompt_logprobs, + mtp_only=mtp_only) + loaded = {index for index in range(config.layers) + if {"q_b_full", "output_full", "sink_full"} <= set(loader.load_layer(index).attention)} + assert loaded == set(loader.shardable_layers) + return loaded + + def scheduled(stop, aux=()): + return {index for index, entry in enumerate(shard_schedule(config, stop, object(), aux)) if entry[0]} + + packed = replicated() + assert sorted(packed) == [0, *range(2, 14), *range(15, 19)] + # Neither mode's packed prefill reaches the draft target layers, so both schedules agree. + assert packed == scheduled(config.decoder_start - 1) == scheduled(config.decoder_start - 1, targets) + scored = replicated(True) + assert sorted(scored) == [index for index in range(39) if index not in (1, 14)] + assert scored == scheduled(config.layers - 1) + # The graphs that load the draft hold its target layers full-row, and the graphs without it do + # not, so the loader's set is the wider of the two. + assert scheduled(39, targets) == set(range(39)) - {1, 14, 37, 38} + assert packed < scheduled(39, targets) < scored + # A draft-only plan builds no prefill graph without the draft, so its scored prefill shards + # every layer below the three target layers and the loader stops replicating those three. + draft_only = replicated(True, mtp_only=True) + assert sorted(draft_only) == [index for index in range(37) if index not in (1, 14)] + assert draft_only == scheduled(config.layers - 1, targets) and draft_only < scored + # Without scored prefill the stop lies below every target layer, so the set does not change. + assert replicated(mtp_only=True) == packed + + +def test_a_schedule_planning_did_not_qualify_still_loads_without_row_shard_attention(monkeypatch): + """Only a row-sharded attention load reads the schedule the plan sizes its charge from.""" + from ignition.components import block_fp8_weights + from ignition.components.parallel import TensorParallelShard + from ignition.recipes.deepseek41 import weights + from ignition.recipes.deepseek41.config import ModelConfig + + # Another Engram set, another draft target set, and a checkpoint that declares no targets, which + # reads as an empty set. + for change in ({"engram_layer_ids": [1, 13]}, {"dspark_target_layer_ids": [36, 37, 38]}, + {"dspark_target_layer_ids": []}): + config = ModelConfig.from_dict(_checkpoint(**change)) + assert config.layers == 40 + monkeypatch.setattr(ModelConfig, "load", classmethod(lambda cls, snapshot: config)) + monkeypatch.setattr(block_fp8_weights.BlockFP8WeightLoader, "__init__", lambda self, *a, **k: None) + # A plan without the option replicates no attention weights, so it loads this checkpoint. + plain = weights.DeepSeekWeightLoader(SimpleNamespace(metadata_path=None), None, + shard=TensorParallelShard(0, 4)) + assert not plain.row_shard_attention + # The charge comes from `QUALIFIED_SCHEDULE` before the checkpoint is read, so the option + # refuses a schedule the plan did not size. + with pytest.raises(ValueError, match="row-sharded attention"): + weights.DeepSeekWeightLoader(SimpleNamespace(metadata_path=None), None, + shard=TensorParallelShard(0, 4), row_shard_attention=True) + + +def _surfaces(monkeypatch, *, abi=11, pipe_bytes=1 << 26): + import b12x.comm.roce._proxy as proxy + from ignition.components import collectives + + monkeypatch.setattr(proxy, "ABI_VERSION", abi) + monkeypatch.setattr(collectives, "_roce", lambda runtime: True) + return SimpleNamespace(world_size=2, pipe_bytes=pipe_bytes, _group=object(), + should_reduce_scatter=lambda inp: True, should_gather_shards=lambda inp: True) + + +def test_attention_scratch_keeps_the_full_row_layout_unless_a_shard_is_asked_for(): + from ignition.recipes.deepseek41.attention import AttentionScratch + + plain = AttentionScratch.allocate(1024, 4, "cpu", candidate_tokens=16) + # Without a shard: one rank's 16 heads and a full-row query side. + assert plain.query.shape == plain.attended.shape == (1024, 16, 512) + assert plain.projected.shape == (1024, 1792) and plain.window_values.shape == (1024, 512) + assert plain.index_keys.shape == plain.normalized_keys.shape == (1024, 128) + + +def test_attention_scratch_shard_holds_every_head_and_full_row_index_keys(): + from ignition.recipes.deepseek41.attention import AttentionScratch + + shard = AttentionScratch.allocate(1024, 4, "cpu", candidate_tokens=16, shard_rows=256) + assert shard.query.shape == shard.attended.shape == (256, 64, 512) + assert shard.projected.shape == (256, 1792) and shard.window_values.shape == (256, 512) + assert shard.query_low.shape == (256, 1280) and shard.selected.shape == (256, 512) + assert shard.candidates.shape == (16, 16384) and shard.candidates.dtype is torch.int32 + # The compressor still runs over every row, so its key planes stay full-row. + assert shard.index_keys.shape == shard.normalized_keys.shape == (1024, 128) + narrowed = shard.rows(128, 1024) + assert narrowed.query.shape == (128, 64, 512) and narrowed.selected.shape == (128, 512) + # A narrower graph keeps the key planes at that graph's row count, which indexes its slots. + assert narrowed.index_keys.shape == narrowed.normalized_keys.shape == (1024, 128) + assert shard.rows(128, 512).index_keys.shape == (512, 128) + with pytest.raises(ValueError, match="attention scratch rows"): + shard.rows(257, 1024) + with pytest.raises(ValueError, match="attention scratch tokens"): + shard.rows(128, 1025) + + +def test_row_shard_allocates_attention_and_full_row_gather_targets_only_when_asked(): + from ignition.recipes.deepseek41.decoder import RowShard + + plain = RowShard.allocate(2048, 2, 1, "cpu", candidate_tokens=16) + assert (plain.rows, plain.start) == (1024, 1024) and plain.attention is None + assert plain.window_values is None and plain.router_logits is None + + asked = RowShard.allocate(2048, 2, 1, "cpu", candidate_tokens=16, attention=True) + assert (asked.rows, asked.start, asked.rows_index[0].item()) == (1024, 1024, 1024) + assert asked.attention.query.shape == (1024, 64, 512) + # The KV latent target spans every row: it is what the gathered window write stores. + assert asked.window_values.shape == (2048, 512) + assert asked.router_logits.shape == (1024, 384) and asked.router_logits.dtype is torch.float32 + + narrowed = asked.graph(1024) + # A narrower graph gives every rank a shorter block, so rank 1 starts at 512, not at 1024. + assert (narrowed.rows, narrowed.start) == (512, 512) + assert narrowed.rows_index.tolist() == list(range(512, 1024)) + assert narrowed.attention.query.shape == (512, 64, 512) + # The narrowed graph's index keys cover its own full rows, and its KV target all of them. + assert narrowed.attention.index_keys.shape == (1024, 128) + assert narrowed.window_values.shape == (1024, 512) and narrowed.router_logits.shape == (512, 384) + assert asked.graph(1023) is None + + +@pytest.mark.parametrize("tokens", [8192, 6144, 4096, 2048]) +@pytest.mark.parametrize("rank", [0, 1, 2, 3]) +def test_a_narrower_graph_keeps_each_rank_inside_its_own_rows(tokens, rank): + from ignition.recipes.deepseek41.decoder import RowShard + + captured = RowShard.allocate(8192, 4, rank, "cpu", candidate_tokens=16, attention=True) + shard = captured.graph(tokens) + rows = tokens // 4 + # One prefill chunk size captures the layout; every smaller graph re-derives this rank's block + # from the graph's own row count, so an attention slice never runs past the graph's rows. + assert (shard.rows, shard.start) == (rows, rank * rows) + assert shard.rows_index.tolist() == list(range(rank * rows, (rank + 1) * rows)) + assert captured.graph(tokens).start == shard.start + + +@pytest.mark.parametrize("abi,ask,ready", [ + (11, True, True), (10, True, False), (11, False, False)]) +def test_attention_shard_ready_requires_the_abi_and_the_shard_it_gathers(monkeypatch, abi, ask, ready): + from ignition.recipes.deepseek41.decoder import RowShard, attention_shard_ready + + runtime = _surfaces(monkeypatch, abi=abi) + tokens = 1024 + shard = RowShard.allocate(tokens, 2, 0, "cpu", candidate_tokens=16, attention=ask) + scratch = SimpleNamespace(router_logits=torch.zeros(tokens, 384, dtype=torch.float32), + attention=SimpleNamespace(selected=torch.zeros(tokens, 512, dtype=torch.int32))) + assert attention_shard_ready(scratch, shard, runtime) is ready + + +def test_attention_shard_ready_rejects_a_gather_that_does_not_fit_the_pipe(monkeypatch): + from ignition.recipes.deepseek41.decoder import RowShard, attention_shard_ready + + shard = RowShard.allocate(1024, 2, 0, "cpu", candidate_tokens=16, attention=True) + scratch = SimpleNamespace(router_logits=torch.zeros(1024, 384, dtype=torch.float32), + attention=SimpleNamespace(selected=torch.zeros(1024, 512, dtype=torch.int32))) + # Every sharded row is gathered whole; the widest one decides whether the runtime serves it. + runtime = _surfaces(monkeypatch) + assert attention_shard_ready(scratch, shard, runtime) is True + assert attention_shard_ready(scratch, shard, _surfaces(monkeypatch, pipe_bytes=1 << 12)) is False + assert attention_shard_ready(scratch, shard, _surfaces(monkeypatch, abi=10)) is False + monkeypatch.setattr("ignition.components.collectives._roce", lambda runtime: False) + assert attention_shard_ready(scratch, shard, runtime) is False + + +def _indexing_config(sources, candidate): + return SimpleNamespace(decoder_start=candidate, text={"candidate_source_layer_id": candidate}, + index_source=lambda layer: next((i for i in reversed(sources) if i <= layer), None)) + + +def _schedule_config(engram=(1, 14)): + from ignition.recipes.deepseek41.config import ModelConfig + + return ModelConfig({"num_hidden_layers": 40, "engram_layer_ids": list(engram)}, {}) + + +def test_selection_moves_follow_the_row_transition_around_the_engram_layer(): + """Layers 15-18 read layer 14's latent pages, so they must carry its selection state. + + A selection is not recomputed between index sources: the value one layer returns is the one the + next layer reads, and its rows are those of the layer that wrote it. Layer 14 runs on full rows + and layer 15 on a shard, so layer 15 takes this rank's rows; layer 18 crosses back and gathers + them again for the packed suffix. + """ + from ignition.recipes.deepseek41.decoder import SelectionMove, selection_moves, shard_schedule + + config = _indexing_config([2, 8, 14, 20, 24, 28, 32, 36], 20) + packed = selection_moves(config, shard_schedule(_schedule_config(), 18, object()), attention_shard=True) + # A source reads no carried selection, so layer 14 only writes its own, in full rows. + assert all(move is None for move in packed[:15]) + assert packed[15] == SelectionMove(14, "full_to_shard") + assert all(move is None for move in packed[16:18]) + assert packed[18] == SelectionMove(14, "shard_to_full") + assert all(move is None for move in packed[19:]) + # Attention sharding is the only thing that creates the two representations. + assert selection_moves(config, shard_schedule(_schedule_config(), 18, object()), + attention_shard=False) == (None,) * 40 + + +def test_a_full_row_consumer_gathers_a_sharded_sources_selection(): + """Scored prefill has no packed suffix, so its last full-row layer consumes a shard's state.""" + from ignition.recipes.deepseek41.decoder import SelectionMove, selection_moves, shard_schedule + + config = _indexing_config([2, 8, 14, 20, 24, 28, 32, 36], 20) + scored = selection_moves(config, shard_schedule(_schedule_config(), 39, object()), attention_shard=True) + assert scored[15] == SelectionMove(14, "full_to_shard") + assert all(move is None for move in scored[16:39]) + assert scored[39] == SelectionMove(36, "shard_to_full") + + +def test_a_shard_takes_its_own_rows_of_the_carried_selection(): + """Rows, candidate rows and counts all move by the shard's row numbers, not from row zero.""" + from ignition.recipes.deepseek41 import attention, decoder + + tokens, world, rows = 128, 2, 64 + full = attention.AttentionScratch.allocate(tokens, world, "cpu", candidate_tokens=tokens) + shard = attention.AttentionScratch.allocate(tokens, world, "cpu", candidate_tokens=tokens, shard_rows=rows) + full.selected.copy_(torch.arange(tokens, dtype=torch.int32)[:, None].expand(tokens, 512)) + full.candidates.copy_(torch.arange(tokens, dtype=torch.int32)[:, None].expand(tokens, 16384)) + full.candidate_counts.copy_(torch.arange(tokens, dtype=torch.int32)) + prepared = decoder.PreparedSelectionRefill( + decoder.SelectionMove(14, "full_to_shard", "full_to_shard"), full=full, shard=shard, + rows_index=torch.arange(rows) + rows, runtime=None, session=None, coordinator=None, name="test") + selected, candidates, counts = prepared((full.selected, full.candidates, full.candidate_counts)) + assert torch.equal(selected, full.selected[rows:]) + assert torch.equal(candidates, full.candidates[rows:]) + assert torch.equal(counts, full.candidate_counts[rows:]) + assert not torch.equal(selected, full.selected[:rows]) + + +def test_a_sharded_candidate_shortlist_is_refused_rather_than_read_stale(): + from ignition.recipes.deepseek41 import attention, decoder + + full = attention.AttentionScratch.allocate(64, 2, "cpu", candidate_tokens=64) + shard = attention.AttentionScratch.allocate(64, 2, "cpu", candidate_tokens=64, shard_rows=32) + with pytest.raises(ValueError, match="candidate shortlist"): + decoder.PreparedSelectionRefill(decoder.SelectionMove(20, None, "shard_to_full"), + full=full, shard=shard, rows_index=torch.arange(32) + 32, runtime=None, session=None, + coordinator=None, name="test") + + +class _GatheredRows: + """Stand-in for the shard gather: this rank's rows land in its block, the others keep theirs.""" + + def __init__(self, shard, source, output, **options): + self.shard, self.source, self.output, self.plan = shard, source, output, object() + + def functional(self, **options): + return self + + def __call__(self, source): + gathered = self.output.clone() + gathered[self.shard.start:self.shard.start + self.shard.rows] = source + return gathered + + +def _stub(*args, **kwargs): + return SimpleNamespace(plan=object(), functional=lambda **options: None) + + +def _full_row_layer(monkeypatch, scratch, metadata, *, index, selection_move, shard): + """`PreparedLayer` as the decoder builds the full-row layer that crosses back off the shard.""" + from ignition.recipes.deepseek41 import decoder + + for name in ("PreparedMHC", "PreparedAttention", "PreparedAllReduce", "PreparedReduceScatter", + "PreparedChunkedGatherShards", "UnquantizedProjection", "PreparedRoutedExperts", + "PreparedMLP", "PreparedMHCPost", "PreparedRouter"): + monkeypatch.setattr(decoder, name, _stub) + monkeypatch.setattr(decoder, "PreparedGatherShards", + lambda source, output, **options: _GatheredRows(shard, source, output, **options)) + monkeypatch.setattr(decoder, "_mhc_weights", lambda *args: None) + text = dict(rms_norm_eps=1e-6, hc_eps=1e-6, hc_sinkhorn_iters=20, scoring_func="sigmoid", + norm_topk_prob=True, routed_scaling_factor=1., swiglu_limit=10.) + weights = SimpleNamespace(engram=None, attention=None, experts=SimpleNamespace( + vision_correction=None, router=None, correction=None, experts=None, shared=None)) + return decoder.PreparedLayer(index, weights, None, scratch, metadata, + SimpleNamespace(config=SimpleNamespace(text=text)), None, + shard=SimpleNamespace(world_size=2), mode="prefill", runtime=None, coordinator=None, + session=object(), name="layer", workspace=SimpleNamespace(overlap=False), carry=False, + defer_post=False, row_shard=None, attention_shard=False, selection_move=selection_move) + + +@pytest.mark.parametrize("scored", [False, True]) +def test_the_layer_that_crosses_back_to_full_rows_gathers_the_carried_selection(monkeypatch, scored): + """Packed V4.1 prefill stops at layer 19, which reads the selection layers 15-18 wrote on the shard. + + `shard_schedule` leaves the layer at the stop, and the DSpark target layers a scored prefill + loads, on full rows, so `selection_moves` gives that layer a shard-to-full move. It runs no + shard of its own, so the move gathers through the decoder's shard into the full-row selection + its own attention reads. + """ + from ignition.recipes.deepseek41 import decoder + + config = _model_config() + stop = config.prefill_stop(scored) + schedule = decoder.shard_schedule(config, stop, object(), config.draft.target_layers) + moves = decoder.selection_moves(config, schedule, attention_shard=True) + crossing = config.draft.target_layers[0] if scored else stop + assert schedule[crossing][0] is False + assert moves[crossing] == decoder.SelectionMove(14 if not scored else 36, "shard_to_full") + + tokens, world = 1024, 2 + scratch = decoder.Scratch.allocate(tokens, 8, world, "cpu", row_shard=SimpleNamespace(rank=1), + attention_shard=True) + shard = scratch.row_shard + metadata = SimpleNamespace(token_ids=torch.empty(tokens), + live_counts=torch.tensor([tokens, 1], dtype=torch.int32)) + layer = _full_row_layer(monkeypatch, scratch, metadata, index=crossing, + selection_move=moves[crossing], shard=shard) + assert layer.selection_move is not None + gather = layer.selection_move.gather + # The move reads the decoder's shard and writes the full-row selection this layer's attention reads. + assert gather.source.data_ptr() == shard.attention.selected.data_ptr() + assert gather.output.data_ptr() == scratch.attention.selected.data_ptr() + # Shape check only: the stub stands in for the collective, so this says nothing about the rows. + selected, _, _ = layer.selection_move((shard.attention.selected, None, None)) + assert selected.shape == (tokens, 512) and selected.dtype is torch.int32 + + +class _Recorder: + """A stand-in for one prepared component that returns the surface the forward reads.""" + + def __init__(self, value=None): + self.value, self.calls, self.kwargs = value, [], {} + + def __call__(self, *args, **kwargs): + self.calls.append(args) + self.kwargs = kwargs + return self.value + + def _run(self, *args, **kwargs): + self.calls.append(args) + self.kwargs = kwargs + return self.value + + +def _attention_layer(scratch): + """`PreparedLayer` over recorders, for a layer that runs attention on its own rows.""" + from ignition.recipes.deepseek41 import decoder + + shard = scratch.row_shard + layer = object.__new__(decoder.PreparedLayer) + layer.scratch, layer.row_shard, layer.attention_shard = scratch, shard, True + layer.reseed, layer.selection_move, layer.engram, layer.boundary = False, None, None, None + layer.concurrent_shared = None + layer.post = _Recorder(shard.residual) + layer.has_image_mask, layer.image_mask = False, None + branch = shard.attention_hc + layer.attention_hc = _Recorder((branch.residual, branch.post, branch.comb, branch.normalized)) + layer.attention_gather = None + # The shard's attention returns this rank's completed rows; no reduction follows. + attended = torch.zeros(scratch.row_shard.rows, 5120, dtype=torch.bfloat16) + layer.attention = _Recorder((attended, {}, None, {})) + layer.attention_reduce = None + ffn = shard.ffn_hc + layer.ffn_hc = _Recorder((ffn.residual, ffn.post, ffn.comb, ffn.normalized)) + layer.ffn_gather = _Recorder(scratch.ffn_hc.normalized) + layer.router_projection = _Recorder(shard.router_logits) + layer.router_gather = _Recorder(scratch.router_logits) + layer.router = _Recorder(SimpleNamespace(ids=torch.zeros(1, dtype=torch.int32), + weights=torch.zeros(1), selected_logits=None)) + layer.experts = _Recorder(scratch.routed) + layer.shared = _Recorder(scratch.shared.output) + layer.mlp_reduce = _Recorder(scratch.mlp_output) + layer.residual_gather, layer.mix_gather = _Recorder(scratch.residual), _Recorder(scratch.pre_mix) + return layer, attended + + +def test_a_sharded_layer_needs_no_attention_gather_or_reduction(): + """Attention reads this rank's branch input and its own rows are complete once it returns.""" + from ignition.recipes.deepseek41 import decoder + + scratch = decoder.Scratch.allocate(1024, 8, 2, "cpu", row_shard=SimpleNamespace(rank=1), + attention_shard=True) + shard = scratch.row_shard + layer, attended = _attention_layer(scratch) + layer._run((shard.residual,), shard.pre_mix, SimpleNamespace(token_ids=torch.zeros(1024, dtype=torch.int64)), + None, (torch.full((shard.rows, 512), -1, dtype=torch.int32), None, None)) + (attention_source,) = layer.attention.calls[0][:1] + assert attention_source.data_ptr() == shard.attention_hc.normalized.data_ptr() + attention_output = layer.ffn_hc.calls[0][0] + assert attention_output.data_ptr() == attended.data_ptr() + assert attended.shape == (shard.rows, 5120) + + +def test_a_sharded_layer_projects_its_router_logits_and_gathers_them_to_every_row(): + """Top-k reads one row at a time, so the shard projects its rows and one gather feeds the router.""" + from ignition.recipes.deepseek41 import decoder + + scratch = decoder.Scratch.allocate(1024, 8, 2, "cpu", row_shard=SimpleNamespace(rank=1), + attention_shard=True) + shard = scratch.row_shard + layer, _ = _attention_layer(scratch) + layer._run((shard.residual,), shard.pre_mix, SimpleNamespace(token_ids=torch.zeros(1024, dtype=torch.int64)), + None, (torch.full((shard.rows, 512), -1, dtype=torch.int32), None, None)) + (router_source,) = layer.router_projection.calls[0] + assert router_source.data_ptr() == shard.ffn_hc.normalized.data_ptr() + (logits,) = layer.router_gather.calls[0] + # The gather moves BF16 packs of the FP32 logits it was prepared over. + assert logits.dtype is torch.bfloat16 and logits.shape == (shard.rows, 768) + assert layer.router.calls[0][0].data_ptr() == scratch.router_logits.data_ptr() + # The routed experts keep every row. + assert layer.experts.calls[0][0].data_ptr() == scratch.ffn_hc.normalized.data_ptr() + (residual,) = layer.residual_gather.calls[0] + assert residual.data_ptr() == shard.residual.data_ptr() + + +def test_compressor_gathers_values_and_gates_through_the_view_they_were_prepared_from(monkeypatch): + from ignition.recipes.deepseek41 import attention + + values, gates = torch.zeros(16, 512), torch.zeros(16, 512) + compressor = object.__new__(attention.PreparedCompressor) + compressor.ratio, compressor.layer, compressor.deferred, compressor.total = 2, 0, True, 64 + compressor.weight, compressor.plan = None, SimpleNamespace(handle=None) + compressor.metadata = SimpleNamespace(state_rows=4, starts=torch.zeros(4, dtype=torch.int64), + query_start_loc=torch.zeros(5, dtype=torch.int32), live_counts=torch.zeros(2, dtype=torch.int32)) + compressor.pool = SimpleNamespace(region=lambda name: None) + compressor.local_values, compressor.local_gates = values, gates + compressor.project_values, compressor.project_gates = _Recorder(values), _Recorder(gates) + compressor.values_gather = _Recorder(torch.zeros(64, 512, dtype=torch.float32)) + compressor.gates_gather = _Recorder(torch.zeros(64, 512, dtype=torch.float32)) + monkeypatch.setattr(attention, "gather_rows_three", lambda *args: None) + calls = [] + import b12x.attention.mla_compress._impl as implementation + monkeypatch.setattr(implementation, "_run_ratio2", lambda *args: calls.append(args)) + compressor(torch.zeros(16, 512), compressor.metadata, None) + # A ratio-two compressor holds FP32 values and gates; both gathers move BF16 packs of them. + (gathered_gates,) = compressor.gates_gather.calls[0] + assert gathered_gates.dtype is torch.bfloat16 and gathered_gates.shape == (16, 1024) + assert gathered_gates.data_ptr() == gates.data_ptr() + (gathered_values,) = compressor.values_gather.calls[0] + assert gathered_values.dtype is torch.bfloat16 and gathered_values.shape == (16, 1024) + assert gathered_values.data_ptr() == values.data_ptr() + # The compression kernel still spans every row, so it reads the gathered products. + kernel_values, kernel_gates = calls[0][1], calls[0][2] + assert kernel_values.shape == kernel_gates.shape == (64, 512) + assert kernel_values.dtype is torch.float32 + + +def test_compiled_row_shard_forwards_read_no_rank_offset_as_a_python_integer(): + """One TP forward compiles per shape and every rank binds that artifact. + + Dynamo turns an attribute integer read inside a compiled forward into a guard, so a rank's row + offset would be compiled into the artifact and every other rank would be rejected with a binding + mismatch. Each forward therefore reads per-rank data only through the row numbers the graph has. + """ + import inspect + import re + + from ignition.recipes.deepseek41 import attention, decoder + + for forward in (attention.PreparedAttention._run, decoder.PreparedLayer._run): + source = inspect.getsource(forward) + assert re.search(r"\b(?:self|row_shard|shard)\.start\b", source) is None + prepared = inspect.getsource(attention.PreparedAttention._run) + assert "rows_index = self.rows_index" in prepared + assert "full.index_select(0, rows_index)" in prepared + assert "indices.window_indices[self" not in prepared + assert "m.token_page_table[self" not in prepared + + +class _Operand: + """A stand-in prepared operand that returns its buffer and is its own functional result.""" + + def __init__(self, value): + self.value, self.plan = value, object() + + def functional(self, **options): + return self + + def __call__(self, *args, **kwargs): + return self.value + + +@pytest.mark.parametrize("ratio,row_shard", [(2, False), (2, True), (1, False), (1, True)]) +def test_a_prepared_compressor_passes_the_norm_its_kernel_reads(monkeypatch, ratio, row_shard): + """The compression kernel takes the norm from the compressor, not from the prepared declaration. + + `self.arguments` is the declaration the preparation binds and it is released with it, so the + forward's operands are attributes of the compressor. + """ + from ignition.recipes.deepseek41 import attention + + import b12x.attention.mla_compress._impl as implementation + + calls = [] + monkeypatch.setattr(implementation, "_run_ratio1", lambda *args: calls.append(("ratio1", args))) + monkeypatch.setattr(implementation, "_run_ratio2", lambda *args: calls.append(("ratio2", args))) + monkeypatch.setattr(attention, "declare", lambda *args, **kwargs: None) + monkeypatch.setattr(attention, "register_finalizer", lambda *args, **kwargs: None) + monkeypatch.setattr(attention, "gather_rows_three", lambda *args: None) + monkeypatch.setattr(attention, "UnquantizedProjection", + lambda source, weight, out, **options: _Operand(out)) + monkeypatch.setattr(attention, "PreparedGatherShards", + lambda source, output, **options: _Operand(output)) + # The native planner requires CUDA, and this case runs the compressor's operand flow on CPU. + monkeypatch.setattr(attention.mla_compress, "Caps", lambda **options: SimpleNamespace(**options)) + monkeypatch.setattr(attention.mla_compress, "plan", + lambda caps: SimpleNamespace(handle=object(), request=lambda **options: _Operand(None))) + tokens, rows = 64, 4 + text = {"compress_ratios": [0, 0] + [2] * 18 + [1] * 20} + pool = SimpleNamespace(config=SimpleNamespace(text=text), region=lambda name: None) + metadata = SimpleNamespace(starts=torch.zeros(rows, dtype=torch.int64), + query_start_loc=torch.zeros(rows + 1, dtype=torch.int32), + state_rows=torch.zeros(rows, dtype=torch.int32), + lengths=torch.zeros(rows, dtype=torch.int32), + live_counts=torch.zeros(2, dtype=torch.int32), + positions=torch.zeros(tokens, dtype=torch.int64)) + norm = torch.zeros(512) + weights = {"norm": norm, "values": torch.zeros(8), "gates": torch.zeros(8)} + compressor = attention.PreparedCompressor(2 if ratio == 2 else 20, weights, + torch.zeros((tokens, 512), dtype=torch.bfloat16), metadata, pool, + torch.zeros(tokens, dtype=torch.int64), session=object(), name="test", deferred=True, + row_shard=SimpleNamespace() if row_shard else None, runtime=None, coordinator=None) + compressor(torch.zeros((tokens, 512), dtype=torch.bfloat16), metadata, torch.zeros(tokens, dtype=torch.int64)) + kind, args = calls[0] + # `_run_ratio2(handle, values, gates, weight, ...)` and `_run_ratio1(handle, values, weight, ...)`. + assert args[3 if kind == "ratio2" else 2].data_ptr() == norm.data_ptr() diff --git a/tests/recipes/test_deepseek41_shard_schedule.py b/tests/recipes/test_deepseek41_shard_schedule.py index 00714da28..5219e645e 100644 --- a/tests/recipes/test_deepseek41_shard_schedule.py +++ b/tests/recipes/test_deepseek41_shard_schedule.py @@ -8,21 +8,15 @@ after it, and only a layer that runs its own post may change the dimension. """ -from dataclasses import dataclass - +from ignition.recipes.deepseek41.config import ModelConfig from ignition.recipes.deepseek41.decoder import shard_schedule -@dataclass(frozen=True) -class _Layer: - engram: object = None - - SHARD = object() -def layers(count=40, engram=()): - return tuple(_Layer("engram" if index in engram else None) for index in range(count)) +def config(count=40, engram=(1, 14)): + return ModelConfig({"num_hidden_layers": count, "engram_layer_ids": list(engram)}, {}) def flags(schedule, column): @@ -30,55 +24,55 @@ def flags(schedule, column): def test_shard_resumes_after_each_engram_layer(): - schedule = shard_schedule(layers(engram=(1, 14)), 19, SHARD) + schedule = shard_schedule(config(), 19, SHARD) assert flags(schedule, 0) == set(range(19)) - {1, 14} def test_shard_refills_from_full_rows_where_it_resumes(): - schedule = shard_schedule(layers(engram=(1, 14)), 19, SHARD) + schedule = shard_schedule(config(), 19, SHARD) assert flags(schedule, 2) == {2, 15} def test_only_same_row_layers_defer_their_post(): - schedule = shard_schedule(layers(engram=(1, 14)), 19, SHARD) + schedule = shard_schedule(config(), 19, SHARD) # Layers 19-38 defer their post while both neighbouring layers remain full-row. assert flags(schedule, 1) == {*range(2, 13), *range(15, 18), *range(19, 39)} def test_engram_layer_itself_stays_full_row(): - schedule = shard_schedule(layers(engram=(1, 14)), 19, SHARD) + schedule = shard_schedule(config(), 19, SHARD) for index in (1, 14): assert schedule[index][0] is False assert schedule[index][1] is False def test_tail_from_stop_on_stays_full_row(): - schedule = shard_schedule(layers(engram=(1, 14)), 19, SHARD) + schedule = shard_schedule(config(), 19, SHARD) assert not any(entry[0] for entry in schedule[19:]) def test_engram_first_layer_does_not_disable_the_rest(): - schedule = shard_schedule(layers(engram=(0,)), 19, SHARD) + schedule = shard_schedule(config(engram=(0,)), 19, SHARD) assert flags(schedule, 0) == set(range(1, 19)) assert flags(schedule, 2) == {1} assert schedule[0][0] is False def test_aux_layer_runs_full_row_and_restarts_the_shard(): - schedule = shard_schedule(layers(), 19, SHARD, aux_layers=(10,)) + schedule = shard_schedule(config(engram=()), 19, SHARD, aux_layers=(10,)) assert 10 not in flags(schedule, 0) assert 9 not in flags(schedule, 1) assert 11 in flags(schedule, 2) def test_stop_below_one_shards_nothing(): - schedule = shard_schedule(layers(engram=(1, 14)), 0, SHARD) + schedule = shard_schedule(config(), 0, SHARD) assert not any(entry[0] for entry in schedule) assert not any(entry[2] for entry in schedule) def test_without_a_shard_every_layer_keeps_the_legacy_defer(): - schedule = shard_schedule(layers(engram=(1, 14)), 19, None) + schedule = shard_schedule(config(), 19, None) expected = [index + 1 not in (1, 14) for index in range(39)] + [False] assert [entry[1] for entry in schedule] == expected assert not any(entry[0] or entry[2] for entry in schedule)