From 62f3089e00e3730f5c7a81d0b2a6aef9a70f72c6 Mon Sep 17 00:00:00 2001 From: Jason Cook Date: Sun, 15 Nov 2026 06:03:54 -0500 Subject: [PATCH] Shorten Inkling 2 prefill with BF16 router operands and a fused mix and norm Inkling 2 Small prefill on DGX Spark TP4 spent 994 ms of a 3,808 ms 8,192-row chunk in the expert router's projection, which widened its BF16 source and weight to FP32 so that only b12x's FP32-operand kernels were eligible, and 376 ms in attention-residual mixtures that stored an FP32 mixture per stream for a separate RMSNorm launch. The router needs FP32 accumulation, not FP32 operands: the product of two BF16 values is exact in FP32. The mixture and its norm read and write the same rows, so one program can normalize without storing the mixture. The router projection takes its BF16 source and a BF16 router weight and declares accumulation="fp32", which UnquantizedProjection passes to b12x so that only plans carrying the whole sum in FP32 are raced. The weight loader delivers the FP32 checkpoint tensor as BF16 only after checking that every value is an exact BF16 upcast. PreparedResidualMix normalizes each stream in the mixture kernel with FP32 mean square, scale and weight and one rounding to BF16; PreparedRMSNorm leaves this path. Prefill attention writes its partial sum in the cross-rank reduction's element type, BF16 over RoCEnante and FP32 on one host, in place of FP32 followed by a cast; decode and verification keep FP32. Prefill time to first token on four DGX Sparks falls by a factor of 1.5 to 1.6. Router selection is unchanged in value class: operands are the same numbers and only summation order differs. The fused norm differs from the unfused one by at most one BF16 unit in at most ten elements per million. Evidence: one exported core per tree served by flare run at TP4 on GB10, 8,192-row chunks, warm and uncached, three repeats. Time to first token was 2.59 to 2.69 s at 7,668 prompt tokens (4.18 to 4.51 s before), 22.4 to 23.0 s at 67,145 (35.2 to 35.4 s) and 48.8 to 49.8 s at 131,990 (73.5 s). The measured core also contained FlashInfer's swapped-operand FC1 selection and the drafting changes, so the factor is not attributable to this commit alone. In a kernel trace of rank 0, an 8,192-row chunk spans 2,509 ms with routing at 159 ms and mixtures with norms at 247 ms. On one GB10 the router projection for 8,192 rows takes 2.14 ms against 19.5 ms, with the same top-8 experts on every row. tests/recipes/test_inkling2_target.py passes 87 tests on a GB10 with the checkpoint, including attention against the reference in prefill, decode and verification and the fused-norm bound. Limitations: the projection requires GemvQuery.accumulation from FlashInfer (local-inference-lab/flashinfer#110), which the pinned FlashInfer revision lacks. Completion length was compared between a core from this tree with the drafting changes and a core from the unmodified tree: over lookup prompts with seeds 1 to 30, median completion tokens were 363 against 420 and the share at or above 1,024 tokens differed by -1.7 points (95% interval -7.2 to +3.8), with no wrong answer in 540 and 480 responses. tests/server/test_inkling2.py's prefix-reuse case, which fixes one seed and a 1,024-token limit, fails in 9 of 20 repeats on the changed core and 5 of 20 on the unmodified one. Single-host RTX PRO 6000 serving was not run. Design-Signed-Off-By: flare-02; design-review=ed22 --- ignition/components/linear.py | 11 +++- ignition/recipes/inkling2/attention.py | 9 +-- ignition/recipes/inkling2/decoder.py | 10 +++- ignition/recipes/inkling2/residual.py | 82 ++++++++++++++++++-------- ignition/recipes/inkling2/weights.py | 10 +++- tests/recipes/test_inkling2_target.py | 48 +++++++++++++-- 6 files changed, 129 insertions(+), 41 deletions(-) diff --git a/ignition/components/linear.py b/ignition/components/linear.py index ff92a995d..48b6d495b 100644 --- a/ignition/components/linear.py +++ b/ignition/components/linear.py @@ -144,7 +144,13 @@ def __init__( session: PreparationSession, name: str, bias: torch.Tensor | None = None, + accumulation: str = "any", ) -> None: + """Declare `source @ weight.T` (plus `bias`) into `output`. + + `accumulation="fp32"` admits only b12x plans that carry the whole sum of products in FP32, the + accuracy of the product of FP32 operands; `"any"` also admits tensor-core and library accumulation. + """ if any(t.dtype not in (torch.bfloat16, torch.float32) for t in (source, weight, output)): raise TypeError("unquantized projections require BF16 or FP32 operands") for tensor in (weight, bias): @@ -152,7 +158,7 @@ def __init__( record_numeric_tensor(tensor) # FlashInfer's GEMMs read and write contiguous rows; column slices of wider rows keep b12x's GEMV. rows = source.numel() // source.shape[-1] - self.dense = (sm103_projection.is_sm103(source.device) and bias is None + self.dense = (sm103_projection.is_sm103(source.device) and bias is None and accumulation == "any" and source.dtype == weight.dtype == torch.bfloat16 and source.is_contiguous() and output.is_contiguous() and sm103_projection.dense_serves(rows, output.shape[-1])) @@ -169,7 +175,8 @@ def __init__( session=session), weight=weight, bias=None)),)) return - query = bf16_gemv.query_from_call(source, weight, out=output, bias=bias) + query = bf16_gemv.query_from_call(source, weight, out=output, bias=bias, + **({} if accumulation == "any" else {"accumulation": accumulation})) self.plan = bf16_gemv.plan(query) self.weight = weight self.bias = bias diff --git a/ignition/recipes/inkling2/attention.py b/ignition/recipes/inkling2/attention.py index 4b430ddfa..e0c885333 100644 --- a/ignition/recipes/inkling2/attention.py +++ b/ignition/recipes/inkling2/attention.py @@ -8,8 +8,9 @@ or convolution histories. Accepted counts commit those histories afterward. Full-attention candidates write private request pages. Attention uses BF16 queries and relative-bias operands, FP8 K/V and FP32 row sinks. The gated -BF16 result projects to FP32 partial sums for cross-rank reduction before the -replicated BF16 output convolution. +BF16 result projects to partial sums of the cross-rank reduction's element type +(FP32 on one host, BF16 over RoCEnante, where the projection rounds its FP32 +accumulation once) before the replicated BF16 output convolution. """ from __future__ import annotations @@ -173,7 +174,7 @@ class PreparedAttention(torch.nn.Module): """A rank's attention for prefill, decode or speculative verification.""" def __init__(self, layer, weights: AttentionWeights, text, cache: LayerCache, metadata, *, mode, tokens, - session, workspace, name, examples=None, tuning_page_count=None): + session, workspace, name, examples=None, tuning_page_count=None, partial_dtype=torch.float32): super().__init__() if mode not in ("prefill", "decode", "verify"): raise ValueError(f"unsupported Inkling attention mode: {mode}") @@ -201,7 +202,7 @@ def example(role, shape, dtype=torch.bfloat16): for index, (weight, width) in enumerate(zip(weights.projections, self.columns.widths(), strict=True))) attended = example("inkling.attention.attended", (tokens, self.heads * self.dim)) self.output_projection = UnquantizedProjection(attended, weights.output, - example("inkling.attention.partial", (tokens, text.hidden), torch.float32), + example("inkling.attention.partial", (tokens, text.hidden), partial_dtype), session=session, name=f"{name}.output").functional(session=session) query = example("inkling.attention.query", (tokens, self.heads, self.dim)) output = example("inkling.attention.output", (tokens, self.heads, self.dim)) diff --git a/ignition/recipes/inkling2/decoder.py b/ignition/recipes/inkling2/decoder.py index 89e6241b5..d86763cd4 100644 --- a/ignition/recipes/inkling2/decoder.py +++ b/ignition/recipes/inkling2/decoder.py @@ -59,6 +59,9 @@ def __init__(self, layer, weights, text, pool, metadata, reduction, reduction_dt self.reduction, self.reduction_dtype = reduction, reduction_dtype self.attention = PreparedAttention(layer, weights.attention, text, LayerCache.of(pool, layer), metadata, mode=mode, tokens=tokens, session=session, workspace=workspace, name=name + ".attention", examples=examples, + # Prefill writes the attention partial sum in the reduction's element type. At prefill row counts the + # BF16 projection equals the FP32 projection rounded once; decode and verification keep FP32. + partial_dtype=reduction_dtype if mode == "prefill" else torch.float32, tuning_page_count=((int(pool.memory_capacity[0].item()) - 1) * pool.ring_row_pages + pool.geometry.ring // 128 if text.local(layer) else int(pool.memory_capacity[1].item()))) device = weights.router.device @@ -66,8 +69,9 @@ def example(role, shape, dtype=torch.bfloat16): return declaration_tensor(examples, "inkling.mlp." + role, shape, dtype, device) source = example("source", (tokens, text.hidden)) logits = example("router_logits", (tokens, text.experts), torch.float32) - self.router_projection = UnquantizedProjection(example("router_source", source.shape, torch.float32), - weights.router, logits, session=session, name=name + ".router").functional(session=session) + # BF16 operands with the sum carried in FP32: each product of two BF16 values is exact in FP32. + self.router_projection = UnquantizedProjection(source, weights.router, logits, session=session, + name=name + ".router", accumulation="fp32").functional(session=session) routes = RouterScratch.allocate(tokens, device, experts=text.experts, top_k=text.experts_per_token) self.router = PreparedRouter(logits, routes, session=session, name=name + ".routing", score_func="sigmoid", renormalize=True, routed_scaling_factor=text.route_scale, correction_bias=weights.correction, @@ -100,7 +104,7 @@ def forward(self, state, metadata): mlp_input = self.mlp_mix((*state.blocks, state.partial, attention))[0] else: mlp_input = self.mlp_mix((*state.blocks, state.partial), addend=attention)[0] - routes = self.router(self.router_projection(mlp_input.float()), num_valid_tokens=m.live_counts[:1]) + routes = self.router(self.router_projection(mlp_input), num_valid_tokens=m.live_counts[:1]) expert_partial = self.experts(mlp_input, routes.ids, routes.weights) mlp_raw = self.reduction(expert_partial.to(self.reduction_dtype)).to(torch.bfloat16) mlp, mlp_previous = self.mlp_convolution(mlp_raw, m) diff --git a/ignition/recipes/inkling2/residual.py b/ignition/recipes/inkling2/residual.py index 2081eece3..b3b748fa3 100644 --- a/ignition/recipes/inkling2/residual.py +++ b/ignition/recipes/inkling2/residual.py @@ -1,24 +1,31 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 Flare community contributors # SPDX-License-Identifier: Apache-2.0 -"""Prepared FP32 attention-residual mixtures over normalized source values.""" +"""Prepared attention-residual mixtures over normalized source values, fused with their RMSNorm. + +One program per row mixes the sources in FP32 and normalizes the mixture without storing it: the mean +square, the product with the reciprocal root and then with the norm weight are FP32, and only the +result rounds to BF16. That is the arithmetic of the mixture followed by `PreparedRMSNorm`, whose +tuned plans differ from one another only in the summation order of the mean square, so the fused +result differs from any of them by at most one BF16 unit in a few elements per million. +""" import torch import triton import triton.language as tl -from ignition.components.rms_norm import PreparedRMSNorm from ignition.preparation.examples import declaration_tensor from ignition.preparation.triton import launch_triton, prepare_triton @triton.jit def _mixture(Source0, Source1, Source2, Source3, Source4, Source5, Source6, Source7, Source8, - Queries, Output0, Output1, Output2, Addend, STRIDES: tl.constexpr, HIDDEN: tl.constexpr, - SOURCES: tl.constexpr, STREAMS: tl.constexpr, ADD: tl.constexpr, ADD_STRIDE: tl.constexpr, - EPS: tl.constexpr, BLOCK: tl.constexpr): + Queries, Output0, Output1, Output2, Addend, Norm0, Norm1, Norm2, STRIDES: tl.constexpr, + HIDDEN: tl.constexpr, SOURCES: tl.constexpr, STREAMS: tl.constexpr, ADD: tl.constexpr, + ADD_STRIDE: tl.constexpr, EPS: tl.constexpr, NORMALIZE: tl.constexpr, BLOCK: tl.constexpr): Sources = (Source0, Source1, Source2, Source3, Source4, Source5, Source6, Source7, Source8) Outputs = (Output0, Output1, Output2) + Norms = (Norm0, Norm1, Norm2) row = tl.program_id(0).to(tl.int64) columns = tl.arange(0, BLOCK) valid = columns < HIDDEN @@ -39,47 +46,72 @@ def _mixture(Source0, Source1, Source2, Source3, Source4, Source5, Source6, Sour denominator = denominator * rescale + probability mixture = mixture * rescale + value * (inverse * probability) maximum = peak - tl.store(Outputs[stream] + row * HIDDEN + columns, mixture / denominator, valid) + mixed = mixture / denominator + if NORMALIZE: + # The mixture is scaled before the weight is applied, as the unfused RMSNorm does. + scale = tl.rsqrt(tl.sum(mixed * mixed, 0) / HIDDEN + EPS) + weight = tl.load(Norms[stream] + columns, valid, other=0.).to(tl.float32) + tl.store(Outputs[stream] + row * HIDDEN + columns, (mixed * scale * weight).to(tl.bfloat16), valid) + else: + tl.store(Outputs[stream] + row * HIDDEN + columns, mixed, valid) class PreparedResidualMix: + """Normalized attention-residual mixtures of one to nine BF16 sources for one to three output streams. + + Calling the object returns each stream's BF16 `RMSNorm(mixture) * norm` from one kernel launch. + `mix` returns the FP32 mixtures alone, which the fused kernel never stores; tests compare the two. + """ + def __init__(self, source, queries, norms, *, sources, eps, session, name, addend=False, examples=None): + del session, name if not 1 <= sources <= 9 or not 1 <= len(norms) <= 3: raise ValueError('attention-residual mixtures require one to nine sources and one to three output streams') if queries.dtype != torch.float32 or queries.shape != (len(norms), source.shape[1]): raise ValueError('attention-residual queries must contain one FP32 row per output stream') - self.queries, self.norms, self.sources = queries, norms, sources + if any(norm.shape != source.shape[1:] or not norm.is_contiguous() + or norm.dtype not in (torch.bfloat16, torch.float32) for norm in norms): + raise ValueError('attention-residual norm weights must be contiguous BF16 or FP32 rows of the hidden width') + self.queries, self.norms, self.sources = queries, tuple(norms), sources self.eps, self.streams, self.addend = eps, len(norms), addend self.tokens, self.hidden = source.shape self.options = dict(HIDDEN=self.hidden, SOURCES=sources, STREAMS=self.streams, ADD=addend, ADD_STRIDE=source.stride(0), EPS=eps, BLOCK=triton.next_power_of_2(self.hidden), STRIDES=(source.stride(0),) * sources, num_warps=8, enable_fp_fusion=False) declarations = {} if examples is None else examples - mixtures = tuple(declaration_tensor(declarations, f"inkling.residual.mixture{index}", - tuple(source.shape), torch.float32, source.device) for index in range(len(norms))) outputs = tuple(declaration_tensor(declarations, f"inkling.residual.output{index}", tuple(source.shape), torch.bfloat16, source.device) for index in range(len(norms))) - arguments = ((source,) * 9 + (queries,) + mixtures - + (mixtures[0],) * (3 - self.streams) + (source,)) - self.compiled = prepare_triton(_mixture, arguments, (self.tokens, 1, 1), **self.options).compiled - self.normalizers = tuple(PreparedRMSNorm(value, norm, output, - eps=eps, session=session, name=f'{name}.norm{index}').functional() - for index, (value, norm, output) in enumerate(zip(mixtures, norms, outputs, strict=True))) + self.compiled = prepare_triton(_mixture, self._arguments((source,) * sources, outputs, source), + (self.tokens, 1, 1), NORMALIZE=True, **self.options).compiled + self.unnormalized = None - def mix(self, sources, *, addend=None): + def _arguments(self, sources, outputs, addend): + return (sources + (sources[0],) * (9 - self.sources) + (self.queries,) + outputs + + (outputs[0],) * (3 - self.streams) + (addend,) + + self.norms + (self.norms[0],) * (3 - self.streams)) + + def _check(self, sources, addend): if len(sources) != self.sources: raise ValueError('attention-residual source count differs from the prepared layer') - output = tuple(torch.empty((self.tokens, self.hidden), dtype=torch.float32, - device=sources[0].device) for _ in range(self.streams)) if (addend is not None) != self.addend: raise ValueError('attention-residual addend differs from the prepared layer') - options = self.options - arguments = (sources + (sources[0],) * (9 - self.sources) + (self.queries,) + output - + (output[0],) * (3 - self.streams) + (sources[0] if addend is None else addend,)) - launch_triton(_mixture, self.compiled, - arguments, (self.tokens, 1, 1), **options) + + def mix(self, sources, *, addend=None): + """The FP32 mixtures without normalization.""" + self._check(sources, addend) + output = tuple(torch.empty((self.tokens, self.hidden), dtype=torch.float32, + device=sources[0].device) for _ in range(self.streams)) + arguments = self._arguments(sources, output, sources[0] if addend is None else addend) + if self.unnormalized is None: + self.unnormalized = prepare_triton(_mixture, arguments, (self.tokens, 1, 1), NORMALIZE=False, + **self.options).compiled + launch_triton(_mixture, self.unnormalized, arguments, (self.tokens, 1, 1), NORMALIZE=False, **self.options) return output def __call__(self, sources, *, addend=None): - return tuple(norm(value) for norm, value in zip(self.normalizers, - self.mix(sources, addend=addend), strict=True)) + self._check(sources, addend) + output = tuple(torch.empty((self.tokens, self.hidden), dtype=torch.bfloat16, + device=sources[0].device) for _ in range(self.streams)) + launch_triton(_mixture, self.compiled, self._arguments(sources, output, sources[0] if addend is None else addend), + (self.tokens, 1, 1), NORMALIZE=True, **self.options) + return output diff --git a/ignition/recipes/inkling2/weights.py b/ignition/recipes/inkling2/weights.py index f30ff6fd5..7a851c2d1 100644 --- a/ignition/recipes/inkling2/weights.py +++ b/ignition/recipes/inkling2/weights.py @@ -15,6 +15,14 @@ from .config import ModelConfig, TEXT_PREFIX +def load_exact_bfloat16(loader, name): + """An FP32 checkpoint tensor whose values are exact BF16 upcasts, delivered as BF16.""" + narrowed = loader.load_tensor(name, dtype=torch.bfloat16) + if not torch.equal(narrowed.float(), loader.load_tensor(name, temporary=True)): + raise ValueError(f"{name}: FP32 values are not exact BF16 upcasts, so BF16 delivery would round them") + return narrowed + + def scaled_query(query, hidden): return persistent_copy(query.float().reshape(-1, hidden) * hidden ** -0.5) @@ -160,7 +168,7 @@ def load_layer(self, index): attention_query = None if index == 0 else scaled_query( self.load_tensor(prefix + ".attention_input.query"), c.hidden) return LayerWeights(attention, load_experts(self, index, shard=s), - self.load_tensor(prefix + ".mlp.gate.weight"), + load_exact_bfloat16(self, prefix + ".mlp.gate.weight"), self.load_tensor(prefix + ".mlp.gate.e_score_correction_bias"), self.load_tensor(prefix + ".post_attention_convolution.conv1d.weight"), self.load_tensor(prefix + ".post_mlp_convolution.conv1d.weight"), norms, diff --git a/tests/recipes/test_inkling2_target.py b/tests/recipes/test_inkling2_target.py index 1977c63a1..fb0d9fee9 100644 --- a/tests/recipes/test_inkling2_target.py +++ b/tests/recipes/test_inkling2_target.py @@ -224,7 +224,6 @@ def test_prepared_attention_residual_reference_and_capture(tokens, streams, sour workload_schema='inkling2-residual', numerical_policy='default', autotune_effort='off') as session: operation = PreparedResidualMix(values[0], queries, norms, sources=sources, eps=1e-6, session=session, name='residual', addend=add) - session.tune() reference = residual_mix(values, queries, addend=addend) mixture = operation.mix(values, addend=addend) for actual, expected in zip(mixture, reference, strict=True): @@ -234,11 +233,11 @@ def test_prepared_attention_residual_reference_and_capture(tokens, streams, sour for got, want in zip(actual, expected, strict=True): torch.testing.assert_close(got, want, rtol=2 ** -7, atol=2e-5) graph = torch.cuda.CUDAGraph(keep_graph=True) - with session.capture(), torch.cuda.graph(graph): + with torch.cuda.graph(graph): outputs = operation(values, addend=addend) status, _, count = driver.cuGraphGetNodes(graph.raw_cuda_graph()) assert status == driver.CUresult.CUDA_SUCCESS - assert count == 1 + streams + assert count == 1 for _ in range(3): before = torch.cuda.memory_stats()['allocation.all.allocated'] graph.replay() @@ -276,7 +275,6 @@ def test_prepared_attention_residual_aot_restore_and_replay(tmp_path, tokens, st workload_schema='inkling2-residual-aot', numerical_policy='default') as session: operation = PreparedResidualMix(values[0], queries, norms, sources=sources, eps=1e-6, session=session, name='residual', addend=add) - session.tune() forward = _ResidualForward(operation, values, addend) expected = forward() compiled = aot_forward(forward) @@ -289,9 +287,8 @@ def test_prepared_attention_residual_aot_restore_and_replay(tmp_path, tokens, st restored = torch.compiler.load_compiled_function(file) restored(forward) graph = torch.cuda.CUDAGraph() - with session.capture(), torch.cuda.graph(graph): + with torch.cuda.graph(graph): outputs = restored(forward) - session.freeze() for _ in range(3): for value in values: value.normal_() @@ -384,3 +381,42 @@ def forward(): assert (((gated.double() - expected_gate).abs() - bound).max() <= 0).item() torch.testing.assert_close(normalized, expected_norm, rtol=2 ** -7, atol=2 ** -126) graph.reset() + + +@pytest.mark.skipif(os.environ.get('FLARE_RUN_CUDA_TESTS') != '1', reason='requires an assigned SM120 GPU') +@pytest.mark.parametrize('effort', ['off', 'default']) +@pytest.mark.parametrize('tokens,streams,sources,add', [(4096, 1, 5, True), (4096, 3, 9, False), (128, 1, 2, False)]) +@torch.no_grad() +def test_fused_residual_norm_stays_within_the_norm_plans_variation(effort, tokens, streams, sources, add): + """The fused kernel against the FP32 mixture followed by `PreparedRMSNorm`. + + The two differ only in the summation order of the mean square, as the norm's own tuned plans do + (85 of 37.7 million elements between two plans on a GB10): no element differs by more than one + BF16 unit, and at most ten elements per million differ. + """ + from ignition.components.rms_norm import PreparedRMSNorm + from ignition.preparation.tuning import RecordedPreparationSession + from ignition.recipes.inkling2.residual import PreparedResidualMix + + torch.manual_seed(23) + device = torch.device('cuda', 0) + values = tuple(torch.randn(tokens, 4608, device=device).bfloat16() for _ in range(sources)) + queries = torch.randn(streams, 4608, device=device) * 4608 ** -.5 + norms = tuple((1 + .1 * torch.randn(4608, device=device)).bfloat16() for _ in range(streams)) + addend = (.3 * torch.randn_like(values[-1].float())).bfloat16() if add else None + with RecordedPreparationSession(device=device, checkpoint='inkling2-residual-bound', + workload_schema='inkling2-residual-bound', numerical_policy='default', autotune_effort=effort) as session: + operation = PreparedResidualMix(values[0], queries, norms, sources=sources, eps=1e-6, + session=session, name='residual', addend=add) + mixture = torch.empty((tokens, 4608), dtype=torch.float32, device=device) + unfused = [PreparedRMSNorm(mixture, norm, torch.empty((tokens, 4608), dtype=torch.bfloat16, device=device), + eps=1e-6, session=session, name=f'norm{index}') for index, norm in enumerate(norms)] + session.tune() + fused = operation(values, addend=addend) + differing = 0 + for got, value, norm in zip(fused, operation.mix(values, addend=addend), unfused, strict=True): + want = norm(value) + units = (got.view(torch.int16).int() - want.view(torch.int16).int()).abs() + assert int(units.max()) <= 1 + differing += int((units != 0).sum()) + assert differing <= max(1, streams * tokens * 4608 // 100_000)