diff --git a/ignition/components/linear.py b/ignition/components/linear.py index ff92a995d..48b6d495b 100644 --- a/ignition/components/linear.py +++ b/ignition/components/linear.py @@ -144,7 +144,13 @@ def __init__( session: PreparationSession, name: str, bias: torch.Tensor | None = None, + accumulation: str = "any", ) -> None: + """Declare `source @ weight.T` (plus `bias`) into `output`. + + `accumulation="fp32"` admits only b12x plans that carry the whole sum of products in FP32, the + accuracy of the product of FP32 operands; `"any"` also admits tensor-core and library accumulation. + """ if any(t.dtype not in (torch.bfloat16, torch.float32) for t in (source, weight, output)): raise TypeError("unquantized projections require BF16 or FP32 operands") for tensor in (weight, bias): @@ -152,7 +158,7 @@ def __init__( record_numeric_tensor(tensor) # FlashInfer's GEMMs read and write contiguous rows; column slices of wider rows keep b12x's GEMV. rows = source.numel() // source.shape[-1] - self.dense = (sm103_projection.is_sm103(source.device) and bias is None + self.dense = (sm103_projection.is_sm103(source.device) and bias is None and accumulation == "any" and source.dtype == weight.dtype == torch.bfloat16 and source.is_contiguous() and output.is_contiguous() and sm103_projection.dense_serves(rows, output.shape[-1])) @@ -169,7 +175,8 @@ def __init__( session=session), weight=weight, bias=None)),)) return - query = bf16_gemv.query_from_call(source, weight, out=output, bias=bias) + query = bf16_gemv.query_from_call(source, weight, out=output, bias=bias, + **({} if accumulation == "any" else {"accumulation": accumulation})) self.plan = bf16_gemv.plan(query) self.weight = weight self.bias = bias diff --git a/ignition/recipes/inkling2/attention.py b/ignition/recipes/inkling2/attention.py index 4b430ddfa..e0c885333 100644 --- a/ignition/recipes/inkling2/attention.py +++ b/ignition/recipes/inkling2/attention.py @@ -8,8 +8,9 @@ or convolution histories. Accepted counts commit those histories afterward. Full-attention candidates write private request pages. Attention uses BF16 queries and relative-bias operands, FP8 K/V and FP32 row sinks. The gated -BF16 result projects to FP32 partial sums for cross-rank reduction before the -replicated BF16 output convolution. +BF16 result projects to partial sums of the cross-rank reduction's element type +(FP32 on one host, BF16 over RoCEnante, where the projection rounds its FP32 +accumulation once) before the replicated BF16 output convolution. """ from __future__ import annotations @@ -173,7 +174,7 @@ class PreparedAttention(torch.nn.Module): """A rank's attention for prefill, decode or speculative verification.""" def __init__(self, layer, weights: AttentionWeights, text, cache: LayerCache, metadata, *, mode, tokens, - session, workspace, name, examples=None, tuning_page_count=None): + session, workspace, name, examples=None, tuning_page_count=None, partial_dtype=torch.float32): super().__init__() if mode not in ("prefill", "decode", "verify"): raise ValueError(f"unsupported Inkling attention mode: {mode}") @@ -201,7 +202,7 @@ def example(role, shape, dtype=torch.bfloat16): for index, (weight, width) in enumerate(zip(weights.projections, self.columns.widths(), strict=True))) attended = example("inkling.attention.attended", (tokens, self.heads * self.dim)) self.output_projection = UnquantizedProjection(attended, weights.output, - example("inkling.attention.partial", (tokens, text.hidden), torch.float32), + example("inkling.attention.partial", (tokens, text.hidden), partial_dtype), session=session, name=f"{name}.output").functional(session=session) query = example("inkling.attention.query", (tokens, self.heads, self.dim)) output = example("inkling.attention.output", (tokens, self.heads, self.dim)) diff --git a/ignition/recipes/inkling2/decoder.py b/ignition/recipes/inkling2/decoder.py index 89e6241b5..d86763cd4 100644 --- a/ignition/recipes/inkling2/decoder.py +++ b/ignition/recipes/inkling2/decoder.py @@ -59,6 +59,9 @@ def __init__(self, layer, weights, text, pool, metadata, reduction, reduction_dt self.reduction, self.reduction_dtype = reduction, reduction_dtype self.attention = PreparedAttention(layer, weights.attention, text, LayerCache.of(pool, layer), metadata, mode=mode, tokens=tokens, session=session, workspace=workspace, name=name + ".attention", examples=examples, + # Prefill writes the attention partial sum in the reduction's element type. At prefill row counts the + # BF16 projection equals the FP32 projection rounded once; decode and verification keep FP32. + partial_dtype=reduction_dtype if mode == "prefill" else torch.float32, tuning_page_count=((int(pool.memory_capacity[0].item()) - 1) * pool.ring_row_pages + pool.geometry.ring // 128 if text.local(layer) else int(pool.memory_capacity[1].item()))) device = weights.router.device @@ -66,8 +69,9 @@ def example(role, shape, dtype=torch.bfloat16): return declaration_tensor(examples, "inkling.mlp." + role, shape, dtype, device) source = example("source", (tokens, text.hidden)) logits = example("router_logits", (tokens, text.experts), torch.float32) - self.router_projection = UnquantizedProjection(example("router_source", source.shape, torch.float32), - weights.router, logits, session=session, name=name + ".router").functional(session=session) + # BF16 operands with the sum carried in FP32: each product of two BF16 values is exact in FP32. + self.router_projection = UnquantizedProjection(source, weights.router, logits, session=session, + name=name + ".router", accumulation="fp32").functional(session=session) routes = RouterScratch.allocate(tokens, device, experts=text.experts, top_k=text.experts_per_token) self.router = PreparedRouter(logits, routes, session=session, name=name + ".routing", score_func="sigmoid", renormalize=True, routed_scaling_factor=text.route_scale, correction_bias=weights.correction, @@ -100,7 +104,7 @@ def forward(self, state, metadata): mlp_input = self.mlp_mix((*state.blocks, state.partial, attention))[0] else: mlp_input = self.mlp_mix((*state.blocks, state.partial), addend=attention)[0] - routes = self.router(self.router_projection(mlp_input.float()), num_valid_tokens=m.live_counts[:1]) + routes = self.router(self.router_projection(mlp_input), num_valid_tokens=m.live_counts[:1]) expert_partial = self.experts(mlp_input, routes.ids, routes.weights) mlp_raw = self.reduction(expert_partial.to(self.reduction_dtype)).to(torch.bfloat16) mlp, mlp_previous = self.mlp_convolution(mlp_raw, m) diff --git a/ignition/recipes/inkling2/residual.py b/ignition/recipes/inkling2/residual.py index 2081eece3..b3b748fa3 100644 --- a/ignition/recipes/inkling2/residual.py +++ b/ignition/recipes/inkling2/residual.py @@ -1,24 +1,31 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 Flare community contributors # SPDX-License-Identifier: Apache-2.0 -"""Prepared FP32 attention-residual mixtures over normalized source values.""" +"""Prepared attention-residual mixtures over normalized source values, fused with their RMSNorm. + +One program per row mixes the sources in FP32 and normalizes the mixture without storing it: the mean +square, the product with the reciprocal root and then with the norm weight are FP32, and only the +result rounds to BF16. That is the arithmetic of the mixture followed by `PreparedRMSNorm`, whose +tuned plans differ from one another only in the summation order of the mean square, so the fused +result differs from any of them by at most one BF16 unit in a few elements per million. +""" import torch import triton import triton.language as tl -from ignition.components.rms_norm import PreparedRMSNorm from ignition.preparation.examples import declaration_tensor from ignition.preparation.triton import launch_triton, prepare_triton @triton.jit def _mixture(Source0, Source1, Source2, Source3, Source4, Source5, Source6, Source7, Source8, - Queries, Output0, Output1, Output2, Addend, STRIDES: tl.constexpr, HIDDEN: tl.constexpr, - SOURCES: tl.constexpr, STREAMS: tl.constexpr, ADD: tl.constexpr, ADD_STRIDE: tl.constexpr, - EPS: tl.constexpr, BLOCK: tl.constexpr): + Queries, Output0, Output1, Output2, Addend, Norm0, Norm1, Norm2, STRIDES: tl.constexpr, + HIDDEN: tl.constexpr, SOURCES: tl.constexpr, STREAMS: tl.constexpr, ADD: tl.constexpr, + ADD_STRIDE: tl.constexpr, EPS: tl.constexpr, NORMALIZE: tl.constexpr, BLOCK: tl.constexpr): Sources = (Source0, Source1, Source2, Source3, Source4, Source5, Source6, Source7, Source8) Outputs = (Output0, Output1, Output2) + Norms = (Norm0, Norm1, Norm2) row = tl.program_id(0).to(tl.int64) columns = tl.arange(0, BLOCK) valid = columns < HIDDEN @@ -39,47 +46,72 @@ def _mixture(Source0, Source1, Source2, Source3, Source4, Source5, Source6, Sour denominator = denominator * rescale + probability mixture = mixture * rescale + value * (inverse * probability) maximum = peak - tl.store(Outputs[stream] + row * HIDDEN + columns, mixture / denominator, valid) + mixed = mixture / denominator + if NORMALIZE: + # The mixture is scaled before the weight is applied, as the unfused RMSNorm does. + scale = tl.rsqrt(tl.sum(mixed * mixed, 0) / HIDDEN + EPS) + weight = tl.load(Norms[stream] + columns, valid, other=0.).to(tl.float32) + tl.store(Outputs[stream] + row * HIDDEN + columns, (mixed * scale * weight).to(tl.bfloat16), valid) + else: + tl.store(Outputs[stream] + row * HIDDEN + columns, mixed, valid) class PreparedResidualMix: + """Normalized attention-residual mixtures of one to nine BF16 sources for one to three output streams. + + Calling the object returns each stream's BF16 `RMSNorm(mixture) * norm` from one kernel launch. + `mix` returns the FP32 mixtures alone, which the fused kernel never stores; tests compare the two. + """ + def __init__(self, source, queries, norms, *, sources, eps, session, name, addend=False, examples=None): + del session, name if not 1 <= sources <= 9 or not 1 <= len(norms) <= 3: raise ValueError('attention-residual mixtures require one to nine sources and one to three output streams') if queries.dtype != torch.float32 or queries.shape != (len(norms), source.shape[1]): raise ValueError('attention-residual queries must contain one FP32 row per output stream') - self.queries, self.norms, self.sources = queries, norms, sources + if any(norm.shape != source.shape[1:] or not norm.is_contiguous() + or norm.dtype not in (torch.bfloat16, torch.float32) for norm in norms): + raise ValueError('attention-residual norm weights must be contiguous BF16 or FP32 rows of the hidden width') + self.queries, self.norms, self.sources = queries, tuple(norms), sources self.eps, self.streams, self.addend = eps, len(norms), addend self.tokens, self.hidden = source.shape self.options = dict(HIDDEN=self.hidden, SOURCES=sources, STREAMS=self.streams, ADD=addend, ADD_STRIDE=source.stride(0), EPS=eps, BLOCK=triton.next_power_of_2(self.hidden), STRIDES=(source.stride(0),) * sources, num_warps=8, enable_fp_fusion=False) declarations = {} if examples is None else examples - mixtures = tuple(declaration_tensor(declarations, f"inkling.residual.mixture{index}", - tuple(source.shape), torch.float32, source.device) for index in range(len(norms))) outputs = tuple(declaration_tensor(declarations, f"inkling.residual.output{index}", tuple(source.shape), torch.bfloat16, source.device) for index in range(len(norms))) - arguments = ((source,) * 9 + (queries,) + mixtures - + (mixtures[0],) * (3 - self.streams) + (source,)) - self.compiled = prepare_triton(_mixture, arguments, (self.tokens, 1, 1), **self.options).compiled - self.normalizers = tuple(PreparedRMSNorm(value, norm, output, - eps=eps, session=session, name=f'{name}.norm{index}').functional() - for index, (value, norm, output) in enumerate(zip(mixtures, norms, outputs, strict=True))) + self.compiled = prepare_triton(_mixture, self._arguments((source,) * sources, outputs, source), + (self.tokens, 1, 1), NORMALIZE=True, **self.options).compiled + self.unnormalized = None - def mix(self, sources, *, addend=None): + def _arguments(self, sources, outputs, addend): + return (sources + (sources[0],) * (9 - self.sources) + (self.queries,) + outputs + + (outputs[0],) * (3 - self.streams) + (addend,) + + self.norms + (self.norms[0],) * (3 - self.streams)) + + def _check(self, sources, addend): if len(sources) != self.sources: raise ValueError('attention-residual source count differs from the prepared layer') - output = tuple(torch.empty((self.tokens, self.hidden), dtype=torch.float32, - device=sources[0].device) for _ in range(self.streams)) if (addend is not None) != self.addend: raise ValueError('attention-residual addend differs from the prepared layer') - options = self.options - arguments = (sources + (sources[0],) * (9 - self.sources) + (self.queries,) + output - + (output[0],) * (3 - self.streams) + (sources[0] if addend is None else addend,)) - launch_triton(_mixture, self.compiled, - arguments, (self.tokens, 1, 1), **options) + + def mix(self, sources, *, addend=None): + """The FP32 mixtures without normalization.""" + self._check(sources, addend) + output = tuple(torch.empty((self.tokens, self.hidden), dtype=torch.float32, + device=sources[0].device) for _ in range(self.streams)) + arguments = self._arguments(sources, output, sources[0] if addend is None else addend) + if self.unnormalized is None: + self.unnormalized = prepare_triton(_mixture, arguments, (self.tokens, 1, 1), NORMALIZE=False, + **self.options).compiled + launch_triton(_mixture, self.unnormalized, arguments, (self.tokens, 1, 1), NORMALIZE=False, **self.options) return output def __call__(self, sources, *, addend=None): - return tuple(norm(value) for norm, value in zip(self.normalizers, - self.mix(sources, addend=addend), strict=True)) + self._check(sources, addend) + output = tuple(torch.empty((self.tokens, self.hidden), dtype=torch.bfloat16, + device=sources[0].device) for _ in range(self.streams)) + launch_triton(_mixture, self.compiled, self._arguments(sources, output, sources[0] if addend is None else addend), + (self.tokens, 1, 1), NORMALIZE=True, **self.options) + return output diff --git a/ignition/recipes/inkling2/weights.py b/ignition/recipes/inkling2/weights.py index f30ff6fd5..7a851c2d1 100644 --- a/ignition/recipes/inkling2/weights.py +++ b/ignition/recipes/inkling2/weights.py @@ -15,6 +15,14 @@ from .config import ModelConfig, TEXT_PREFIX +def load_exact_bfloat16(loader, name): + """An FP32 checkpoint tensor whose values are exact BF16 upcasts, delivered as BF16.""" + narrowed = loader.load_tensor(name, dtype=torch.bfloat16) + if not torch.equal(narrowed.float(), loader.load_tensor(name, temporary=True)): + raise ValueError(f"{name}: FP32 values are not exact BF16 upcasts, so BF16 delivery would round them") + return narrowed + + def scaled_query(query, hidden): return persistent_copy(query.float().reshape(-1, hidden) * hidden ** -0.5) @@ -160,7 +168,7 @@ def load_layer(self, index): attention_query = None if index == 0 else scaled_query( self.load_tensor(prefix + ".attention_input.query"), c.hidden) return LayerWeights(attention, load_experts(self, index, shard=s), - self.load_tensor(prefix + ".mlp.gate.weight"), + load_exact_bfloat16(self, prefix + ".mlp.gate.weight"), self.load_tensor(prefix + ".mlp.gate.e_score_correction_bias"), self.load_tensor(prefix + ".post_attention_convolution.conv1d.weight"), self.load_tensor(prefix + ".post_mlp_convolution.conv1d.weight"), norms, diff --git a/tests/recipes/test_inkling2_target.py b/tests/recipes/test_inkling2_target.py index 1977c63a1..fb0d9fee9 100644 --- a/tests/recipes/test_inkling2_target.py +++ b/tests/recipes/test_inkling2_target.py @@ -224,7 +224,6 @@ def test_prepared_attention_residual_reference_and_capture(tokens, streams, sour workload_schema='inkling2-residual', numerical_policy='default', autotune_effort='off') as session: operation = PreparedResidualMix(values[0], queries, norms, sources=sources, eps=1e-6, session=session, name='residual', addend=add) - session.tune() reference = residual_mix(values, queries, addend=addend) mixture = operation.mix(values, addend=addend) for actual, expected in zip(mixture, reference, strict=True): @@ -234,11 +233,11 @@ def test_prepared_attention_residual_reference_and_capture(tokens, streams, sour for got, want in zip(actual, expected, strict=True): torch.testing.assert_close(got, want, rtol=2 ** -7, atol=2e-5) graph = torch.cuda.CUDAGraph(keep_graph=True) - with session.capture(), torch.cuda.graph(graph): + with torch.cuda.graph(graph): outputs = operation(values, addend=addend) status, _, count = driver.cuGraphGetNodes(graph.raw_cuda_graph()) assert status == driver.CUresult.CUDA_SUCCESS - assert count == 1 + streams + assert count == 1 for _ in range(3): before = torch.cuda.memory_stats()['allocation.all.allocated'] graph.replay() @@ -276,7 +275,6 @@ def test_prepared_attention_residual_aot_restore_and_replay(tmp_path, tokens, st workload_schema='inkling2-residual-aot', numerical_policy='default') as session: operation = PreparedResidualMix(values[0], queries, norms, sources=sources, eps=1e-6, session=session, name='residual', addend=add) - session.tune() forward = _ResidualForward(operation, values, addend) expected = forward() compiled = aot_forward(forward) @@ -289,9 +287,8 @@ def test_prepared_attention_residual_aot_restore_and_replay(tmp_path, tokens, st restored = torch.compiler.load_compiled_function(file) restored(forward) graph = torch.cuda.CUDAGraph() - with session.capture(), torch.cuda.graph(graph): + with torch.cuda.graph(graph): outputs = restored(forward) - session.freeze() for _ in range(3): for value in values: value.normal_() @@ -384,3 +381,42 @@ def forward(): assert (((gated.double() - expected_gate).abs() - bound).max() <= 0).item() torch.testing.assert_close(normalized, expected_norm, rtol=2 ** -7, atol=2 ** -126) graph.reset() + + +@pytest.mark.skipif(os.environ.get('FLARE_RUN_CUDA_TESTS') != '1', reason='requires an assigned SM120 GPU') +@pytest.mark.parametrize('effort', ['off', 'default']) +@pytest.mark.parametrize('tokens,streams,sources,add', [(4096, 1, 5, True), (4096, 3, 9, False), (128, 1, 2, False)]) +@torch.no_grad() +def test_fused_residual_norm_stays_within_the_norm_plans_variation(effort, tokens, streams, sources, add): + """The fused kernel against the FP32 mixture followed by `PreparedRMSNorm`. + + The two differ only in the summation order of the mean square, as the norm's own tuned plans do + (85 of 37.7 million elements between two plans on a GB10): no element differs by more than one + BF16 unit, and at most ten elements per million differ. + """ + from ignition.components.rms_norm import PreparedRMSNorm + from ignition.preparation.tuning import RecordedPreparationSession + from ignition.recipes.inkling2.residual import PreparedResidualMix + + torch.manual_seed(23) + device = torch.device('cuda', 0) + values = tuple(torch.randn(tokens, 4608, device=device).bfloat16() for _ in range(sources)) + queries = torch.randn(streams, 4608, device=device) * 4608 ** -.5 + norms = tuple((1 + .1 * torch.randn(4608, device=device)).bfloat16() for _ in range(streams)) + addend = (.3 * torch.randn_like(values[-1].float())).bfloat16() if add else None + with RecordedPreparationSession(device=device, checkpoint='inkling2-residual-bound', + workload_schema='inkling2-residual-bound', numerical_policy='default', autotune_effort=effort) as session: + operation = PreparedResidualMix(values[0], queries, norms, sources=sources, eps=1e-6, + session=session, name='residual', addend=add) + mixture = torch.empty((tokens, 4608), dtype=torch.float32, device=device) + unfused = [PreparedRMSNorm(mixture, norm, torch.empty((tokens, 4608), dtype=torch.bfloat16, device=device), + eps=1e-6, session=session, name=f'norm{index}') for index, norm in enumerate(norms)] + session.tune() + fused = operation(values, addend=addend) + differing = 0 + for got, value, norm in zip(fused, operation.mix(values, addend=addend), unfused, strict=True): + want = norm(value) + units = (got.view(torch.int16).int() - want.view(torch.int16).int()).abs() + assert int(units.max()) <= 1 + differing += int((units != 0).sum()) + assert differing <= max(1, streams * tokens * 4608 // 100_000)