Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions ignition/components/linear.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,15 +144,21 @@ def __init__(
session: PreparationSession,
name: str,
bias: torch.Tensor | None = None,
accumulation: str = "any",
) -> None:
"""Declare `source @ weight.T` (plus `bias`) into `output`.

`accumulation="fp32"` admits only b12x plans that carry the whole sum of products in FP32, the
accuracy of the product of FP32 operands; `"any"` also admits tensor-core and library accumulation.
"""
if any(t.dtype not in (torch.bfloat16, torch.float32) for t in (source, weight, output)):
raise TypeError("unquantized projections require BF16 or FP32 operands")
for tensor in (weight, bias):
if tensor is not None:
record_numeric_tensor(tensor)
# FlashInfer's GEMMs read and write contiguous rows; column slices of wider rows keep b12x's GEMV.
rows = source.numel() // source.shape[-1]
self.dense = (sm103_projection.is_sm103(source.device) and bias is None
self.dense = (sm103_projection.is_sm103(source.device) and bias is None and accumulation == "any"
and source.dtype == weight.dtype == torch.bfloat16
and source.is_contiguous() and output.is_contiguous()
and sm103_projection.dense_serves(rows, output.shape[-1]))
Expand All @@ -169,7 +175,8 @@ def __init__(
session=session), weight=weight,
bias=None)),))
return
query = bf16_gemv.query_from_call(source, weight, out=output, bias=bias)
query = bf16_gemv.query_from_call(source, weight, out=output, bias=bias,
**({} if accumulation == "any" else {"accumulation": accumulation}))
self.plan = bf16_gemv.plan(query)
self.weight = weight
self.bias = bias
Expand Down
9 changes: 5 additions & 4 deletions ignition/recipes/inkling2/attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@
or convolution histories. Accepted counts commit those histories afterward.
Full-attention candidates write private request pages. Attention uses BF16
queries and relative-bias operands, FP8 K/V and FP32 row sinks. The gated
BF16 result projects to FP32 partial sums for cross-rank reduction before the
replicated BF16 output convolution.
BF16 result projects to partial sums of the cross-rank reduction's element type
(FP32 on one host, BF16 over RoCEnante, where the projection rounds its FP32
accumulation once) before the replicated BF16 output convolution.
"""

from __future__ import annotations
Expand Down Expand Up @@ -173,7 +174,7 @@ class PreparedAttention(torch.nn.Module):
"""A rank's attention for prefill, decode or speculative verification."""

def __init__(self, layer, weights: AttentionWeights, text, cache: LayerCache, metadata, *, mode, tokens,
session, workspace, name, examples=None, tuning_page_count=None):
session, workspace, name, examples=None, tuning_page_count=None, partial_dtype=torch.float32):
super().__init__()
if mode not in ("prefill", "decode", "verify"):
raise ValueError(f"unsupported Inkling attention mode: {mode}")
Expand Down Expand Up @@ -201,7 +202,7 @@ def example(role, shape, dtype=torch.bfloat16):
for index, (weight, width) in enumerate(zip(weights.projections, self.columns.widths(), strict=True)))
attended = example("inkling.attention.attended", (tokens, self.heads * self.dim))
self.output_projection = UnquantizedProjection(attended, weights.output,
example("inkling.attention.partial", (tokens, text.hidden), torch.float32),
example("inkling.attention.partial", (tokens, text.hidden), partial_dtype),
session=session, name=f"{name}.output").functional(session=session)
query = example("inkling.attention.query", (tokens, self.heads, self.dim))
output = example("inkling.attention.output", (tokens, self.heads, self.dim))
Expand Down
10 changes: 7 additions & 3 deletions ignition/recipes/inkling2/decoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,15 +59,19 @@ def __init__(self, layer, weights, text, pool, metadata, reduction, reduction_dt
self.reduction, self.reduction_dtype = reduction, reduction_dtype
self.attention = PreparedAttention(layer, weights.attention, text, LayerCache.of(pool, layer), metadata,
mode=mode, tokens=tokens, session=session, workspace=workspace, name=name + ".attention", examples=examples,
# Prefill writes the attention partial sum in the reduction's element type. At prefill row counts the
# BF16 projection equals the FP32 projection rounded once; decode and verification keep FP32.
partial_dtype=reduction_dtype if mode == "prefill" else torch.float32,
tuning_page_count=((int(pool.memory_capacity[0].item()) - 1) * pool.ring_row_pages + pool.geometry.ring // 128
if text.local(layer) else int(pool.memory_capacity[1].item())))
device = weights.router.device
def example(role, shape, dtype=torch.bfloat16):
return declaration_tensor(examples, "inkling.mlp." + role, shape, dtype, device)
source = example("source", (tokens, text.hidden))
logits = example("router_logits", (tokens, text.experts), torch.float32)
self.router_projection = UnquantizedProjection(example("router_source", source.shape, torch.float32),
weights.router, logits, session=session, name=name + ".router").functional(session=session)
# BF16 operands with the sum carried in FP32: each product of two BF16 values is exact in FP32.
self.router_projection = UnquantizedProjection(source, weights.router, logits, session=session,
name=name + ".router", accumulation="fp32").functional(session=session)
routes = RouterScratch.allocate(tokens, device, experts=text.experts, top_k=text.experts_per_token)
self.router = PreparedRouter(logits, routes, session=session, name=name + ".routing", score_func="sigmoid",
renormalize=True, routed_scaling_factor=text.route_scale, correction_bias=weights.correction,
Expand Down Expand Up @@ -100,7 +104,7 @@ def forward(self, state, metadata):
mlp_input = self.mlp_mix((*state.blocks, state.partial, attention))[0]
else:
mlp_input = self.mlp_mix((*state.blocks, state.partial), addend=attention)[0]
routes = self.router(self.router_projection(mlp_input.float()), num_valid_tokens=m.live_counts[:1])
routes = self.router(self.router_projection(mlp_input), num_valid_tokens=m.live_counts[:1])
expert_partial = self.experts(mlp_input, routes.ids, routes.weights)
mlp_raw = self.reduction(expert_partial.to(self.reduction_dtype)).to(torch.bfloat16)
mlp, mlp_previous = self.mlp_convolution(mlp_raw, m)
Expand Down
82 changes: 57 additions & 25 deletions ignition/recipes/inkling2/residual.py
Original file line number Diff line number Diff line change
@@ -1,24 +1,31 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 Flare community contributors
# SPDX-License-Identifier: Apache-2.0

"""Prepared FP32 attention-residual mixtures over normalized source values."""
"""Prepared attention-residual mixtures over normalized source values, fused with their RMSNorm.

One program per row mixes the sources in FP32 and normalizes the mixture without storing it: the mean
square, the product with the reciprocal root and then with the norm weight are FP32, and only the
result rounds to BF16. That is the arithmetic of the mixture followed by `PreparedRMSNorm`, whose
tuned plans differ from one another only in the summation order of the mean square, so the fused
result differs from any of them by at most one BF16 unit in a few elements per million.
"""

import torch
import triton
import triton.language as tl

from ignition.components.rms_norm import PreparedRMSNorm
from ignition.preparation.examples import declaration_tensor
from ignition.preparation.triton import launch_triton, prepare_triton


@triton.jit
def _mixture(Source0, Source1, Source2, Source3, Source4, Source5, Source6, Source7, Source8,
Queries, Output0, Output1, Output2, Addend, STRIDES: tl.constexpr, HIDDEN: tl.constexpr,
SOURCES: tl.constexpr, STREAMS: tl.constexpr, ADD: tl.constexpr, ADD_STRIDE: tl.constexpr,
EPS: tl.constexpr, BLOCK: tl.constexpr):
Queries, Output0, Output1, Output2, Addend, Norm0, Norm1, Norm2, STRIDES: tl.constexpr,
HIDDEN: tl.constexpr, SOURCES: tl.constexpr, STREAMS: tl.constexpr, ADD: tl.constexpr,
ADD_STRIDE: tl.constexpr, EPS: tl.constexpr, NORMALIZE: tl.constexpr, BLOCK: tl.constexpr):
Sources = (Source0, Source1, Source2, Source3, Source4, Source5, Source6, Source7, Source8)
Outputs = (Output0, Output1, Output2)
Norms = (Norm0, Norm1, Norm2)
row = tl.program_id(0).to(tl.int64)
columns = tl.arange(0, BLOCK)
valid = columns < HIDDEN
Expand All @@ -39,47 +46,72 @@ def _mixture(Source0, Source1, Source2, Source3, Source4, Source5, Source6, Sour
denominator = denominator * rescale + probability
mixture = mixture * rescale + value * (inverse * probability)
maximum = peak
tl.store(Outputs[stream] + row * HIDDEN + columns, mixture / denominator, valid)
mixed = mixture / denominator
if NORMALIZE:
# The mixture is scaled before the weight is applied, as the unfused RMSNorm does.
scale = tl.rsqrt(tl.sum(mixed * mixed, 0) / HIDDEN + EPS)
weight = tl.load(Norms[stream] + columns, valid, other=0.).to(tl.float32)
tl.store(Outputs[stream] + row * HIDDEN + columns, (mixed * scale * weight).to(tl.bfloat16), valid)
else:
tl.store(Outputs[stream] + row * HIDDEN + columns, mixed, valid)


class PreparedResidualMix:
"""Normalized attention-residual mixtures of one to nine BF16 sources for one to three output streams.

Calling the object returns each stream's BF16 `RMSNorm(mixture) * norm` from one kernel launch.
`mix` returns the FP32 mixtures alone, which the fused kernel never stores; tests compare the two.
"""

def __init__(self, source, queries, norms, *, sources, eps, session, name, addend=False, examples=None):
del session, name
if not 1 <= sources <= 9 or not 1 <= len(norms) <= 3:
raise ValueError('attention-residual mixtures require one to nine sources and one to three output streams')
if queries.dtype != torch.float32 or queries.shape != (len(norms), source.shape[1]):
raise ValueError('attention-residual queries must contain one FP32 row per output stream')
self.queries, self.norms, self.sources = queries, norms, sources
if any(norm.shape != source.shape[1:] or not norm.is_contiguous()
or norm.dtype not in (torch.bfloat16, torch.float32) for norm in norms):
raise ValueError('attention-residual norm weights must be contiguous BF16 or FP32 rows of the hidden width')
self.queries, self.norms, self.sources = queries, tuple(norms), sources
self.eps, self.streams, self.addend = eps, len(norms), addend
self.tokens, self.hidden = source.shape
self.options = dict(HIDDEN=self.hidden, SOURCES=sources, STREAMS=self.streams, ADD=addend,
ADD_STRIDE=source.stride(0), EPS=eps, BLOCK=triton.next_power_of_2(self.hidden),
STRIDES=(source.stride(0),) * sources, num_warps=8, enable_fp_fusion=False)
declarations = {} if examples is None else examples
mixtures = tuple(declaration_tensor(declarations, f"inkling.residual.mixture{index}",
tuple(source.shape), torch.float32, source.device) for index in range(len(norms)))
outputs = tuple(declaration_tensor(declarations, f"inkling.residual.output{index}",
tuple(source.shape), torch.bfloat16, source.device) for index in range(len(norms)))
arguments = ((source,) * 9 + (queries,) + mixtures
+ (mixtures[0],) * (3 - self.streams) + (source,))
self.compiled = prepare_triton(_mixture, arguments, (self.tokens, 1, 1), **self.options).compiled
self.normalizers = tuple(PreparedRMSNorm(value, norm, output,
eps=eps, session=session, name=f'{name}.norm{index}').functional()
for index, (value, norm, output) in enumerate(zip(mixtures, norms, outputs, strict=True)))
self.compiled = prepare_triton(_mixture, self._arguments((source,) * sources, outputs, source),
(self.tokens, 1, 1), NORMALIZE=True, **self.options).compiled
self.unnormalized = None

def mix(self, sources, *, addend=None):
def _arguments(self, sources, outputs, addend):
return (sources + (sources[0],) * (9 - self.sources) + (self.queries,) + outputs
+ (outputs[0],) * (3 - self.streams) + (addend,)
+ self.norms + (self.norms[0],) * (3 - self.streams))

def _check(self, sources, addend):
if len(sources) != self.sources:
raise ValueError('attention-residual source count differs from the prepared layer')
output = tuple(torch.empty((self.tokens, self.hidden), dtype=torch.float32,
device=sources[0].device) for _ in range(self.streams))
if (addend is not None) != self.addend:
raise ValueError('attention-residual addend differs from the prepared layer')
options = self.options
arguments = (sources + (sources[0],) * (9 - self.sources) + (self.queries,) + output
+ (output[0],) * (3 - self.streams) + (sources[0] if addend is None else addend,))
launch_triton(_mixture, self.compiled,
arguments, (self.tokens, 1, 1), **options)

def mix(self, sources, *, addend=None):
"""The FP32 mixtures without normalization."""
self._check(sources, addend)
output = tuple(torch.empty((self.tokens, self.hidden), dtype=torch.float32,
device=sources[0].device) for _ in range(self.streams))
arguments = self._arguments(sources, output, sources[0] if addend is None else addend)
if self.unnormalized is None:
self.unnormalized = prepare_triton(_mixture, arguments, (self.tokens, 1, 1), NORMALIZE=False,
**self.options).compiled
launch_triton(_mixture, self.unnormalized, arguments, (self.tokens, 1, 1), NORMALIZE=False, **self.options)
return output

def __call__(self, sources, *, addend=None):
return tuple(norm(value) for norm, value in zip(self.normalizers,
self.mix(sources, addend=addend), strict=True))
self._check(sources, addend)
output = tuple(torch.empty((self.tokens, self.hidden), dtype=torch.bfloat16,
device=sources[0].device) for _ in range(self.streams))
launch_triton(_mixture, self.compiled, self._arguments(sources, output, sources[0] if addend is None else addend),
(self.tokens, 1, 1), NORMALIZE=True, **self.options)
return output
10 changes: 9 additions & 1 deletion ignition/recipes/inkling2/weights.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,14 @@
from .config import ModelConfig, TEXT_PREFIX


def load_exact_bfloat16(loader, name):
"""An FP32 checkpoint tensor whose values are exact BF16 upcasts, delivered as BF16."""
narrowed = loader.load_tensor(name, dtype=torch.bfloat16)
if not torch.equal(narrowed.float(), loader.load_tensor(name, temporary=True)):
raise ValueError(f"{name}: FP32 values are not exact BF16 upcasts, so BF16 delivery would round them")
return narrowed


def scaled_query(query, hidden):
return persistent_copy(query.float().reshape(-1, hidden) * hidden ** -0.5)

Expand Down Expand Up @@ -160,7 +168,7 @@ def load_layer(self, index):
attention_query = None if index == 0 else scaled_query(
self.load_tensor(prefix + ".attention_input.query"), c.hidden)
return LayerWeights(attention, load_experts(self, index, shard=s),
self.load_tensor(prefix + ".mlp.gate.weight"),
load_exact_bfloat16(self, prefix + ".mlp.gate.weight"),
self.load_tensor(prefix + ".mlp.gate.e_score_correction_bias"),
self.load_tensor(prefix + ".post_attention_convolution.conv1d.weight"),
self.load_tensor(prefix + ".post_mlp_convolution.conv1d.weight"), norms,
Expand Down
Loading