From 22cc72886b761853b3808cd154d04819d476baf3 Mon Sep 17 00:00:00 2001 From: lichang Date: Thu, 12 Nov 2026 17:13:57 -0700 Subject: [PATCH 1/3] hip: add RDNA3.5 MMDQ prefill and quantized FFN fusions Assisted-by: Codex --- ggml/src/ggml-cuda/common.cuh | 7 + ggml/src/ggml-cuda/ggml-cuda.cu | 134 +++ ggml/src/ggml-cuda/mmdq.cu | 1228 +++++++++++++++++++++++++ ggml/src/ggml-cuda/mmdq.cuh | 39 + ggml/src/ggml-cuda/mmq-load-tiles.cuh | 14 +- ggml/src/ggml-cuda/mmq.cu | 12 +- ggml/src/ggml-cuda/mmq.cuh | 23 +- ggml/src/ggml-cuda/quantize.cu | 227 +++-- ggml/src/ggml-cuda/quantize.cuh | 11 + tests/test-backend-ops.cpp | 199 ++++ 10 files changed, 1832 insertions(+), 62 deletions(-) create mode 100644 ggml/src/ggml-cuda/mmdq.cu create mode 100644 ggml/src/ggml-cuda/mmdq.cuh diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 4d7f945673ac..410e886261c0 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1579,6 +1579,13 @@ struct ggml_backend_cuda_context { ggml_cuda_stream_context concurrent_stream_context; + // opt-in (GGML_CUDA_MMDQ_SHARE_SRC1): f16 copy of one src1 shared by the next mmdq matmuls that read it (attention Q/K/V) + struct { + const ggml_tensor * src1 = nullptr; + int remaining = 0; + std::unique_ptr> buf; + } mmdq_src1; + // dedicated buffer for the branches of overlapped concurrent regions (attention QKV, MoE shared // expert), reused across layers so their scratch never aliases tensors read across the region ggml_backend_buffer_t concurrent_scratch = nullptr; diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 929353e00ec9..74b81a1887cf 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -35,6 +35,7 @@ #include "ggml-cuda/im2col.cuh" #include "ggml-cuda/mmf.cuh" #include "ggml-cuda/mmq.cuh" +#include "ggml-cuda/mmdq.cuh" #include "ggml-cuda/mmvf.cuh" #include "ggml-cuda/mmvq.cuh" #include "ggml-cuda/moe-weighted-reduction.cuh" @@ -2296,6 +2297,10 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst); return; } + if (ggml_cuda_should_use_mmdq(src0, src1, dst, cc)) { + ggml_cuda_mul_mat_dq(ctx, src0, src1, dst); + return; + } if (ggml_cuda_should_use_mmq(src0->type, cc, ne11, /*n_experts =*/ 0)) { ggml_cuda_mul_mat_q(ctx, src0, src1, nullptr, dst); return; @@ -2367,6 +2372,10 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * } } + if (ggml_cuda_should_use_mmdq_id(src0, src1, ids, dst, cc)) { + ggml_cuda_mul_mat_id_dq(ctx, src0, src1, ids, dst); + return; + } if (ggml_cuda_should_use_mmq(src0->type, cc, ne12, /*n_experts=*/ne02)) { ggml_cuda_mul_mat_q(ctx, src0, src1, ids, dst); return; @@ -4079,6 +4088,45 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, } // try and fuse nodes and return the number of nodes to skip +static const ggml_tensor * ggml_cuda_mmdq_add_bias(const ggml_tensor * add, const ggml_tensor * prev) { + if (add->op != GGML_OP_ADD || (add->src[0] != prev && add->src[1] != prev)) { + return nullptr; + } + const ggml_tensor * b = add->src[0] == prev ? add->src[1] : add->src[0]; + const uintptr_t a0 = (uintptr_t) add->data; + const uintptr_t b0 = (uintptr_t) b->data; + if (add->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32 || !ggml_are_same_shape(add, prev) || !ggml_is_contiguous(add) || + !ggml_is_contiguous(b) || b->ne[0] != prev->ne[0] || ggml_nrows(b) != 1 || (a0 < b0 + ggml_nbytes(b) && b0 < a0 + ggml_nbytes(add))) { + return nullptr; + } + return b; +} + +static bool ggml_cuda_can_use_packed_mmq(ggml_backend_cuda_context & ctx, const ggml_tensor * mm, bool allow_d4 = false) { + const ggml_tensor * w = mm->src[0]; + const ggml_tensor * x = mm->src[1]; + const int cc = ggml_cuda_info().devices[ctx.device].cc; + if (ggml_cuda_should_use_mmdq(mm->src[0], mm->src[1], mm, cc)) { + return false; + } + return GGML_CUDA_CC_IS_RDNA3_5(cc) && (w->type == GGML_TYPE_Q4_K || w->type == GGML_TYPE_Q5_K || (allow_d4 && w->type == GGML_TYPE_Q6_K)) && + x->type == GGML_TYPE_F32 && mm->type == GGML_TYPE_F32 && + ggml_is_contiguous(w) && ggml_is_contiguous(x) && ggml_is_contiguous(mm) && + w->ne[2] == 1 && w->ne[3] == 1 && x->ne[2] == 1 && x->ne[3] == 1 && x->ne[1] >= 64 && x->ne[1] <= 65535 && + ggml_get_op_params_i32(mm, 0) == GGML_PREC_DEFAULT && ggml_prec_get_src(mm, 1) == GGML_PREC_DEFAULT && ggml_get_op_params_i32(mm, 1) == 0 && + !ggml_cuda_should_use_mmvq(w->type, cc, x->ne[1]) && + ggml_cuda_should_use_mmq(w->type, cc, x->ne[1], 0) && + !(w->view_src && ggml_backend_buffer_get_usage(w->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && + ggml_nbytes(w) != ggml_backend_buffer_get_alloc_size(w->buffer, w)); +} + +static size_t ggml_cuda_packed_mmq_size(ggml_backend_cuda_context & ctx, const ggml_tensor * mm) { + GGML_UNUSED(ctx); + // All supported RDNA3.5 tiles fit within 128 columns and 1024 load bytes. + const size_t padding = GGML_PAD(128*sizeof(block_q8_1_mmq), 1024); + return GGML_PAD(mm->src[1]->ne[0], MATRIX_ROW_PADDING)*ggml_nrows(mm->src[1])/QK8_1_MMQ*sizeof(block_q8_1_mmq) + padding; +} + static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { if (ggml_cuda_fusion_disabled()) { @@ -4087,6 +4135,88 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph ggml_tensor * node = cgraph->nodes[i]; + if (node->op == GGML_OP_GLU && (ggml_get_glu_op(node) == GGML_GLU_OP_GEGLU || ggml_get_glu_op(node) == GGML_GLU_OP_SWIGLU) && node->src[1] && ggml_get_op_params_i32(node, 1) == 0 && + node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_F32 && + ggml_is_contiguous(node->src[0]) && ggml_is_contiguous(node->src[1]) && + ggml_cuda_is_aligned(node->src[0], 16) && ggml_cuda_is_aligned(node->src[1], 16) && + ggml_are_same_shape(node, node->src[0]) && ggml_are_same_shape(node, node->src[1])) { + const ggml_op ops[] = { GGML_OP_GLU, GGML_OP_MUL_MAT }; + const int output_idx = i + 1; + if (ggml_can_fuse_subgraph(cgraph, i, 2, ops, &output_idx, 1) && + ggml_cuda_check_fusion_memory_ranges(cgraph, i, 2, &output_idx, 1)) { + ggml_tensor * mm = cgraph->nodes[output_idx]; + if (GGML_CUDA_MMQ_OPT_ENABLED(GGML_CUDA_FUSE_GLU_PACK, true) && mm->src[1] == node && node->ne[0] % 4 == 0 && + ggml_cuda_can_use_packed_mmq(*cuda_ctx, mm, true)) { + ggml_cuda_pool_alloc packed(cuda_ctx->pool(), ggml_cuda_packed_mmq_size(*cuda_ctx, mm)); + const auto quantize = mm->src[0]->type == GGML_TYPE_Q6_K ? quantize_glu_mmq_q8_1_d4_cuda : quantize_glu_mmq_q8_1_cuda; + quantize((const float *) node->src[0]->data, (const float *) node->src[1]->data, + packed.get(), node->ne[0], node->ne[1], ggml_get_glu_op(node), cuda_ctx->stream()); + ggml_cuda_mul_mat_q(*cuda_ctx, mm->src[0], node, nullptr, mm, nullptr, packed.get()); + return 1; + } + + } + } + + if (GGML_CUDA_MMQ_OPT_ENABLED(GGML_CUDA_FUSE_NORM_PACK, true) && node->op == GGML_OP_RMS_NORM) { + const ggml_op ops[] = { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_MUL_MAT, GGML_OP_MUL_MAT }; + const int outputs[] = { i + 2, i + 3 }; + if (ggml_can_fuse_subgraph(cgraph, i, 4, ops, outputs, 2)) { + ggml_tensor * mul = cgraph->nodes[i + 1]; + ggml_tensor * mm0 = cgraph->nodes[i + 2]; + ggml_tensor * mm1 = cgraph->nodes[i + 3]; + const ggml_tensor * weight = mul->src[mul->src[0] == node ? 1 : 0]; + if ((mul->src[0] == node || mul->src[1] == node) && mm0->src[1] == mul && mm1->src[1] == mul && + node->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(node->src[0]) && + ggml_cuda_is_aligned(node->src[0], 16) && ggml_cuda_is_aligned(weight, 16) && + node->ne[0] % MATRIX_ROW_PADDING == 0 && node->ne[0] <= 4096 && + weight->type == GGML_TYPE_F32 && ggml_is_contiguous(weight) && + ggml_nelements(weight) == node->ne[0] && weight->ne[0] == node->ne[0] && + ggml_cuda_can_use_packed_mmq(*cuda_ctx, mm0) && ggml_cuda_can_use_packed_mmq(*cuda_ctx, mm1) && + ggml_cuda_check_fusion_memory_ranges(cgraph, i, 4, outputs, 2)) { + const size_t size = std::max(ggml_cuda_packed_mmq_size(*cuda_ctx, mm0), ggml_cuda_packed_mmq_size(*cuda_ctx, mm1)); + ggml_cuda_pool_alloc packed(cuda_ctx->pool(), size); + quantize_rms_norm_mmq_q8_1_cuda((const float *) node->src[0]->data, (const float *) weight->data, + packed.get(), node->ne[0], node->ne[1], ggml_get_op_params_f32(node, 0), cuda_ctx->stream()); + ggml_cuda_mul_mat_q(*cuda_ctx, mm0->src[0], mul, nullptr, mm0, nullptr, packed.get()); + ggml_cuda_mul_mat_q(*cuda_ctx, mm1->src[0], mul, nullptr, mm1, nullptr, packed.get()); + return 3; + } + } + } + + static const bool mmdq_fuse_bias = []() { + const char * env = getenv("GGML_CUDA_MMDQ_FUSE_BIAS"); + return env != nullptr && atoi(env) != 0; + }(); + if (mmdq_fuse_bias && node->op == GGML_OP_MUL_MAT && + ggml_cuda_should_use_mmdq(node->src[0], node->src[1], node, ggml_cuda_info().devices[cuda_ctx->device].cc) && + ggml_can_fuse(cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_ADD })) { + const ggml_tensor * bias = ggml_cuda_mmdq_add_bias(cgraph->nodes[i + 1], node); + const int output_idx = i + 1; + if (bias && ggml_cuda_check_fusion_memory_ranges(cgraph, i, 2, &output_idx, 1)) { + ggml_cuda_mmdq_src1_share(*cuda_ctx, cgraph, i); + ggml_cuda_mul_mat_dq(*cuda_ctx, node->src[0], node->src[1], cgraph->nodes[i + 1], (const float *) bias->data); + return 1; + } + } + + if ((node->op == GGML_OP_MUL_MAT || node->op == GGML_OP_MUL_MAT_ID) && i + 3 < cgraph->n_nodes) { + ggml_tensor * glu = cgraph->nodes[i + 2]; + ggml_tensor * down = cgraph->nodes[i + 3]; + const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc; + const ggml_op ops[] = { node->op, node->op, GGML_OP_GLU, node->op }; + const int output_idx = i + 3; + if (glu->op == GGML_OP_GLU && (glu->src[0] == node || glu->src[1] == node) && + (glu->src[0] == cgraph->nodes[i + 1] || glu->src[1] == cgraph->nodes[i + 1]) && + ggml_cuda_should_use_mmdq_glu(glu, down, cc) && + ggml_can_fuse_subgraph(cgraph, i, 4, ops, &output_idx, 1)) { + // Input and IDs are staged before down writes dst; gate/up use a private FP16 buffer. + ggml_cuda_mul_mat_dq_glu(*cuda_ctx, glu, down, down); + return 3; + } + } + if (node->op == GGML_OP_MUL_MAT_ID && cuda_ctx->stream_context().concurrent_events.empty() && ggml_cuda_match_shared_expert(cgraph, i, i + 3) && ggml_cuda_should_fuse_mul_mat_vec_q(cgraph->nodes[i + 2]->src[1])) { @@ -4888,6 +5018,8 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph return 2; } + ggml_cuda_mmdq_src1_share(*cuda_ctx, cgraph, i); + return 0; } @@ -5141,6 +5273,8 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud } } + ggml_cuda_mmdq_src1_release(*cuda_ctx); + #ifdef USE_CUDA_GRAPH if (use_cuda_graph && cuda_graph_update_required) { // End CUDA graph capture ggml_cuda_graph_launch_chunk(cuda_ctx, graph, n_chunks++); diff --git a/ggml/src/ggml-cuda/mmdq.cu b/ggml/src/ggml-cuda/mmdq.cu new file mode 100644 index 000000000000..2c955fd1216b --- /dev/null +++ b/ggml/src/ggml-cuda/mmdq.cu @@ -0,0 +1,1228 @@ +#include "mmdq.cuh" +#include "convert.cuh" +#include "mmid.cuh" +#include "unary.cuh" +#include "vecdotq.cuh" + +#include + +// The structure follows the gfx11 W4A16 Triton GEMM in vLLM (Apache-2.0, rdna_hybrid_w4a16.py): +// fp16 activations, 4-bit -> fp16 via the 0x6400 exponent trick, fp16 WMMA with fp32 accumulators. + +#define MMDQ_BK 64 // K per main loop step; one Q4_K/Q6_K block holds 4 steps +#define MMDQ_PAD 8 // halfs of LDS row padding, keeps the 16-row fragment reads free of bank conflicts + +#if (defined(GGML_USE_HIP) && defined(RDNA3)) +typedef _Float16 mmdq_half16_t __attribute__((ext_vector_type(16))); +typedef float mmdq_float8_t __attribute__((ext_vector_type(8))); +#endif // (defined(GGML_USE_HIP) && defined(RDNA3)) + +// bytes 0,1 (lo) or 2,3 (hi) of v, each < 64, as fp16 pair (1024 + b) +static __device__ __forceinline__ half2 mmdq_bytes_lo(const uint32_t v) { + const uint32_t u = (v & 0xFFu) | ((v << 8) & 0xFF0000u) | 0x64006400u; + return *(const half2 *) &u; +} + +static __device__ __forceinline__ half2 mmdq_bytes_hi(const uint32_t v) { + const uint32_t u = ((v >> 16) & 0xFFu) | ((v >> 8) & 0xFF0000u) | 0x64006400u; + return *(const half2 *) &u; +} + +static __device__ __forceinline__ uint32_t mmdq_h2_as_u32(const half2 h) { + return *(const uint32_t *) &h; +} + +// raw weight data for one MMDQ_BK step of one row, as loaded from global memory +template struct mmdq_raw; + +template <> struct mmdq_raw { + static constexpr int threads_per_row = 2; + uint4 hdr; // d, dmin, scales[12] + uint4 qs; // 16 bytes of qs: lo nibbles of sub-block 2j, hi nibbles of sub-block 2j+1 +}; + +template <> struct mmdq_raw { + static constexpr int threads_per_row = 2; + uint4 hdr; // d, dmin, scales[12] + uint4 qs; // as for Q4_K + uint4 qh; // qh bytes of the same 16 positions, bit 2j/2j+1 is bit 4 of the low/high nibble +}; + +template <> struct mmdq_raw { + static constexpr int threads_per_row = 2; + int ql[8]; + int qh[8]; + int sc; // 2 int8 scales + half d; +}; + +template <> struct mmdq_raw { + static constexpr int threads_per_row = 2; + half2 dm; + uint32_t qh; + uint4 qs; // one block of 32 values per thread +}; + +template <> struct mmdq_raw { + static constexpr int threads_per_row = 2; + half d; + uint32_t qh; + uint4 qs; // one block of 32 values per thread +}; + +template <> struct mmdq_raw { + static constexpr int threads_per_row = 2; + int qs[8]; // one block of 32 values per thread + half d; +}; + +// h: which part of the row this thread handles, kq: K index of the step inside the block +static __device__ __forceinline__ void mmdq_load(mmdq_raw & raw, const char * row, const int ib, const int kq, const int h) { + const block_q4_K * b = (const block_q4_K *) row + ib; + raw.hdr = *(const uint4 *) b; + raw.qs = *(const uint4 *) (b->qs + 32*(kq/64) + 16*h); +} + +static __device__ __forceinline__ void mmdq_load(mmdq_raw & raw, const char * row, const int ib, const int kq, const int h) { + const block_q5_K * b = (const block_q5_K *) row + ib; + raw.hdr = *(const uint4 *) b; + raw.qs = *(const uint4 *) (b->qs + 32*(kq/64) + 16*h); + raw.qh = *(const uint4 *) (b->qh + 16*h); +} + +static __device__ __forceinline__ void mmdq_load(mmdq_raw & raw, const char * row, const int ib, const int kq, const int h) { + const block_q6_K * b = (const block_q6_K *) row + ib; + const int c = kq/64; + const int ip = c/2; +#pragma unroll + for (int i = 0; i < 8; ++i) { + raw.ql[i] = get_int_b2(b->ql + 64*ip + 32*h, i); + raw.qh[i] = get_int_b2(b->qh + 32*ip, i); + } + const uint16_t * sc16 = (const uint16_t *) b->scales; + raw.sc = sc16[2*c + h]; + raw.d = b->d; +} + +// ib/kq are in units of QK_K like for the K-quants, thread h takes the block kq/32 + h +static __device__ __forceinline__ void mmdq_load(mmdq_raw & raw, const char * row, const int ib, const int kq, const int h) { + const block_q5_1 * b = (const block_q5_1 *) row + ib*(QK_K/QK5_1) + kq/QK5_1 + h; + raw.dm = *(const half2 *) &b->dm; + raw.qh = *(const uint32_t *) b->qh; + const uint2 * qs = (const uint2 *) b->qs; + const uint2 q0 = qs[0]; + const uint2 q1 = qs[1]; + raw.qs = make_uint4(q0.x, q0.y, q1.x, q1.y); +} + +// Q5_0 blocks are 22 bytes, read with 2 byte loads +static __device__ __forceinline__ void mmdq_load(mmdq_raw & raw, const char * row, const int ib, const int kq, const int h) { + const block_q5_0 * b = (const block_q5_0 *) row + ib*(QK_K/QK5_0) + kq/QK5_0 + h; + raw.d = b->d; + raw.qh = get_int_b2(b->qh, 0); + raw.qs = make_uint4(get_int_b2(b->qs, 0), get_int_b2(b->qs, 1), get_int_b2(b->qs, 2), get_int_b2(b->qs, 3)); +} + +static __device__ __forceinline__ void mmdq_load(mmdq_raw & raw, const char * row, const int ib, const int kq, const int h) { + const block_q8_0 * b = (const block_q8_0 *) row + ib*(QK_K/QK8_0) + kq/QK8_0 + h; +#pragma unroll + for (int i = 0; i < 8; ++i) { + raw.qs[i] = get_int_b2(b->qs, i); + } + raw.d = b->d; +} + +// get_scale_min_k4 on the scales held in hdr.y/z/w +static __device__ __forceinline__ void mmdq_scale_min_q4_K(const uint4 & hdr, const int j, int & sc, int & m) { + if (j < 4) { + sc = (hdr.y >> (8*j)) & 63; + m = (hdr.z >> (8*j)) & 63; + } else { + const int s = 8*(j - 4); + sc = ((hdr.w >> s) & 0xF) | (((hdr.y >> (s + 6)) & 3) << 4); + m = ((hdr.w >> (s + 4)) & 0xF) | (((hdr.z >> (s + 6)) & 3) << 4); + } +} + +// Q4_K/Q5_K: dequantize 32 fp16 values, qh5 holds the Q5_K high bits (0 for Q4_K) +// v[0..7]: sub-block 2j values 16h..16h+15, v[8..15]: sub-block 2j+1 values 16h..16h+15 +static __device__ __forceinline__ void mmdq_dequant_k(const uint4 & hdr, const uint4 & qs, const uint4 & qh5, const int kq, uint32_t * v) { + const int j = kq/64; + int s0, m0, s1, m1; + mmdq_scale_min_q4_K(hdr, 2*j + 0, s0, m0); + mmdq_scale_min_q4_K(hdr, 2*j + 1, s1, m1); + const half2 dm = *(const half2 *) &hdr.x; + const float d = __low2float(dm); + const float dmin = __high2float(dm); + const half2 d0 = __float2half2_rn(d*s0); + const half2 b0 = __float2half2_rn(-dmin*m0); + const half2 d1 = __float2half2_rn(d*s1); + const half2 b1 = __float2half2_rn(-dmin*m1); + const half2 off = __float2half2_rn(1024.0f); + + const uint32_t q[4] = {qs.x, qs.y, qs.z, qs.w}; + const uint32_t u[4] = {qh5.x, qh5.y, qh5.z, qh5.w}; +#pragma unroll + for (int i = 0; i < 4; ++i) { + const uint32_t ql = (q[i] & 0x0F0F0F0Fu) | (((u[i] >> (2*j + 0)) & 0x01010101u) << 4); + const uint32_t qh = ((q[i] >> 4) & 0x0F0F0F0Fu) | (((u[i] >> (2*j + 1)) & 0x01010101u) << 4); + v[2*i + 0] = mmdq_h2_as_u32(__hfma2(__hsub2(mmdq_bytes_lo(ql), off), d0, b0)); + v[2*i + 1] = mmdq_h2_as_u32(__hfma2(__hsub2(mmdq_bytes_hi(ql), off), d0, b0)); + v[2*i + 8] = mmdq_h2_as_u32(__hfma2(__hsub2(mmdq_bytes_lo(qh), off), d1, b1)); + v[2*i + 9] = mmdq_h2_as_u32(__hfma2(__hsub2(mmdq_bytes_hi(qh), off), d1, b1)); + } +} + +static __device__ __forceinline__ void mmdq_dequant(const mmdq_raw & raw, const int kq, const int h, uint32_t * v) { + GGML_UNUSED(h); + mmdq_dequant_k(raw.hdr, raw.qs, make_uint4(0, 0, 0, 0), kq, v); +} + +static __device__ __forceinline__ void mmdq_dequant(const mmdq_raw & raw, const int kq, const int h, uint32_t * v) { + GGML_UNUSED(h); + mmdq_dequant_k(raw.hdr, raw.qs, raw.qh, kq, v); +} + +// Q6_K: v[0..15] are the values 32h..32h+31 of the 64-step +static __device__ __forceinline__ void mmdq_dequant(const mmdq_raw & raw, const int kq, const int h, uint32_t * v) { + const int odd = (kq/64) % 2; + const int sl = 4*odd; + const int sh = 4*odd + 2*h; + const float d = __half2float(raw.d); + const half2 dA = __float2half2_rn(d*(int8_t) (raw.sc & 0xFF)); + const half2 dB = __float2half2_rn(d*(int8_t) ((raw.sc >> 8) & 0xFF)); + const half2 off = __float2half2_rn(1056.0f); // 1024 + 32 +#pragma unroll + for (int i = 0; i < 8; ++i) { + const uint32_t q = (((uint32_t) raw.ql[i] >> sl) & 0x0F0F0F0Fu) | ((((uint32_t) raw.qh[i] >> sh) & 0x03030303u) << 4); + const half2 ds = i < 4 ? dA : dB; + v[2*i + 0] = mmdq_h2_as_u32(__hmul2(__hsub2(mmdq_bytes_lo(q), off), ds)); + v[2*i + 1] = mmdq_h2_as_u32(__hmul2(__hsub2(mmdq_bytes_hi(q), off), ds)); + } +} + +// Q5_1: v[0..15] are the values 32h..32h+31 of the 64-step +static __device__ __forceinline__ void mmdq_dequant(const mmdq_raw & raw, const int kq, const int h, uint32_t * v) { + GGML_UNUSED_VARS(kq, h); + const half2 d = __low2half2(raw.dm); + const half2 m = __high2half2(raw.dm); + const half2 off = __float2half2_rn(1024.0f); + const uint32_t q[4] = {raw.qs.x, raw.qs.y, raw.qs.z, raw.qs.w}; +#pragma unroll + for (int i = 0; i < 4; ++i) { + // bit j of qh is bit 4 of value j, values 4i..4i+3 (low nibbles) and 16+4i..16+4i+3 (high nibbles) + const uint32_t hl = (raw.qh >> (4*i)) & 0xFu; + const uint32_t hh = (raw.qh >> (16 + 4*i)) & 0xFu; + const uint32_t bl = ((hl & 1u) << 4) | ((hl & 2u) << 11) | ((hl & 4u) << 18) | ((hl & 8u) << 25); + const uint32_t bh = ((hh & 1u) << 4) | ((hh & 2u) << 11) | ((hh & 4u) << 18) | ((hh & 8u) << 25); + const uint32_t ql = (q[i] & 0x0F0F0F0Fu) | bl; + const uint32_t qu = ((q[i] >> 4) & 0x0F0F0F0Fu) | bh; + v[2*i + 0] = mmdq_h2_as_u32(__hfma2(__hsub2(mmdq_bytes_lo(ql), off), d, m)); + v[2*i + 1] = mmdq_h2_as_u32(__hfma2(__hsub2(mmdq_bytes_hi(ql), off), d, m)); + v[2*i + 8] = mmdq_h2_as_u32(__hfma2(__hsub2(mmdq_bytes_lo(qu), off), d, m)); + v[2*i + 9] = mmdq_h2_as_u32(__hfma2(__hsub2(mmdq_bytes_hi(qu), off), d, m)); + } +} + +// Q5_0: as Q5_1 with value = (q - 16)*d +static __device__ __forceinline__ void mmdq_dequant(const mmdq_raw & raw, const int kq, const int h, uint32_t * v) { + GGML_UNUSED_VARS(kq, h); + const half2 d = __half2half2(raw.d); + const half2 off = __float2half2_rn(1040.0f); // 1024 + 16 + const uint32_t q[4] = {raw.qs.x, raw.qs.y, raw.qs.z, raw.qs.w}; +#pragma unroll + for (int i = 0; i < 4; ++i) { + const uint32_t hl = (raw.qh >> (4*i)) & 0xFu; + const uint32_t hh = (raw.qh >> (16 + 4*i)) & 0xFu; + const uint32_t bl = ((hl & 1u) << 4) | ((hl & 2u) << 11) | ((hl & 4u) << 18) | ((hl & 8u) << 25); + const uint32_t bh = ((hh & 1u) << 4) | ((hh & 2u) << 11) | ((hh & 4u) << 18) | ((hh & 8u) << 25); + const uint32_t ql = (q[i] & 0x0F0F0F0Fu) | bl; + const uint32_t qu = ((q[i] >> 4) & 0x0F0F0F0Fu) | bh; + v[2*i + 0] = mmdq_h2_as_u32(__hmul2(__hsub2(mmdq_bytes_lo(ql), off), d)); + v[2*i + 1] = mmdq_h2_as_u32(__hmul2(__hsub2(mmdq_bytes_hi(ql), off), d)); + v[2*i + 8] = mmdq_h2_as_u32(__hmul2(__hsub2(mmdq_bytes_lo(qu), off), d)); + v[2*i + 9] = mmdq_h2_as_u32(__hmul2(__hsub2(mmdq_bytes_hi(qu), off), d)); + } +} + +// Q8_0: v[0..15] are the values 32h..32h+31 of the 64-step; the int8 values are biased by 128 to use the 0x6400 trick +static __device__ __forceinline__ void mmdq_dequant(const mmdq_raw & raw, const int kq, const int h, uint32_t * v) { + GGML_UNUSED_VARS(kq, h); + const half2 d = __half2half2(raw.d); + const half2 off = __float2half2_rn(1152.0f); // 1024 + 128 +#pragma unroll + for (int i = 0; i < 8; ++i) { + const uint32_t q = (uint32_t) raw.qs[i] ^ 0x80808080u; + v[2*i + 0] = mmdq_h2_as_u32(__hmul2(__hsub2(mmdq_bytes_lo(q), off), d)); + v[2*i + 1] = mmdq_h2_as_u32(__hmul2(__hsub2(mmdq_bytes_hi(q), off), d)); + } +} + +static __device__ __forceinline__ void mmdq_store_v16(const uint32_t * v, half * dst0, half * dst1) { + uint4 * d0 = (uint4 *) dst0; + uint4 * d1 = (uint4 *) dst1; + d0[0] = make_uint4(v[0], v[1], v[2], v[3]); + d0[1] = make_uint4(v[4], v[5], v[6], v[7]); + d1[0] = make_uint4(v[8], v[9], v[10], v[11]); + d1[1] = make_uint4(v[12], v[13], v[14], v[15]); +} + +// write the dequantized values of one row and one 64-step to the LDS row +static __device__ __forceinline__ void mmdq_store(const mmdq_raw & raw, half * srow, const int kq, const int h) { + uint32_t v[16]; + mmdq_dequant(raw, kq, h, v); + mmdq_store_v16(v, srow + 16*h, srow + 32 + 16*h); +} + +static __device__ __forceinline__ void mmdq_store(const mmdq_raw & raw, half * srow, const int kq, const int h) { + uint32_t v[16]; + mmdq_dequant(raw, kq, h, v); + mmdq_store_v16(v, srow + 16*h, srow + 32 + 16*h); +} + +static __device__ __forceinline__ void mmdq_store(const mmdq_raw & raw, half * srow, const int kq, const int h) { + uint32_t v[16]; + mmdq_dequant(raw, kq, h, v); + mmdq_store_v16(v, srow + 32*h, srow + 32*h + 16); +} + +static __device__ __forceinline__ void mmdq_store(const mmdq_raw & raw, half * srow, const int kq, const int h) { + uint32_t v[16]; + mmdq_dequant(raw, kq, h, v); + mmdq_store_v16(v, srow + 32*h, srow + 32*h + 16); +} + +static __device__ __forceinline__ void mmdq_store(const mmdq_raw & raw, half * srow, const int kq, const int h) { + uint32_t v[16]; + mmdq_dequant(raw, kq, h, v); + mmdq_store_v16(v, srow + 32*h, srow + 32*h + 16); +} + +// MUL_MAT_ID: the rows of y are sorted by expert (see mm_ids_helper), each block works on BT rows of one expert. +struct mmdq_ids_args { + const int32_t * ids_src1; // compact row -> row of y + const int32_t * ids_dst; // compact row -> row of dst + const int32_t * expert_bounds; // compact rows of expert e: [expert_bounds[e], expert_bounds[e+1]) + const int32_t * tile_start; // first block of expert e, tile_start[n_experts] = number of blocks + int n_experts; + int64_t stride_expert_x; +}; + +// glu: 0 = plain GEMM, 1 = SwiGLU, 2 = dense GEGLU. Hidden output uses FP16. +// x is the gate weight and x_up the up weight. GEGLU is separately opt-in. +// LDS rows alternate between 16 gate rows and the same 16 up rows, so each wave holds both (needs an even TO). +// Without glu (dense only): dst[t][o] += bias[o] + res[t][o] for the non-null ones. +struct mmdq_epi_args { + const char * x_up; + half * dst_h; + const float * bias; + const float * res; + int64_t stride_col_res; +}; + +// dst[t][o] = sum_k y[t][k] * x[o][k]; t: token (src1 column), o: output row (src0 row) +// BT x BO output tile per block, NWT x NWO waves, each wave computes (BT/NWT) x (BO/NWO) +// add: bit 0 = add epi.bias, bit 1 = add epi.res (compile time, a runtime check in the store loop slows Q4_K down by up to 16%) +template +__launch_bounds__(32*NWT*NWO, 1) +static __global__ void mul_mat_dq( + const char * __restrict__ x, const half * __restrict__ y, float * __restrict__ dst, + const int ncols_x, const int nrows_x, const int ncols_y, const int64_t stride_row_x, const int64_t stride_col_dst, + const mmdq_ids_args ids, const mmdq_epi_args epi) { +#if (defined(GGML_USE_HIP) && defined(RDNA3)) + static_assert(!glu || (!has_ids && (BO/(16*NWO)) % 2 == 0), "bad glu config"); + static_assert(add == 0 || (!has_ids && !glu), "bad add config"); + constexpr int nthreads = 32*NWT*NWO; + constexpr int TT = BT/(16*NWT); + constexpr int TO = BO/(16*NWO); + constexpr int LDK = MMDQ_BK + MMDQ_PAD; + constexpr int tpr = mmdq_raw::threads_per_row; + constexpr int nload_y = BT*(MMDQ_BK/8)/nthreads; + constexpr int nload_x = BO*tpr/nthreads; + static_assert(nload_y*nthreads == BT*(MMDQ_BK/8), "bad y tile"); + static_assert(nload_x*nthreads == BO*tpr, "bad x tile"); + + __shared__ half sy[BT*LDK]; + __shared__ half sx[BO*LDK]; + + const int tid = threadIdx.x; + const int lane = tid % 32; + const int wave = tid / 32; + const int wt = wave / NWO; + const int wo = wave % NWO; + const int bx = blockIdx.x; + const int by = blockIdx.y; + const int o0 = by*(glu ? BO/2 : BO); + + int t0 = bx*BT; + int t_end = ncols_y; + if constexpr (has_ids) { + const int b = blockIdx.x; + if (b >= ids.tile_start[ids.n_experts]) { + return; + } + int lo = 0; + int hi = ids.n_experts; + while (hi - lo > 1) { + const int mid = (lo + hi)/2; + if (ids.tile_start[mid] <= b) { + lo = mid; + } else { + hi = mid; + } + } + t0 = ids.expert_bounds[lo] + (b - ids.tile_start[lo])*BT; + t_end = ids.expert_bounds[lo + 1]; + x += lo*ids.stride_expert_x; + } + + uint4 ry[nload_y]; + mmdq_raw rx[nload_x]; + + auto load_global = [&](const int k0) { +#pragma unroll + for (int i = 0; i < nload_y; ++i) { + const int idx = tid + i*nthreads; + const int t = t0 + idx/(MMDQ_BK/8); + const int c = idx % (MMDQ_BK/8); + if (t < t_end) { + const int row = has_ids ? ids.ids_src1[t] : t; + ry[i] = *(const uint4 *) (y + (int64_t) row*ncols_x + k0 + 8*c); + } else { + ry[i] = make_uint4(0, 0, 0, 0); + } + } +#pragma unroll + for (int i = 0; i < nload_x; ++i) { + const int idx = tid + i*nthreads; + if constexpr (glu) { + const int r = idx/tpr; + const int o = min(o0 + (r/32)*16 + r % 16, nrows_x - 1); + mmdq_load(rx[i], ((r/16) % 2 ? epi.x_up : x) + o*stride_row_x, k0/QK_K, k0 % QK_K, idx % tpr); + } else { + const int o = min(o0 + idx/tpr, nrows_x - 1); + mmdq_load(rx[i], x + o*stride_row_x, k0/QK_K, k0 % QK_K, idx % tpr); + } + } + }; + + auto store_lds = [&](const int k0) { +#pragma unroll + for (int i = 0; i < nload_y; ++i) { + const int idx = tid + i*nthreads; + *(uint4 *) (sy + (idx/(MMDQ_BK/8))*LDK + 8*(idx % (MMDQ_BK/8))) = ry[i]; + } +#pragma unroll + for (int i = 0; i < nload_x; ++i) { + const int idx = tid + i*nthreads; + mmdq_store(rx[i], sx + (idx/tpr)*LDK, k0 % QK_K, idx % tpr); + } + }; + + mmdq_float8_t acc[TT][TO]; +#pragma unroll + for (int i = 0; i < TT; ++i) { +#pragma unroll + for (int j = 0; j < TO; ++j) { + acc[i][j] = mmdq_float8_t{0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + } + } + + load_global(0); + store_lds(0); + __syncthreads(); + + for (int k0 = 0; k0 < ncols_x; k0 += MMDQ_BK) { + const bool has_next = k0 + MMDQ_BK < ncols_x; + if (has_next) { + load_global(k0 + MMDQ_BK); + } + +#pragma unroll + for (int kk = 0; kk < MMDQ_BK; kk += 16) { + mmdq_half16_t A[TT]; + mmdq_half16_t B[TO]; +#pragma unroll + for (int i = 0; i < TT; ++i) { + const uint4 * src = (const uint4 *) (sy + (wt*TT*16 + i*16 + lane % 16)*LDK + kk); + uint4 tmp[2] = {src[0], src[1]}; + memcpy(&A[i], tmp, sizeof(A[i])); + } +#pragma unroll + for (int j = 0; j < TO; ++j) { + const uint4 * src = (const uint4 *) (sx + (wo*TO*16 + j*16 + lane % 16)*LDK + kk); + uint4 tmp[2] = {src[0], src[1]}; + memcpy(&B[j], tmp, sizeof(B[j])); + } +#pragma unroll + for (int i = 0; i < TT; ++i) { +#pragma unroll + for (int j = 0; j < TO; ++j) { + acc[i][j] = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(A[i], B[j], acc[i][j]); + } + } + } + + if (has_next) { + __syncthreads(); + store_lds(k0 + MMDQ_BK); + __syncthreads(); + } + } + + // acc element l of a lane: token 2*l + lane/16, output row lane % 16 + if constexpr (glu) { +#pragma unroll + for (int i = 0; i < TT; ++i) { +#pragma unroll + for (int j = 0; j < TO; j += 2) { + const int o = o0 + (wo*TO + j)/2*16 + lane % 16; + if (o >= nrows_x) { + continue; + } +#pragma unroll + for (int l = 0; l < 8; ++l) { + const int t = t0 + wt*TT*16 + i*16 + 2*l + lane/16; + if (t < t_end) { + epi.dst_h[(int64_t) t*stride_col_dst + o] = __float2half((glu == 2 ? ggml_cuda_op_gelu_single(acc[i][j][l]) : ggml_cuda_op_silu_single(acc[i][j][l])) * acc[i][j + 1][l]); + } + } + } + } + return; + } +#pragma unroll + for (int i = 0; i < TT; ++i) { +#pragma unroll + for (int j = 0; j < TO; ++j) { + const int o = o0 + wo*TO*16 + j*16 + lane % 16; + if (o >= nrows_x) { + continue; + } +#pragma unroll + for (int l = 0; l < 8; ++l) { + const int t = t0 + wt*TT*16 + i*16 + 2*l + lane/16; + if (t < t_end) { + const int row = has_ids ? ids.ids_dst[t] : t; + float v = acc[i][j][l]; + if constexpr (add & 1) { + v += epi.bias[o]; + } + if constexpr (add & 2) { + v += epi.res[(int64_t) row*epi.stride_col_res + o]; + } + dst[row*stride_col_dst + o] = v; + } + } + } + } +#else + GGML_UNUSED_VARS(x, y, dst, ncols_x, nrows_x, ncols_y, stride_row_x, stride_col_dst, ids, epi); + NO_DEVICE_CODE; +#endif // (defined(GGML_USE_HIP) && defined(RDNA3)) +} + +// tile_start[e] = number of BT-row blocks of all experts before e +static __global__ void mmdq_tile_start(const int32_t * __restrict__ expert_bounds, int32_t * __restrict__ tile_start, const int n_experts, const int BT) { + __shared__ int buf[1024]; + const int e = threadIdx.x; + const int n = e < n_experts ? (expert_bounds[e + 1] - expert_bounds[e] + BT - 1)/BT : 0; + buf[e] = n; + __syncthreads(); + for (int off = 1; off < 1024; off *= 2) { + const int v = e >= off ? buf[e - off] : 0; + __syncthreads(); + buf[e] += v; + __syncthreads(); + } + if (e < n_experts) { + tile_start[e] = buf[e] - n; + } + if (e == n_experts - 1) { + tile_start[n_experts] = buf[e]; + } +} + +// MoE experts with few tokens per expert: each lane reads its weight row from global memory into registers and dequantizes it there, +// so every weight is read once per expert and token group; only the activations go through LDS. +// Block: NW waves x 16 output rows, up to 16*NT tokens of one expert. +#define MMDQ_MOE_KC 128 // K per LDS activation chunk + +// The dequantized 64-step of a lane is 2 groups of 8 dwords. Group g holds 16-K step s_own[g] in lanes 0-15 and step s_other[g] in +// lanes 16-31, so for s_own[g] lanes 16-31 take the group from lanes 0-15 and for s_other[g] the other way around. +template struct mmdq_moe_map { + static constexpr int s_own[2] = {0, 2}; + static constexpr int s_other[2] = {1, 3}; +}; + +template <> struct mmdq_moe_map { + static constexpr int s_own[2] = {0, 1}; + static constexpr int s_other[2] = {2, 3}; +}; + +template <> struct mmdq_moe_map : mmdq_moe_map {}; + +#define MMDQ_MOE_MAX_EXPERTS 512 + +// glu: waves [0, NW/2) compute gate rows, waves [NW/2, NW) the same up rows, dst_h = half(silu(gate) * up) +template +__launch_bounds__(32*NW, 1) +static __global__ void mul_mat_dq_moe( + const char * __restrict__ x, const half * __restrict__ y, float * __restrict__ dst, + const int ncols_x, const int nrows_x, const int64_t stride_row_x, const int64_t stride_col_dst, const mmdq_ids_args ids, + const mmdq_epi_args epi) { +#if (defined(GGML_USE_HIP) && defined(RDNA3)) + constexpr int NWR = glu ? NW/2 : NW; // waves per 16-row group + static_assert(!glu || (NW % 2 == 0 && 16*NT*16*NWR*sizeof(float) <= 16*NT*(MMDQ_MOE_KC + MMDQ_PAD)*sizeof(half)), "bad glu config"); + constexpr int TMAX = 16*NT; + constexpr int LDK = MMDQ_MOE_KC + MMDQ_PAD; + constexpr int nthreads = 32*NW; + constexpr int nload = TMAX*(MMDQ_MOE_KC/8)/nthreads; // uint4 of activations per thread and chunk + static_assert(nload*nthreads == TMAX*(MMDQ_MOE_KC/8), "bad activation tile"); + + __shared__ half sy[TMAX*LDK]; + __shared__ int s_tile_start[MMDQ_MOE_MAX_EXPERTS + 1]; + __shared__ int s_rows[TMAX]; + + const int tid = threadIdx.x; + const int lane = tid % 32; + const int wave = tid / 32; + const int h = lane / 16; + + // find the expert of this block with the tile table in LDS, a search in global memory is a chain of dependent loads + for (int e = tid; e <= ids.n_experts; e += nthreads) { + s_tile_start[e] = ids.tile_start[e]; + } + __syncthreads(); + // blockIdx.x: row block, blockIdx.y: token tile, so that the blocks reading the same activations run close together in time + const int b = blockIdx.y; + if (b >= s_tile_start[ids.n_experts]) { + return; + } + int lo = 0; + int hi = ids.n_experts; + while (hi - lo > 1) { + const int mid = (lo + hi)/2; + if (s_tile_start[mid] <= b) { + lo = mid; + } else { + hi = mid; + } + } + const int t0 = ids.expert_bounds[lo] + (b - s_tile_start[lo])*TMAX; + const int ntok = min(TMAX, ids.expert_bounds[lo + 1] - t0); + const int ntiles = (ntok + 15)/16; + for (int t = tid; t < TMAX; t += nthreads) { + s_rows[t] = t < ntok ? ids.ids_src1[t0 + t] : -1; + } + + const bool is_up = glu && wave >= NWR; + const int o = blockIdx.x*16*NWR + (wave % NWR)*16 + lane % 16; + const char * xrow = (is_up ? epi.x_up : x) + lo*ids.stride_expert_x + min(o, nrows_x - 1)*stride_row_x; + + mmdq_float8_t acc[NT]; +#pragma unroll + for (int i = 0; i < NT; ++i) { + acc[i] = mmdq_float8_t{0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + } + + // the weights and activations of the next chunk are loaded to registers while the current chunk is used + uint4 ry[nload]; + mmdq_raw raw[2]; + mmdq_raw raw_next[2]; + + auto load_y = [&](const int kc) { +#pragma unroll + for (int i = 0; i < nload; ++i) { + const int idx = tid + i*nthreads; + const int t = idx / (MMDQ_MOE_KC/8); + const int c = idx % (MMDQ_MOE_KC/8); + const int row = s_rows[t]; + ry[i] = row >= 0 && kc + 8*c < ncols_x ? *(const uint4 *) (y + (int64_t) row*ncols_x + kc + 8*c) : make_uint4(0, 0, 0, 0); + } + }; + auto store_y = [&]() { +#pragma unroll + for (int i = 0; i < nload; ++i) { + const int idx = tid + i*nthreads; + *(uint4 *) (sy + (idx / (MMDQ_MOE_KC/8))*LDK + 8*(idx % (MMDQ_MOE_KC/8))) = ry[i]; + } + }; + auto load_x = [&](mmdq_raw * r, const int kc) { +#pragma unroll + for (int j = 0; j < MMDQ_MOE_KC/64; ++j) { + const int k = kc + 64*j; + if (k < ncols_x) { + mmdq_load(r[j], xrow, k/QK_K, k % QK_K, h); + } + } + }; + + __syncthreads(); // s_rows + load_x(raw, 0); + load_y(0); + store_y(); + __syncthreads(); + + for (int kc = 0; kc < ncols_x; kc += MMDQ_MOE_KC) { + const bool has_next = kc + MMDQ_MOE_KC < ncols_x; + if (has_next) { + load_x(raw_next, kc + MMDQ_MOE_KC); + load_y(kc + MMDQ_MOE_KC); + } + +#pragma unroll + for (int j = 0; j < MMDQ_MOE_KC/64; ++j) { + const int k0 = kc + 64*j; + if (k0 >= ncols_x) { + break; // K is a multiple of 64, only the last chunk can be partial + } + + uint32_t w[16]; + mmdq_dequant(raw[j], k0 % QK_K, h, w); + + // B fragments of the 4 16-K steps + mmdq_half16_t B[4]; +#pragma unroll + for (int g = 0; g < 2; ++g) { + uint32_t b_own[8]; + uint32_t b_other[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) { + const uint32_t wo = w[8*g + i]; + const uint32_t wx = __builtin_amdgcn_permlanex16(wo, wo, 0x76543210u, 0xfedcba98u, false, false); + b_own[i] = h == 0 ? wo : wx; + b_other[i] = h == 0 ? wx : wo; + } + memcpy(&B[mmdq_moe_map::s_own[g]], b_own, sizeof(B[0])); + memcpy(&B[mmdq_moe_map::s_other[g]], b_other, sizeof(B[0])); + } + +#pragma unroll + for (int i = 0; i < NT; ++i) { + if (i < ntiles) { +#pragma unroll + for (int st = 0; st < 4; ++st) { + const uint4 * src = (const uint4 *) (sy + (16*i + lane % 16)*LDK + 64*j + 16*st); + uint4 tmp[2] = {src[0], src[1]}; + mmdq_half16_t A; + memcpy(&A, tmp, sizeof(A)); + acc[i] = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(A, B[st], acc[i]); + } + } + // keep the A loads of a tile next to its WMMAs, hoisting all of them costs too many VGPRs + __builtin_amdgcn_sched_barrier(0); + } + } + + if (has_next) { +#pragma unroll + for (int j = 0; j < MMDQ_MOE_KC/64; ++j) { + raw[j] = raw_next[j]; + } + __syncthreads(); + store_y(); + __syncthreads(); + } + } + + // acc element l of a lane: token 2*l + lane/16, output row lane % 16 + if constexpr (glu) { + // the up waves pass their results to the gate waves through LDS + float * s_up = (float *) sy; + __syncthreads(); + if (is_up) { +#pragma unroll + for (int i = 0; i < NT; ++i) { +#pragma unroll + for (int l = 0; l < 8; ++l) { + s_up[(16*i + 2*l + h)*16*NWR + (wave - NWR)*16 + lane % 16] = acc[i][l]; + } + } + } + __syncthreads(); + if (is_up || o >= nrows_x) { + return; + } +#pragma unroll + for (int i = 0; i < NT; ++i) { +#pragma unroll + for (int l = 0; l < 8; ++l) { + const int t = 16*i + 2*l + h; + if (t < ntok) { + const float up = s_up[t*16*NWR + wave*16 + lane % 16]; + epi.dst_h[ids.ids_dst[t0 + t]*stride_col_dst + o] = __float2half(ggml_cuda_op_silu_single(acc[i][l]) * up); + } + } + } + return; + } + if (o >= nrows_x) { + return; + } +#pragma unroll + for (int i = 0; i < NT; ++i) { +#pragma unroll + for (int l = 0; l < 8; ++l) { + const int t = 16*i + 2*l + h; + if (t < ntok) { + dst[ids.ids_dst[t0 + t]*stride_col_dst + o] = acc[i][l]; + } + } + } +#else + GGML_UNUSED_VARS(x, y, dst, ncols_x, nrows_x, stride_row_x, stride_col_dst, ids, epi); + NO_DEVICE_CODE; +#endif // (defined(GGML_USE_HIP) && defined(RDNA3)) +} + +template +static void mul_mat_dq_moe_launch( + const char * x, const half * y, float * dst, const int ncols_x, const int nrows_x, const int n_rows, + const int64_t stride_row_x, const int64_t stride_col_dst, const mmdq_ids_args & ids, const mmdq_epi_args & epi, cudaStream_t stream) { + mmdq_tile_start<<<1, 1024, 0, stream>>>(ids.expert_bounds, (int32_t *) ids.tile_start, ids.n_experts, 16*NT); + constexpr int rows_per_block = 16*(glu ? NW/2 : NW); + const dim3 grid((nrows_x + rows_per_block - 1)/rows_per_block, (n_rows + 16*NT - 1)/(16*NT) + ids.n_experts, 1); + mul_mat_dq_moe<<>>(x, y, dst, ncols_x, nrows_x, stride_row_x, stride_col_dst, ids, epi); +} + +template +static void mul_mat_dq_launch( + const char * x, const half * y, float * dst, const int ncols_x, const int nrows_x, const int ncols_y, + const int64_t stride_row_x, const int64_t stride_col_dst, const mmdq_ids_args & ids, const mmdq_epi_args & epi_in, cudaStream_t stream) { + if constexpr (glu && (has_ids || (BO/(16*NWO)) % 2 != 0)) { + GGML_ABORT("mmdq config not supported with glu"); + } else { + const mmdq_epi_args & epi = epi_in; + int nblocks_t = (ncols_y + BT - 1)/BT; + if constexpr (has_ids) { + // ncols_y is the number of compact rows, each expert adds at most one partial block + nblocks_t += ids.n_experts; + mmdq_tile_start<<<1, 1024, 0, stream>>>(ids.expert_bounds, (int32_t *) ids.tile_start, ids.n_experts, BT); + } + constexpr int rows_per_block = glu ? BO/2 : BO; + const dim3 grid(nblocks_t, (nrows_x + rows_per_block - 1)/rows_per_block, 1); + const dim3 block(32*NWT*NWO, 1, 1); + if constexpr (!has_ids && !glu) { + const int add = (epi.bias ? 1 : 0) | (epi.res ? 2 : 0); + switch (add) { + case 1: mul_mat_dq<<>>(x, y, dst, ncols_x, nrows_x, ncols_y, stride_row_x, stride_col_dst, ids, epi); return; + case 2: mul_mat_dq<<>>(x, y, dst, ncols_x, nrows_x, ncols_y, stride_row_x, stride_col_dst, ids, epi); return; + case 3: mul_mat_dq<<>>(x, y, dst, ncols_x, nrows_x, ncols_y, stride_row_x, stride_col_dst, ids, epi); return; + default: break; + } + } + mul_mat_dq<<>>(x, y, dst, ncols_x, nrows_x, ncols_y, stride_row_x, stride_col_dst, ids, epi); + } +} + +static int mmdq_env_int(const char * name, const int def) { + const char * env = getenv(name); + return env != nullptr ? atoi(env) : def; +} + +static bool mmdq_env_flag(const char * name) { + return mmdq_env_int(name, 0) != 0; +} + +// ncols_y_cfg: columns per expert on average for MUL_MAT_ID +template +static void mul_mat_dq_switch( + const char * x, const half * y, float * dst, const int ncols_x, const int nrows_x, const int ncols_y, const int ncols_y_cfg, + const int64_t stride_row_x, const int64_t stride_col_dst, const mmdq_ids_args & ids, const mmdq_epi_args & epi, cudaStream_t stream) { + static const int cfg_env = mmdq_env_int(has_ids ? "GGML_CUDA_MMDQ_ID_CFG" : "GGML_CUDA_MMDQ_CFG", -1); + int cfg = cfg_env; + if (cfg < 0) { + // tuned on gfx1151 with Qwen2.5-VL-7B, Gemma-4-31B and MoE expert shapes + if (has_ids) { + cfg = ncols_y_cfg <= 24 ? 8 : 5; + } else if (ncols_y <= 64) { + cfg = 3; + } else if (ncols_y <= 128) { + cfg = 4; + } else { + // glu: nrows_x is the number of gate rows, the block works on as many up rows + cfg = type == GGML_TYPE_Q4_K && (glu ? 2 : 1)*nrows_x >= 16384 ? 6 : 0; + } + } +#define MMDQ_CASE(id, BT, BO, NWT, NWO) case id: mul_mat_dq_launch(x, y, dst, ncols_x, nrows_x, ncols_y, stride_row_x, stride_col_dst, ids, epi, stream); break + switch (cfg) { + MMDQ_CASE(0, 128, 128, 2, 4); + MMDQ_CASE(1, 64, 128, 1, 4); + MMDQ_CASE(2, 32, 128, 1, 4); + MMDQ_CASE(3, 64, 64, 2, 2); + MMDQ_CASE(4, 128, 64, 2, 2); + MMDQ_CASE(5, 32, 64, 1, 2); + MMDQ_CASE(6, 128, 256, 2, 4); + MMDQ_CASE(7, 64, 32, 2, 1); + MMDQ_CASE(8, 16, 64, 1, 4); + MMDQ_CASE(9, 16, 128, 1, 4); + default: GGML_ABORT("bad mmdq config %d", cfg); + } +#undef MMDQ_CASE +} + +template +static void mul_mat_dq_switch_type( + const ggml_type type, const char * x, const half * y, float * dst, const int ncols_x, const int nrows_x, const int ncols_y, + const int ncols_y_cfg, const int64_t stride_row_x, const int64_t stride_col_dst, const mmdq_ids_args & ids, cudaStream_t stream, + const mmdq_epi_args & epi = {}) { + switch (type) { + case GGML_TYPE_Q4_K: + mul_mat_dq_switch(x, y, dst, ncols_x, nrows_x, ncols_y, ncols_y_cfg, stride_row_x, stride_col_dst, ids, epi, stream); + break; + case GGML_TYPE_Q6_K: + mul_mat_dq_switch(x, y, dst, ncols_x, nrows_x, ncols_y, ncols_y_cfg, stride_row_x, stride_col_dst, ids, epi, stream); + break; + case GGML_TYPE_Q5_0: + if constexpr (!has_ids) { + mul_mat_dq_switch(x, y, dst, ncols_x, nrows_x, ncols_y, ncols_y_cfg, stride_row_x, stride_col_dst, ids, epi, stream); + break; + } + GGML_ABORT("unsupported type for mmdq: %s", ggml_type_name(type)); + case GGML_TYPE_Q5_K: + if constexpr (!glu) { + mul_mat_dq_switch(x, y, dst, ncols_x, nrows_x, ncols_y, ncols_y_cfg, stride_row_x, stride_col_dst, ids, epi, stream); + break; + } + [[fallthrough]]; + case GGML_TYPE_Q8_0: + if constexpr (!has_ids && !glu) { + mul_mat_dq_switch(x, y, dst, ncols_x, nrows_x, ncols_y, ncols_y_cfg, stride_row_x, stride_col_dst, ids, epi, stream); + break; + } + [[fallthrough]]; + default: + GGML_ABORT("unsupported type for mmdq: %s", ggml_type_name(type)); + } +} + +// checks shared by MUL_MAT and MUL_MAT_ID +static bool mmdq_supported_shape(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst, const int cc) { + if (!GGML_CUDA_CC_IS_RDNA3_5(cc)) { + return false; + } + if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || + (ggml_prec_get_acc(dst) != GGML_PREC_DEFAULT && ggml_prec_get_acc(dst) != GGML_PREC_F32) || + ggml_prec_get_src(dst, 1) != GGML_PREC_DEFAULT || + (dst->op == GGML_OP_MUL_MAT && ggml_get_op_params_i32(dst, 1) != 0) || + getenv("GGML_CUDA_CUBLAS_COMPUTE_TYPE") != nullptr) { + return false; + } + if (src0->nb[0] != ggml_type_size(src0->type) || !ggml_is_contiguous(src1) || dst->nb[0] != sizeof(float)) { + return false; + } + // Q4_K/Q5_K rows are read with 16 byte loads, Q6_K/Q8_0 rows with 2 byte loads (Q5_1 is checked by the caller) + if (src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_Q5_1 && src0->type != GGML_TYPE_Q5_0 && ((uintptr_t) src0->data % 16 != 0 || src0->nb[1] % 16 != 0 || src0->nb[2] % 16 != 0)) { + return false; + } + return src0->ne[1] <= INT_MAX && src1->ne[1]*src1->ne[2] <= INT_MAX; +} + +static bool mmdq_supported(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst, const int cc) { + static const bool enabled = mmdq_env_flag("GGML_CUDA_MMDQ"); + return enabled && mmdq_supported_shape(src0, src1, dst, cc); +} + +bool ggml_cuda_should_use_mmdq(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst, int cc) { + static const bool no_q6k = mmdq_env_flag("GGML_CUDA_MMDQ_NO_Q6K"); + // below these batch sizes mmq is as fast for Q4_K; for Q6_K mmq is slower at any batch size + static const int min_batch_q4_K = mmdq_env_int("GGML_CUDA_MMDQ_MIN_BATCH", 64); + static const int min_batch_q6_K = mmdq_env_int("GGML_CUDA_MMDQ_MIN_BATCH", 32); + static const bool q8_0 = mmdq_env_flag("GGML_CUDA_MMDQ_Q8_0"); + static const int min_batch_q8_0 = mmdq_env_int("GGML_CUDA_MMDQ_Q8_0_MIN_BATCH", 576); + + if (!mmdq_supported(src0, src1, dst, cc)) { + return false; + } + static const bool q5_0 = mmdq_env_flag("GGML_CUDA_MMDQ_Q5_0"); + const bool is_q8_0 = src0->type == GGML_TYPE_Q8_0; + const bool is_q5_0 = src0->type == GGML_TYPE_Q5_0; + if (src0->type != GGML_TYPE_Q4_K && (src0->type != GGML_TYPE_Q6_K || no_q6k) && (!is_q8_0 || !q8_0) && (!is_q5_0 || !q5_0)) { + return false; + } + // Q8_0/Q5_0 are read in steps of 64 values + if ((is_q8_0 || is_q5_0) && src0->ne[0] % 64 != 0) { + return false; + } + const int min_batch = is_q8_0 ? min_batch_q8_0 : src0->type == GGML_TYPE_Q4_K ? min_batch_q4_K : min_batch_q6_K; + if (src1->ne[1] < min_batch) { + return false; + } + // Q8_0: mmq is faster for few output rows + if (is_q8_0 && (src0->ne[1] < 512 || (src0->ne[1] < 1024 && src1->ne[1] < 1024))) { + return false; + } + // too few output tiles to fill the GPU + if (src0->ne[1] < 1024 && src1->ne[1] < 512) { + return false; + } + return src0->ne[2] == 1 && src0->ne[3] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1; +} + +static bool mmdq_moe_enabled() { + static const bool enabled = mmdq_env_flag("GGML_CUDA_MMDQ_MOE"); + return enabled; +} + +bool ggml_cuda_should_use_mmdq_id(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, const ggml_tensor * dst, int cc) { + static const int min_batch_moe = mmdq_env_int("GGML_CUDA_MMDQ_MOE_MIN_BATCH", 16); + static const bool mmdq_moe_all = mmdq_env_flag("GGML_CUDA_MMDQ_MOE_ALL"); // skip the shape rules, for tests + + const bool moe = mmdq_moe_enabled(); + if (!moe || !mmdq_supported_shape(src0, src1, dst, cc)) { + return false; + } + if (src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K && (!moe || src0->type != GGML_TYPE_Q5_1)) { + return false; + } + // Q5_1 is read in steps of 64 values with 8 byte loads + if (src0->type == GGML_TYPE_Q5_1 && (src0->ne[0] % 64 != 0 || (uintptr_t) src0->data % 8 != 0 || src0->nb[1] % 8 != 0 || src0->nb[2] % 8 != 0)) { + return false; + } + // src1: [K, n_expert_used or 1, n_tokens], dst: [N, n_expert_used, n_tokens], one weight matrix per expert + if (src1->ne[2] < min_batch_moe || src1->ne[3] != 1 || src0->ne[3] != 1 || src0->ne[2] > MMDQ_MOE_MAX_EXPERTS) { + return false; + } + if (!ggml_is_contiguous(dst) || ids->type != GGML_TYPE_I32 || ids->nb[0] != sizeof(int32_t)) { + return false; + } + if (moe && !mmdq_moe_all) { + // measured on gfx1151: mmq is as fast or faster with < 16 tokens per expert on average, or < 32 for short K-quant rows + const int64_t min_tokens_per_expert = src0->type != GGML_TYPE_Q5_1 && src0->ne[0] < 2048 ? 32 : 16; + if (src1->ne[2]*ids->ne[0] < min_tokens_per_expert*src0->ne[2]) { + return false; + } + } + return ids->ne[0] == dst->ne[1] && src1->ne[2] == dst->ne[2]; +} + +// MUL_MAT_ID with the weight streaming kernel +template +static void mul_mat_dq_moe_switch( + const ggml_type type, const char * x, const half * y, float * dst, const int ncols_x, const int nrows_x, const int n_rows, + const int64_t stride_row_x, const int64_t stride_col_dst, const mmdq_ids_args & ids, cudaStream_t stream, + const mmdq_epi_args & epi = {}) { + // up to 96 tokens of one expert per block was the best or close to the best on every measured shape (gfx1151) + static const int cfg = mmdq_env_int("GGML_CUDA_MMDQ_MOE_CFG", 2); +#define MMDQ_MOE_CASE(NT, NW) \ + switch (type) { \ + case GGML_TYPE_Q4_K: mul_mat_dq_moe_launch(x, y, dst, ncols_x, nrows_x, n_rows, stride_row_x, stride_col_dst, ids, epi, stream); break; \ + case GGML_TYPE_Q5_K: mul_mat_dq_moe_launch(x, y, dst, ncols_x, nrows_x, n_rows, stride_row_x, stride_col_dst, ids, epi, stream); break; \ + case GGML_TYPE_Q6_K: mul_mat_dq_moe_launch(x, y, dst, ncols_x, nrows_x, n_rows, stride_row_x, stride_col_dst, ids, epi, stream); break; \ + case GGML_TYPE_Q5_1: mul_mat_dq_moe_launch(x, y, dst, ncols_x, nrows_x, n_rows, stride_row_x, stride_col_dst, ids, epi, stream); break; \ + default: GGML_ABORT("unsupported type for mmdq moe: %s", ggml_type_name(type)); \ + } + switch (cfg) { + case 1: MMDQ_MOE_CASE(2, 8); break; + case 2: MMDQ_MOE_CASE(6, 8); break; + case 3: MMDQ_MOE_CASE(8, 8); break; + default: GGML_ABORT("bad mmdq moe config %d", cfg); + } +#undef MMDQ_MOE_CASE +} + +void ggml_cuda_mul_mat_dq(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const float * bias, const ggml_tensor * res) { + GGML_TENSOR_BINARY_OP_LOCALS; + + cudaStream_t stream = ctx.stream(); + + auto & shared = ctx.mmdq_src1; + const bool use_shared = shared.src1 == src1 && ctx.curr_stream_no == 0; + ggml_cuda_pool_alloc src1_f16(ctx.pool()); + const half * y; + if (use_shared) { + if (!shared.buf) { + shared.buf = std::make_unique>(ctx.pool(), ne10*ne11); + ggml_get_to_fp16_cuda(GGML_TYPE_F32)(src1->data, shared.buf->get(), ne10*ne11, stream); + } + y = shared.buf->get(); + } else { + src1_f16.alloc(ne10*ne11); + ggml_get_to_fp16_cuda(GGML_TYPE_F32)(src1->data, src1_f16.get(), ne10*ne11, stream); + y = src1_f16.get(); + } + + const mmdq_epi_args epi = {nullptr, nullptr, bias, res ? (const float *) res->data : nullptr, res ? (int64_t) (res->nb[1]/sizeof(float)) : 0}; + mul_mat_dq_switch_type(src0->type, (const char *) src0->data, y, (float *) dst->data, ne00, ne01, ne11, ne11, + nb01, nb1/sizeof(float), mmdq_ids_args{}, stream, epi); + + if (use_shared && --shared.remaining <= 0) { + ggml_cuda_mmdq_src1_release(ctx); + } +} + +void ggml_cuda_mmdq_src1_release(ggml_backend_cuda_context & ctx) { + ctx.mmdq_src1.buf.reset(); + ctx.mmdq_src1.src1 = nullptr; + ctx.mmdq_src1.remaining = 0; +} + +void ggml_cuda_mmdq_src1_share(ggml_backend_cuda_context & ctx, const ggml_cgraph * cgraph, const int i) { + static const bool enabled = mmdq_env_flag("GGML_CUDA_MMDQ_SHARE_SRC1"); + const ggml_tensor * node = cgraph->nodes[i]; + if (!enabled || ctx.mmdq_src1.src1 != nullptr || ctx.curr_stream_no != 0 || node->op != GGML_OP_MUL_MAT) { + return; + } + const int cc = ggml_cuda_info().devices[ctx.device].cc; + if (!ggml_cuda_should_use_mmdq(node->src[0], node->src[1], node, cc)) { + return; + } + // users close by in the graph (attention Q/K/V with their norm/rope nodes in between) + int n = 0; + for (int j = i + 1; j < cgraph->n_nodes && j <= i + 32; ++j) { + const ggml_tensor * t = cgraph->nodes[j]; + if (t->op == GGML_OP_MUL_MAT && t->src[1] == node->src[1] && ggml_cuda_should_use_mmdq(t->src[0], t->src[1], t, cc)) { + n++; + } + } + if (n > 0) { + ctx.mmdq_src1.src1 = node->src[1]; + ctx.mmdq_src1.remaining = n + 1; + } +} + +bool ggml_cuda_should_use_mmdq_glu(const ggml_tensor * glu, const ggml_tensor * down, int cc) { + static const bool enabled = mmdq_env_flag("GGML_CUDA_MMDQ_GLU"); + static const bool geglu_enabled = mmdq_env_flag("GGML_CUDA_MMDQ_GEGLU"); + if (!enabled || glu->op != GGML_OP_GLU || ggml_get_op_params_i32(glu, 1) != 0) { + return false; + } + const ggml_glu_op activation = ggml_get_glu_op(glu); + if (activation != GGML_GLU_OP_SWIGLU && + !(geglu_enabled && activation == GGML_GLU_OP_GEGLU && glu->src[0]->op == GGML_OP_MUL_MAT)) { + return false; + } + const ggml_tensor * gate = glu->src[0]; + const ggml_tensor * up = glu->src[1]; + if (up == nullptr || gate->op != up->op || down->op != gate->op || down->src[1] != glu || + (gate->op != GGML_OP_MUL_MAT && (gate->op != GGML_OP_MUL_MAT_ID || !mmdq_moe_enabled()))) { + return false; + } + const ggml_tensor * w_gate = gate->src[0]; + const ggml_tensor * w_up = up->src[0]; + if (gate->src[1] != up->src[1] || w_gate->type != w_up->type || w_gate->ne[0] != w_up->ne[0] || w_gate->ne[1] != w_up->ne[1] || + w_gate->ne[2] != w_up->ne[2] || w_gate->nb[1] != w_up->nb[1] || w_gate->nb[2] != w_up->nb[2]) { + return false; + } + if (!ggml_are_same_shape(glu, gate) || !ggml_are_same_shape(gate, up) || !ggml_is_contiguous(glu) || !ggml_is_contiguous(gate) || !ggml_is_contiguous(up)) { + return false; + } + if (gate->op == GGML_OP_MUL_MAT) { + return (w_gate->type == GGML_TYPE_Q4_K || w_gate->type == GGML_TYPE_Q6_K || w_gate->type == GGML_TYPE_Q5_0) && + ggml_cuda_should_use_mmdq(w_gate, gate->src[1], gate, cc) && ggml_cuda_should_use_mmdq(w_up, up->src[1], up, cc) && + ggml_cuda_should_use_mmdq(down->src[0], glu, down, cc); + } + // down reads glu row (token, slot) = token*n_expert_used + slot, which is the ids_dst row of gate/up + const ggml_tensor * ids = gate->src[2]; + return up->src[2] == ids && down->src[2] == ids && glu->ne[1] == ids->ne[0] && down->src[0]->ne[2] == w_gate->ne[2] && + ggml_cuda_should_use_mmdq_id(w_gate, gate->src[1], ids, gate, cc) && ggml_cuda_should_use_mmdq_id(w_up, up->src[1], ids, up, cc) && + ggml_cuda_should_use_mmdq_id(down->src[0], glu, ids, down, cc); +} + +static void ggml_cuda_mul_mat_id_dq_glu(ggml_backend_cuda_context & ctx, const ggml_tensor * glu, ggml_tensor * down) { + const ggml_tensor * w_gate = glu->src[0]->src[0]; + const ggml_tensor * w_up = glu->src[1]->src[0]; + const ggml_tensor * w_down = down->src[0]; + const ggml_tensor * x = glu->src[0]->src[1]; + const ggml_tensor * ids = glu->src[0]->src[2]; + + const int64_t n_experts = w_gate->ne[2]; + const int64_t n_expert_used = ids->ne[0]; + const int64_t n_tokens = x->ne[2]; + const int64_t n_rows = n_tokens*n_expert_used; + const int64_t n_ff = w_gate->ne[1]; + + cudaStream_t stream = ctx.stream(); + + ggml_cuda_pool_alloc ids_src1(ctx.pool(), n_rows); + ggml_cuda_pool_alloc ids_dst(ctx.pool(), n_rows); + ggml_cuda_pool_alloc expert_bounds(ctx.pool(), n_experts + 1); + ggml_cuda_pool_alloc tile_start(ctx.pool(), n_experts + 1); + + const int si1 = ids->nb[1] / ggml_element_size(ids); + const int sis1 = x->nb[2] / x->nb[1]; + ggml_cuda_launch_mm_ids_helper((const int32_t *) ids->data, ids_src1.get(), ids_dst.get(), expert_bounds.get(), + n_experts, n_tokens, n_expert_used, x->ne[1], si1, sis1, /*write_inverse =*/ false, stream); + CUDA_CHECK(cudaGetLastError()); + + ggml_cuda_pool_alloc x_f16(ctx.pool(), ggml_nelements(x)); + ggml_get_to_fp16_cuda(GGML_TYPE_F32)(x->data, x_f16.get(), ggml_nelements(x), stream); + + ggml_cuda_pool_alloc h_f16(ctx.pool(), ggml_nelements(glu)); + const mmdq_ids_args args = {ids_src1.get(), ids_dst.get(), expert_bounds.get(), tile_start.get(), (int) n_experts, (int64_t) w_gate->nb[2]}; + mul_mat_dq_moe_switch(w_gate->type, (const char *) w_gate->data, x_f16.get(), nullptr, x->ne[0], n_ff, n_rows, + w_gate->nb[1], n_ff, args, stream, mmdq_epi_args{(const char *) w_up->data, h_f16.get()}); + + const mmdq_ids_args args_down = {ids_dst.get(), ids_dst.get(), expert_bounds.get(), tile_start.get(), (int) n_experts, (int64_t) w_down->nb[2]}; + mul_mat_dq_moe_switch(w_down->type, (const char *) w_down->data, h_f16.get(), (float *) down->data, n_ff, w_down->ne[1], n_rows, + w_down->nb[1], down->nb[1]/sizeof(float), args_down, stream); +} + +void ggml_cuda_mul_mat_dq_glu(ggml_backend_cuda_context & ctx, const ggml_tensor * glu, const ggml_tensor * down, ggml_tensor * dst, const ggml_tensor * res, const half * input_f16) { + if (down->op == GGML_OP_MUL_MAT_ID) { + GGML_ASSERT(res == nullptr && dst == down && input_f16 == nullptr); + ggml_cuda_mul_mat_id_dq_glu(ctx, glu, dst); + return; + } + const ggml_tensor * w_gate = glu->src[0]->src[0]; + const ggml_tensor * w_up = glu->src[1]->src[0]; + const ggml_tensor * w_down = down->src[0]; + const ggml_tensor * x = glu->src[0]->src[1]; + + const int64_t ne10 = x->ne[0]; + const int64_t ne11 = x->ne[1]; + const int64_t n_ff = w_gate->ne[1]; + + cudaStream_t stream = ctx.stream(); + + ggml_cuda_pool_alloc x_f16(ctx.pool()); + if (input_f16 == nullptr) { + x_f16.alloc(ne10*ne11); + ggml_get_to_fp16_cuda(GGML_TYPE_F32)(x->data, x_f16.get(), ne10*ne11, stream); + input_f16 = x_f16.get(); + } + + ggml_cuda_pool_alloc h_f16(ctx.pool(), n_ff*ne11); + // Compile-time activation keeps the existing SwiGLU store loop unchanged. + if (ggml_get_glu_op(glu) == GGML_GLU_OP_GEGLU) { + mul_mat_dq_switch_type(w_gate->type, (const char *) w_gate->data, input_f16, nullptr, ne10, n_ff, ne11, ne11, + w_gate->nb[1], n_ff, mmdq_ids_args{}, stream, mmdq_epi_args{(const char *) w_up->data, h_f16.get()}); + } else { + mul_mat_dq_switch_type(w_gate->type, (const char *) w_gate->data, input_f16, nullptr, ne10, n_ff, ne11, ne11, + w_gate->nb[1], n_ff, mmdq_ids_args{}, stream, mmdq_epi_args{(const char *) w_up->data, h_f16.get()}); + } + + const mmdq_epi_args epi = {nullptr, nullptr, nullptr, res ? (const float *) res->data : nullptr, res ? (int64_t) (res->nb[1]/sizeof(float)) : 0}; + mul_mat_dq_switch_type(w_down->type, (const char *) w_down->data, h_f16.get(), (float *) dst->data, n_ff, w_down->ne[1], ne11, ne11, + w_down->nb[1], dst->nb[1]/sizeof(float), mmdq_ids_args{}, stream, epi); +} + +void ggml_cuda_mul_mat_id_dq(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) { + GGML_TENSOR_BINARY_OP_LOCALS; + + cudaStream_t stream = ctx.stream(); + + const int64_t n_expert_used = ids->ne[0]; + const int64_t n_rows = ne12*n_expert_used; // compact rows, one per (token, used expert) + + ggml_cuda_pool_alloc ids_src1(ctx.pool(), n_rows); + ggml_cuda_pool_alloc ids_dst(ctx.pool(), n_rows); + ggml_cuda_pool_alloc expert_bounds(ctx.pool(), ne02 + 1); + ggml_cuda_pool_alloc tile_start(ctx.pool(), ne02 + 1); + + const int si1 = ids->nb[1] / ggml_element_size(ids); + const int sis1 = nb12 / nb11; + ggml_cuda_launch_mm_ids_helper((const int32_t *) ids->data, ids_src1.get(), ids_dst.get(), expert_bounds.get(), + ne02, ne12, n_expert_used, ne11, si1, sis1, /*write_inverse =*/ false, stream); + CUDA_CHECK(cudaGetLastError()); + + // src1 is converted once, the kernel reads its rows through ids_src1 + ggml_cuda_pool_alloc src1_f16(ctx.pool(), ne10*ne11*ne12); + ggml_get_to_fp16_cuda(GGML_TYPE_F32)(src1->data, src1_f16.get(), ne10*ne11*ne12, stream); + + const mmdq_ids_args args = {ids_src1.get(), ids_dst.get(), expert_bounds.get(), tile_start.get(), (int) ne02, (int64_t) nb02}; + mul_mat_dq_moe_switch(src0->type, (const char *) src0->data, src1_f16.get(), (float *) dst->data, ne00, ne01, n_rows, + nb01, nb1/sizeof(float), args, stream); +} diff --git a/ggml/src/ggml-cuda/mmdq.cuh b/ggml/src/ggml-cuda/mmdq.cuh new file mode 100644 index 000000000000..619ea61d5c3a --- /dev/null +++ b/ggml/src/ggml-cuda/mmdq.cuh @@ -0,0 +1,39 @@ +#pragma once + +#include "common.cuh" + +// Prefill matmul for K-quant weights (dense Q4_K/Q6_K, MoE Q4_K/Q5_K/Q6_K) on RDNA3.5: fp16 activations, weights dequantized to fp16 in the kernel, fp16 WMMA. +// Off by default. Env: +// GGML_CUDA_MMDQ non-zero = enable +// GGML_CUDA_MMDQ_MIN_BATCH minimum src1 columns (tokens) to use it, default 64 for Q4_K, 32 for Q6_K +// GGML_CUDA_MMDQ_NO_Q6K non-zero = keep Q6_K on the old path +// GGML_CUDA_MMDQ_Q5_0 non-zero = also use it for dense Q5_0 incl. the GGML_CUDA_MMDQ_GLU fusion (needs GGML_CUDA_MMDQ) +// GGML_CUDA_MMDQ_Q8_0 non-zero = also use it for dense Q8_0 (needs GGML_CUDA_MMDQ), GGML_CUDA_MMDQ_Q8_0_MIN_BATCH default 576, only for >= 512 rows +// GGML_CUDA_MMDQ_GLU non-zero = fuse gate + up + SWIGLU + down when all three use it (f16 hidden state, no f32 gate/up), dense or MoE (needs GGML_CUDA_MMDQ_MOE) +// GGML_CUDA_MMDQ_GEGLU non-zero = the GGML_CUDA_MMDQ_GLU fusion also for dense GEGLU (Gemma) +// GGML_CUDA_MMDQ_FUSE_BIAS non-zero = fuse a following bias add into the dense matmul +// GGML_CUDA_MMDQ_MOE non-zero = MUL_MAT_ID (Q4_K/Q5_K/Q6_K/Q5_1) with the weight streaming kernel, independent of GGML_CUDA_MMDQ +// GGML_CUDA_MMDQ_MOE_MIN_BATCH minimum number of tokens for it, default 16; used only with >= 16 tokens per expert on average, +// >= 32 for K-quant rows shorter than 2048, GGML_CUDA_MMDQ_MOE_ALL non-zero = for every shape (tests) + +bool ggml_cuda_should_use_mmdq(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst, int cc); + +// bias: [ne01] added to every column, res: same shape as dst added to dst (may alias dst) +void ggml_cuda_mul_mat_dq(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const float * bias = nullptr, const ggml_tensor * res = nullptr); + +bool ggml_cuda_should_use_mmdq_id(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, const ggml_tensor * dst, int cc); + +void ggml_cuda_mul_mat_id_dq(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst); + +// glu: SWIGLU of two MUL_MAT with the same src1, down: MUL_MAT of glu +bool ggml_cuda_should_use_mmdq_glu(const ggml_tensor * glu, const ggml_tensor * down, int cc); + +// dst: down, or the add of down and res (dense only) +void ggml_cuda_mul_mat_dq_glu(ggml_backend_cuda_context & ctx, const ggml_tensor * glu, const ggml_tensor * down, ggml_tensor * dst, const ggml_tensor * res = nullptr, const half * input_f16 = nullptr); + +// opt-in (GGML_CUDA_MMDQ_SHARE_SRC1): if node i of cgraph is an mmdq matmul and later mmdq matmuls read the same src1, convert src1 to f16 once for all of them +void ggml_cuda_mmdq_src1_share(ggml_backend_cuda_context & ctx, const ggml_cgraph * cgraph, int i); + +// drop the shared f16 src1 (end of the graph) +void ggml_cuda_mmdq_src1_release(ggml_backend_cuda_context & ctx); diff --git a/ggml/src/ggml-cuda/mmq-load-tiles.cuh b/ggml/src/ggml-cuda/mmq-load-tiles.cuh index f8505d646956..0dd7d98c1a2d 100644 --- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh +++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh @@ -1553,7 +1553,19 @@ template static __device const block_q6_K * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0, i, stride) + (threadIdx.x % (MMQ_TILE_NE_K/8)) / 4; #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_sc[i*sram_stride + threadIdx.x%4] = get_int_b2(bxi->scales, threadIdx.x % (MMQ_TILE_NE_K/8)); +#if defined(RDNA3_5) + if constexpr (GGML_CUDA_MMQ_Q6_K_SCALE_REUSE && J == 64) { + // The FP16 block scale times an int8 subscale is exact in FP32. + const int scales = get_int_b2(bxi->scales, threadIdx.x % (MMQ_TILE_NE_K/8)); +#pragma unroll + for (int l = 0; l < 4; ++l) { + ((float *) x_sc)[i*sram_stride + 4*(threadIdx.x%4) + l] = (int8_t) (scales >> (8*l)) * (float) bxi->d; + } + } else +#endif + { + x_sc[i*sram_stride + threadIdx.x%4] = get_int_b2(bxi->scales, threadIdx.x % (MMQ_TILE_NE_K/8)); + } #else x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + threadIdx.x%(MMQ_TILE_NE_K/8)] = get_int_b2(bxi->scales, threadIdx.x%(QI6_K/8)); #endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index 493c56fc7c9d..7410e12eae0f 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -169,7 +169,8 @@ static int ggml_cuda_mmq_get_J_best( void ggml_cuda_mul_mat_q( ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, - const ggml_cuda_mm_fusion_args_host * fusion) { + const ggml_cuda_mm_fusion_args_host * fusion, const void * packed) { + GGML_ASSERT(!packed || (!ids && !fusion)); GGML_ASSERT( src1->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID. @@ -275,13 +276,16 @@ void ggml_cuda_mul_mat_q( if (!ids) { const size_t nbytes_src1_q8_1 = ne13*ne12 * ne11*ne10_padded * y_block_size/y_values_per_block + src1_q8_1_padding; - ggml_cuda_pool_alloc src1_q8_1(ctx.pool(), nbytes_src1_q8_1); + ggml_cuda_pool_alloc src1_q8_1(ctx.pool()); + if (!packed) { + src1_q8_1.alloc(nbytes_src1_q8_1); + } ggml_cuda_pool_alloc src1_scale(ctx.pool()); if (src0->type == GGML_TYPE_NVFP4 && use_native_fp4) { src1_scale.alloc(ne13*ne12*ne11); } - { + if (!packed) { const int64_t s11 = src1->nb[1] / ts_src1; const int64_t s12 = src1->nb[2] / ts_src1; const int64_t s13 = src1->nb[3] / ts_src1; @@ -306,7 +310,7 @@ void ggml_cuda_mul_mat_q( const int64_t s13 = ne12*s12; const mmq_args args = { - src0_d, src0->type, (const int *) src1_q8_1.ptr, nullptr, nullptr, dst_d, + src0_d, src0->type, (const int *) (packed ? packed : src1_q8_1.ptr), nullptr, nullptr, dst_d, src0->type == GGML_TYPE_NVFP4 && use_native_fp4 ? src1_scale.ptr : nullptr, ne00, ne01, ne1, s01, ne11, s1, ne02, ne12, s02, s12, s2, diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index 9cbf197d0fd8..3f57ab9e66ee 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -125,12 +125,17 @@ struct tile_x_sizes { #define MMQ_TILE_Y_K (MMQ_TILE_NE_K + MMQ_TILE_NE_K / QI8_1) #define MMQ_TILE_Y_FP4_K MMQ_TILE_Y_K +#ifndef GGML_CUDA_MMQ_Q6_K_SCALE_REUSE +#define GGML_CUDA_MMQ_Q6_K_SCALE_REUSE 1 +#endif + enum ggml_cuda_mmq_sram_layout { GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, + GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K_SCALED, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, // MXFP4 and NVFP4 on Blackwell. GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, // Generic NVFP4 }; @@ -147,6 +152,8 @@ static constexpr __host__ __device__ int ggml_cuda_mmq_get_sram_stride(ggml_cuda return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4; case GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K: return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/QI6_K + MMQ_TILE_NE_K/8 + 7; + case GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K_SCALED: + return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/QI6_K + MMQ_TILE_NE_K/2 + 3; case GGML_CUDA_MMQ_SRAM_LAYOUT_FP4: return 2*MMQ_TILE_NE_K + 8 + 4; case GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4: @@ -161,6 +168,7 @@ static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1) % 8 static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K) % 8 == 4, "Wrong padding."); static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K) % 8 == 4, "Wrong padding."); static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K) % 8 == 4, "Wrong padding."); +static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K_SCALED) % 8 == 4, "Wrong padding."); static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_FP4) % 8 == 4, "Wrong padding."); static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4) % 8 == 4, "Wrong padding."); @@ -239,6 +247,9 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback); config.nthreads = 128; config.I = 64; + if (GGML_CUDA_MMQ_Q6_K_SCALE_REUSE && type == GGML_TYPE_Q6_K && J == 64) { + config.sram_layout = GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K_SCALED; + } return config; } @@ -533,9 +544,13 @@ static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q6_K_q8_1_mma_rdna3 #pragma unroll for (int l = 0; l < tile_C::ne; ++l) { const int i = i0 + n*tile_C::I + tile_C::get_i(l); - const int8_t * sc = (const int8_t *) (x_sc + i*sram_stride + k00/16); - sum[(j0/tile_C::J + n)*tile_C::ne + l] += - C.x[l] * sc[k01/4] * x_df[i*sram_stride] * dB0; + if constexpr (GGML_CUDA_MMQ_Q6_K_SCALE_REUSE && J == 64) { + const float * sc = (const float *) x_sc + i*sram_stride + k00/4; + sum[(j0/tile_C::J + n)*tile_C::ne + l] += (float) C.x[l] * sc[k01/4] * dB0; + } else { + const int8_t * sc = (const int8_t *) (x_sc + i*sram_stride + k00/16); + sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * sc[k01/4] * x_df[i*sram_stride] * dB0; + } } } @@ -2265,7 +2280,7 @@ extern DECL_MMQ_CASE_W4A4(GGML_TYPE_NVFP4); void ggml_cuda_mul_mat_q( ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, - const ggml_cuda_mm_fusion_args_host * fusion = nullptr); + const ggml_cuda_mm_fusion_args_host * fusion = nullptr, const void * packed = nullptr); bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts); diff --git a/ggml/src/ggml-cuda/quantize.cu b/ggml/src/ggml-cuda/quantize.cu index ea089a8c6249..7c5ade1278d4 100644 --- a/ggml/src/ggml-cuda/quantize.cu +++ b/ggml/src/ggml-cuda/quantize.cu @@ -1,4 +1,5 @@ #include "quantize.cuh" +#include "unary.cuh" #include #if defined(BLACKWELL_MMA_AVAILABLE) @@ -123,6 +124,78 @@ __device__ __forceinline__ uint8_t compute_e8m0_scale(float amax) { return static_cast(biased); } +// Keep producer rounding before quantization without a memory round trip. +static __device__ __forceinline__ float4 mmq_round_f32_input(float4 xi) { +#if defined(GGML_USE_HIP) + asm volatile("" : "+v"(xi.x), "+v"(xi.y), "+v"(xi.z), "+v"(xi.w)); +#else + asm volatile("" : "+f"(xi.x), "+f"(xi.y), "+f"(xi.z), "+f"(xi.w)); +#endif + return xi; +} + +struct mmq_q8_1_values { + char4 q; + float d; + float sum; +}; + +template +static __device__ __forceinline__ mmq_q8_1_values quantize_mmq_q8_1_values(const float4 xi) { + constexpr int vals_per_scale = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 64 : 32; + constexpr int vals_per_sum = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 16 : 32; + float amax = fabsf(xi.x); + amax = fmaxf(amax, fabsf(xi.y)); + amax = fmaxf(amax, fabsf(xi.z)); + amax = fmaxf(amax, fabsf(xi.w)); + + // Exchange max. abs. value between vals_per_scale/4 threads. +#pragma unroll + for (int offset = vals_per_scale/8; offset > 0; offset >>= 1) { + amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE)); + } + + float sum = 0.0f; + if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) { + sum = xi.x + xi.y + xi.z + xi.w; + + // Calculate sums across vals_per_sum/4 threads. +#pragma unroll + for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) { + sum += __shfl_xor_sync(0xFFFFFFFF, sum, offset, WARP_SIZE); + } + } + + const float d_inv = 127.0f / amax; + char4 q; + q.x = roundf(xi.x*d_inv); + q.y = roundf(xi.y*d_inv); + q.z = roundf(xi.z*d_inv); + q.w = roundf(xi.w*d_inv); + const float d = 1.0f / d_inv; + + return { q, d, sum }; +} + +template +static __device__ __forceinline__ void store_mmq_q8_1_values(block_q8_1_mmq * y, int iqs, const mmq_q8_1_values v) { + ((char4 *) y->qs)[iqs/4] = v.q; + if constexpr (ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6) { + if (iqs % 16 == 0 && iqs < 96) { + y->d2s6[2 + iqs/16] = v.sum; + if (iqs % 64 == 0) { + y->d2s6[iqs/64] = v.d; + } + } + } else if (iqs % 32 == 0) { + if constexpr (ds_layout == MMQ_Q8_1_DS_LAYOUT_DS4) { + y->ds4[iqs/32] = make_half2(v.d, v.sum); + } else { + y->d4[iqs/32] = v.d; + } + } +} + // scatter: grid over tokens, quantize once, write to all the token's compact rows template static __global__ void quantize_mmq_nvfp4( @@ -453,16 +526,19 @@ static __global__ void quantize_mmq_mxfp4(const float * __restrict__ x, } // scatter: grid over tokens, quantize once, write to all the token's compact rows -template +template static __global__ void quantize_mmq_q8_1( const float * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, const int64_t ne0, const int ne1, const int ne2, const int n_expert_used) { - constexpr int vals_per_scale = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 64 : 32; - constexpr int vals_per_sum = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 16 : 32; + static_assert(!interleave_k || !scatter, "interleaved K does not support scatter"); - const int64_t i0 = ((int64_t)blockDim.x*blockIdx.y + threadIdx.x)*4; + // Keep the feature chunks of each row adjacent in the grid. + const int row = interleave_k ? blockIdx.y : blockIdx.x; + const int block_k = interleave_k ? blockIdx.x : blockIdx.y; + + const int64_t i0 = ((int64_t)blockDim.x*block_k + threadIdx.x)*4; if (i0 >= ne0) { return; @@ -477,7 +553,7 @@ static __global__ void quantize_mmq_q8_1( } else { const int64_t i2 = blockIdx.z % ne2; const int64_t i3 = blockIdx.z / ne2; - const int64_t i01 = ids ? ids[blockIdx.x] : blockIdx.x; + const int64_t i01 = ids ? ids[row] : row; base_idx = i3*s03 + i2*s02 + i01*s01; } @@ -489,35 +565,7 @@ static __global__ void quantize_mmq_q8_1( // Load 4 floats per thread and calculate max. abs. value between them: const float4 xi = i0 < ne00 ? x4[(base_idx + i00)/4] : make_float4(0.0f, 0.0f, 0.0f, 0.0f); - float amax = fabsf(xi.x); - amax = fmaxf(amax, fabsf(xi.y)); - amax = fmaxf(amax, fabsf(xi.z)); - amax = fmaxf(amax, fabsf(xi.w)); - - // Exchange max. abs. value between vals_per_scale/4 threads. -#pragma unroll - for (int offset = vals_per_scale/8; offset > 0; offset >>= 1) { - amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE)); - } - - float sum; - if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) { - sum = xi.x + xi.y + xi.z + xi.w; - - // Calculate sums across vals_per_sum/4 threads. -#pragma unroll - for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) { - sum += __shfl_xor_sync(0xFFFFFFFF, sum, offset, WARP_SIZE); - } - } - - const float d_inv = 127.0f / amax; - char4 q; - q.x = roundf(xi.x*d_inv); - q.y = roundf(xi.y*d_inv); - q.z = roundf(xi.z*d_inv); - q.w = roundf(xi.w*d_inv); - const float d = 1.0f / d_inv; + const auto values = quantize_mmq_q8_1_values(xi); // write the block once (normal) or to each of the token's compact rows (scatter) const int nwrite = scatter ? n_expert_used : 1; @@ -529,31 +577,93 @@ static __global__ void quantize_mmq_q8_1( ib = k_block*ne1 + i; } else { const int64_t ib0 = blockIdx.z*((int64_t)gridDim.x*gridDim.y*blockDim.x/QK8_1); // first block of channel - ib = ib0 + k_block*ne1 + blockIdx.x; + ib = ib0 + k_block*ne1 + row; } - // Write back 4 int8 values as a single 32 bit value for better memory bandwidth: - char4 * yqs4 = (char4 *) y[ib].qs; - yqs4[iqs/4] = q; - - if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6) { - if (iqs % 16 == 0 && iqs < 96) { - y[ib].d2s6[2 + iqs/16] = sum; - if (iqs % 64 == 0) { - y[ib].d2s6[iqs/64] = d; - } - } - } else if (iqs % 32 == 0) { - if (ds_layout == MMQ_Q8_1_DS_LAYOUT_DS4) { - y[ib].ds4[iqs/32] = make_half2(d, sum); - } else { - y[ib].d4[iqs/32] = d; - } - } + store_mmq_q8_1_values(y + ib, iqs, values); } GGML_UNUSED(n_expert_used); } +template +static __global__ void rms_norm_mmq_q8_1(const float * x, const float * weight, block_q8_1_mmq * y, + int ncols, int nrows, float eps) { + const int row = blockIdx.x; + const int tid = threadIdx.x; + x += int64_t(row)*ncols; + float sum = 0.0f; + for (int col = tid; col < ncols; col += block_size) { + const float xi = x[col]; + sum += xi*xi; + } + extern __shared__ float shared_sum[]; + sum = block_reduce(sum, shared_sum); + const float mean = sum/ncols; + const float scale = rsqrtf(mean + eps); + for (int col = 4*tid; col < ncols; col += 4*block_size) { + const float4 v = ((const float4 *) x)[col/4]; + const float4 w = ((const float4 *) weight)[col/4]; + float4 xi = make_float4(scale*v.x*w.x, scale*v.y*w.y, scale*v.z*w.z, scale*v.w*w.w); + xi = mmq_round_f32_input(xi); + const auto values = quantize_mmq_q8_1_values(xi); + store_mmq_q8_1_values(y + int64_t(col/QK8_1_MMQ)*nrows + row, col % QK8_1_MMQ, values); + } +} + +void quantize_rms_norm_mmq_q8_1_cuda(const float * x, const float * weight, void * y, + int ncols, int nrows, float eps, cudaStream_t stream) { + GGML_ASSERT(ncols % MATRIX_ROW_PADDING == 0); + // Use the existing scalar RMS reduction before packing. + if (ncols < 1024) { + rms_norm_mmq_q8_1<256><<>>(x, weight, (block_q8_1_mmq *) y, ncols, nrows, eps); + } else { + rms_norm_mmq_q8_1<1024><<>>(x, weight, (block_q8_1_mmq *) y, ncols, nrows, eps); + } +} + +template +static __global__ void glu_mmq_q8_1(const float * x, const float * gate, block_q8_1_mmq * y, int ncols, int nrows, int ncols_padded) { + const int col = 4*(blockIdx.x*blockDim.x + threadIdx.x); + const int row = blockIdx.y; + if (col >= ncols_padded) { + return; + } + float4 xi = make_float4(0.0f, 0.0f, 0.0f, 0.0f); + if (col < ncols) { + const int64_t i = (int64_t(row)*ncols + col)/4; + const float4 a = ((const float4 *) x)[i]; + const float4 b = ((const float4 *) gate)[i]; + xi = mmq_round_f32_input(make_float4(op(a.x)*b.x, op(a.y)*b.y, op(a.z)*b.z, op(a.w)*b.w)); + } + const auto values = quantize_mmq_q8_1_values(xi); + store_mmq_q8_1_values(y + int64_t(col/QK8_1_MMQ)*nrows + row, col % QK8_1_MMQ, values); +} + +template +static void quantize_glu_mmq_q8_1_cuda_impl(const float * x, const float * gate, void * y, int ncols, int nrows, ggml_glu_op op, cudaStream_t stream) { + GGML_ASSERT(ncols % 4 == 0); + const int ncols_padded = GGML_PAD(ncols, MATRIX_ROW_PADDING); + const dim3 blocks((ncols_padded + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1)/(4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ), nrows); + switch (op) { + case GGML_GLU_OP_GEGLU: + glu_mmq_q8_1<<>>(x, gate, (block_q8_1_mmq *) y, ncols, nrows, ncols_padded); + break; + case GGML_GLU_OP_SWIGLU: + glu_mmq_q8_1<<>>(x, gate, (block_q8_1_mmq *) y, ncols, nrows, ncols_padded); + break; + default: + GGML_ABORT("unsupported GLU packing operation"); + } +} + +void quantize_glu_mmq_q8_1_cuda(const float * x, const float * gate, void * y, int ncols, int nrows, ggml_glu_op op, cudaStream_t stream) { + quantize_glu_mmq_q8_1_cuda_impl(x, gate, y, ncols, nrows, op, stream); +} + +void quantize_glu_mmq_q8_1_d4_cuda(const float * x, const float * gate, void * y, int ncols, int nrows, ggml_glu_op op, cudaStream_t stream) { + quantize_glu_mmq_q8_1_cuda_impl(x, gate, y, ncols, nrows, op, stream); +} + void quantize_row_q8_1_cuda( const float * x, const int32_t * ids, void * vy, const ggml_type type_src0, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, @@ -582,6 +692,17 @@ void quantize_mmq_q8_1_cuda( const int64_t block_num_y = (ne0 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ); const dim3 num_blocks(ne1, block_num_y, ne2*ne3); const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); +#if defined(GGML_USE_HIP) + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + if (GGML_CUDA_MMQ_OPT_ENABLED(GGML_CUDA_QUANTIZE_K_INTERLEAVE, true) && GGML_CUDA_CC_IS_RDNA3_5(cc) && type_src0 == GGML_TYPE_Q4_K && !ids && + block_num_y > 1 && s01 % 1024 == 0 && ne2 == 1 && ne3 == 1 && ne1 <= 65535) { + const dim3 num_blocks_interleaved(block_num_y, ne1, 1); + quantize_mmq_q8_1 + <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); + return; + } +#endif + switch (mmq_get_q8_1_ds_layout(type_src0)) { case MMQ_Q8_1_DS_LAYOUT_D4: quantize_mmq_q8_1 diff --git a/ggml/src/ggml-cuda/quantize.cuh b/ggml/src/ggml-cuda/quantize.cuh index 5f08dcbfe331..ad9556f297a8 100644 --- a/ggml/src/ggml-cuda/quantize.cuh +++ b/ggml/src/ggml-cuda/quantize.cuh @@ -68,3 +68,14 @@ void quantize_scatter_mmq_q8_1_cuda(const float * x, int64_t nrows_dst, int n_expert_used, cudaStream_t stream); + +void quantize_rms_norm_mmq_q8_1_cuda(const float * x, const float * weight, void * y, + int ncols, int nrows, float eps, cudaStream_t stream); + +void quantize_glu_mmq_q8_1_cuda(const float * x, const float * gate, void * y, int ncols, int nrows, ggml_glu_op op, cudaStream_t stream); + +void quantize_glu_mmq_q8_1_d4_cuda(const float * x, const float * gate, void * y, int ncols, int nrows, ggml_glu_op op, cudaStream_t stream); + +// Cache each ablation switch; set it before starting the process. +#define GGML_CUDA_MMQ_OPT_ENABLED(name, fallback) \ + ([] { static const bool enabled = [] { const char * v = std::getenv(#name); return v ? std::atoi(v) != 0 : (fallback); }(); return enabled; }()) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 3e52e4abe10c..fa5be9dc9788 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -5694,6 +5694,47 @@ static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats, bool skip = } // GGML_OP_MUL_MAT_ID +struct test_glu_mat : public test_mul_mat { + const bool extra_consumer; + const bool force_f32; + const ggml_glu_op glu_op; + test_glu_mat(ggml_type type, int64_t n, int64_t k = 512, bool extra_consumer = false, bool force_f32 = false, ggml_glu_op glu_op = GGML_GLU_OP_GEGLU) + : test_mul_mat(type, GGML_TYPE_F32, 256, n, k, {1, 1}, {1, 1}), extra_consumer(extra_consumer), force_f32(force_f32), glu_op(glu_op) {} + bool run_whole_graph() override { return true; } + std::string op_desc(ggml_tensor *) override { return glu_op == GGML_GLU_OP_GEGLU ? "MMQ_GEGLU_MAT" : "MMQ_SWIGLU_MAT"; } + std::string vars() override { return test_mul_mat::vars() + "," + VARS_TO_STR2(extra_consumer, force_f32); } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * mm = test_mul_mat::build_graph(ctx); + ggml_tensor * gate = ggml_dup_tensor(ctx, mm->src[1]); + mm->src[1] = glu_op == GGML_GLU_OP_GEGLU ? ggml_geglu_split(ctx, mm->src[1], gate) : ggml_swiglu_split(ctx, mm->src[1], gate); + if (force_f32) { + ggml_prec_set_acc(mm, GGML_PREC_F32); + } + return extra_consumer ? ggml_add(ctx, mm, ggml_mul_mat(ctx, mm->src[0], mm->src[1])) : mm; + } +}; + +struct test_rms_norm_mmq : public test_case { + const int64_t k, n; + const ggml_type type; + const bool extra_consumer; + test_rms_norm_mmq(int64_t k, int64_t n, ggml_type type = GGML_TYPE_Q4_K, bool extra_consumer = false) + : k(k), n(n), type(type), extra_consumer(extra_consumer) {} + bool run_whole_graph() override { return true; } + std::string op_desc(ggml_tensor *) override { return "RMS_NORM_MMQ"; } + std::string vars() override { return VARS_TO_STR4(k, n, type, extra_consumer); } + double max_nmse_err() override { return 5e-4; } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, n); + ggml_tensor * weight = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, k); + ggml_tensor * norm = ggml_mul(ctx, ggml_rms_norm(ctx, x, 1e-6f), weight); + ggml_tensor * w0 = ggml_new_tensor_2d(ctx, type, k, 256); + ggml_tensor * w1 = ggml_new_tensor_2d(ctx, type, k, 256); + ggml_tensor * out = ggml_add(ctx, ggml_mul_mat(ctx, w0, norm), ggml_mul_mat(ctx, w1, norm)); + return extra_consumer ? ggml_add(ctx, out, ggml_mul_mat(ctx, w0, norm)) : out; + } +}; + struct test_mul_mat_id : public test_case { const ggml_type type_a; const ggml_type type_b; @@ -8044,6 +8085,124 @@ struct test_mul_mat_vec_fusion : public test_case { } }; +struct test_mmdq_mat : public test_case { + const ggml_type type; + const int64_t k, m, n; + const bool bias, share, force_f32; + test_mmdq_mat(ggml_type type, int64_t k, int64_t m, int64_t n, bool bias = false, bool share = false, bool force_f32 = false) + : type(type), k(k), m(m), n(n), bias(bias), share(share), force_f32(force_f32) {} + std::string vars() override { return VARS_TO_STR7(type, k, m, n, bias, share, force_f32); } + std::string op_desc(ggml_tensor *) override { return "MMDQ_MAT"; } + bool run_whole_graph() override { return true; } + double max_nmse_err() override { return 5e-4; } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, n); + ggml_tensor * out = nullptr; + for (int i = 0; i < (share ? 3 : 1); ++i) { + ggml_tensor * w = ggml_new_tensor_2d(ctx, type, k, m); + ggml_tensor * mm = ggml_mul_mat(ctx, w, x); + if (force_f32) { ggml_prec_set_acc(mm, GGML_PREC_F32); } + ggml_tensor * y = bias ? ggml_add(ctx, mm, ggml_new_tensor_1d(ctx, GGML_TYPE_F32, m)) : mm; + out = out ? ggml_add(ctx, out, y) : y; + } + return out; + } +}; + +// MUL_MAT + SWIGLU + MUL_MAT +struct test_ffn_swiglu : public test_case { + const ggml_type type; + const ggml_type type_down; + const int64_t n_ff; + const int64_t n_embd; + const int64_t n_tokens; + const bool with_res; // add a residual to the output + const ggml_glu_op activation; + const bool extra_consumer; + + std::string vars() override { + return VARS_TO_STR8(type, type_down, n_ff, n_embd, n_tokens, with_res, activation, extra_consumer); + } + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return activation == GGML_GLU_OP_GEGLU ? "MMDQ_FFN_GEGLU" : "MMDQ_FFN_SWIGLU"; + } + + bool run_whole_graph() override { return true; } + + test_ffn_swiglu(ggml_type type, ggml_type type_down, int64_t n_ff, int64_t n_embd, int64_t n_tokens, bool with_res = false, ggml_glu_op activation = GGML_GLU_OP_SWIGLU, bool extra_consumer = false) + : type(type), type_down(type_down), n_ff(n_ff), n_embd(n_embd), n_tokens(n_tokens), with_res(with_res), activation(activation), extra_consumer(extra_consumer) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * cur = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_tokens); + ggml_tensor * gate = ggml_new_tensor_2d(ctx, type, n_embd, n_ff); + ggml_tensor * up = ggml_new_tensor_2d(ctx, type, n_embd, n_ff); + ggml_tensor * down = ggml_new_tensor_2d(ctx, type_down, n_ff, n_embd); + ggml_tensor * g = ggml_mul_mat(ctx, gate, cur); + ggml_tensor * u = ggml_mul_mat(ctx, up, cur); + ggml_tensor * h = activation == GGML_GLU_OP_GEGLU ? ggml_geglu_split(ctx, g, u) : ggml_swiglu_split(ctx, g, u); + ggml_tensor * out = ggml_mul_mat(ctx, down, h); + if (with_res) { + out = ggml_add(ctx, out, ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_tokens)); + } + if (extra_consumer) { + out = ggml_add(ctx, out, ggml_sum(ctx, g)); + } + ggml_set_name(out, "out"); + return out; + } + + double max_nmse_err() override { + return 5e-4; + } +}; + +struct test_moe_ffn_swiglu : public test_case { + const ggml_type type; + const ggml_type type_down; + const int n_experts; + const int n_used; + const int64_t n_ff; + const int64_t n_embd; + const int64_t n_tokens; + + std::string vars() override { + return VARS_TO_STR7(type, type_down, n_experts, n_used, n_ff, n_embd, n_tokens); + } + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "MOE_FFN_SWIGLU"; + } + + bool run_whole_graph() override { return true; } + + test_moe_ffn_swiglu(ggml_type type, ggml_type type_down, int n_experts, int n_used, int64_t n_ff, int64_t n_embd, int64_t n_tokens) + : type(type), type_down(type_down), n_experts(n_experts), n_used(n_used), n_ff(n_ff), n_embd(n_embd), n_tokens(n_tokens) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * cur = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd, 1, n_tokens); + ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_experts, n_tokens); + ids = ggml_view_2d(ctx, ids, n_used, n_tokens, ids->nb[1], 0); + ggml_tensor * gate = ggml_new_tensor_3d(ctx, type, n_embd, n_ff, n_experts); + ggml_tensor * up = ggml_new_tensor_3d(ctx, type, n_embd, n_ff, n_experts); + ggml_tensor * down = ggml_new_tensor_3d(ctx, type_down, n_ff, n_embd, n_experts); + ggml_tensor * h = ggml_swiglu_split(ctx, ggml_mul_mat_id(ctx, gate, cur, ids), ggml_mul_mat_id(ctx, up, cur, ids)); + ggml_tensor * out = ggml_mul_mat_id(ctx, down, h, ids); + ggml_set_name(out, "out"); + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + init_mul_mat_id_tensors(ctx, n_experts); + } + + double max_nmse_err() override { + return 5e-4; + } +}; + // GGML_OP_SUM struct test_sum : public test_case { const ggml_type type; @@ -10337,6 +10496,46 @@ static void add_rdna35_mmq_cases(std::vector> & test_ // Test cases for evaluation: should try to cover edge cases while using small input sizes to keep the runtime low static std::vector> make_test_cases_eval() { std::vector> test_cases; + for (ggml_type type : {GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K}) { + for (int64_t n : {65, 128, 258}) { + for (ggml_glu_op op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) { + test_cases.emplace_back(new test_glu_mat(type, n, 768, false, false, op)); + } + } + test_cases.emplace_back(new test_glu_mat(type, 258, 512, true)); + test_cases.emplace_back(new test_glu_mat(type, 128, 512, false, true)); + test_cases.emplace_back(new test_rms_norm_mmq(1536, 258, type)); + } + for (int64_t k : {512, 768, 1536, 6144}) { + for (int64_t n : {65, 128, 258}) { + test_cases.emplace_back(new test_rms_norm_mmq(k, n)); + } + } + test_cases.emplace_back(new test_rms_norm_mmq(1536, 258, GGML_TYPE_Q4_K, true)); + + + for (ggml_type type : {GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_Q5_0, GGML_TYPE_Q8_0}) { + for (int n : {1, 31, 64, 128, 513, 600}) { + test_cases.emplace_back(new test_mmdq_mat(type, 512, 1024, n)); + } + test_cases.emplace_back(new test_mmdq_mat(type, 768, 1089, 600, true)); + test_cases.emplace_back(new test_mmdq_mat(type, 512, 1024, 600, true, true)); + test_cases.emplace_back(new test_mmdq_mat(type, 512, 1024, 128, false, false, true)); + } + for (ggml_type type : {GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_Q5_0}) { + for (ggml_glu_op act : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) { + for (int n : {31, 64, 128, 513}) { + test_cases.emplace_back(new test_ffn_swiglu(type, GGML_TYPE_Q6_K, 1280, 1024, n, false, act)); + } + test_cases.emplace_back(new test_ffn_swiglu(type, GGML_TYPE_Q4_K, 2048, 1024, 200, true, act)); + test_cases.emplace_back(new test_ffn_swiglu(type, GGML_TYPE_Q6_K, 1280, 1024, 128, false, act, true)); + } + } + for (ggml_type type : {GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_Q5_1}) { + test_cases.emplace_back(new test_moe_ffn_swiglu(type, GGML_TYPE_Q6_K, 8, 4, 768, 1024, 129)); + test_cases.emplace_back(new test_moe_ffn_swiglu(type, GGML_TYPE_Q4_K, 16, 4, 512, 2048, 200)); + } + std::default_random_engine rng(0); // unary ops From 59ef62368122524a8a04672d14e93a55170e2955 Mon Sep 17 00:00:00 2001 From: lichang Date: Thu, 12 Nov 2026 19:18:25 -0700 Subject: [PATCH 2/3] hip: retain opt-in MMDQ token-tile grouping Keep the default grid order; enable grouping only when requested and the FP16 input exceeds 8 MiB. Gemma-2B long prompts benefit, while larger models require their measured setting. Assisted-by: Codex --- ggml/src/ggml-cuda/mmdq.cu | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-cuda/mmdq.cu b/ggml/src/ggml-cuda/mmdq.cu index 2c955fd1216b..584a29f36848 100644 --- a/ggml/src/ggml-cuda/mmdq.cu +++ b/ggml/src/ggml-cuda/mmdq.cu @@ -318,6 +318,7 @@ struct mmdq_epi_args { const float * bias; const float * res; int64_t stride_col_res; + int swizzle; // token tiles per block group, 0 keeps grid order }; // dst[t][o] = sum_k y[t][k] * x[o][k]; t: token (src1 column), o: output row (src0 row) @@ -350,8 +351,17 @@ static __global__ void mul_mat_dq( const int wave = tid / 32; const int wt = wave / NWO; const int wo = wave % NWO; - const int bx = blockIdx.x; - const int by = blockIdx.y; + // swizzle > 0 (dense): blocks run in groups of swizzle token tiles x all row tiles, so the activations of a group stay in cache + int bx = blockIdx.x; + int by = blockIdx.y; + if (!has_ids && epi.swizzle > 0) { + const int lin = blockIdx.y*gridDim.x + blockIdx.x; + const int per_g = epi.swizzle*gridDim.y; + const int x0 = (lin / per_g)*epi.swizzle; + const int gx = min(epi.swizzle, (int) gridDim.x - x0); + bx = x0 + (lin % per_g) % gx; + by = (lin % per_g) / gx; + } const int o0 = by*(glu ? BO/2 : BO); int t0 = bx*BT; @@ -793,7 +803,10 @@ static void mul_mat_dq_launch( if constexpr (glu && (has_ids || (BO/(16*NWO)) % 2 != 0)) { GGML_ABORT("mmdq config not supported with glu"); } else { - const mmdq_epi_args & epi = epi_in; + // opt-in (GGML_CUDA_MMDQ_SWIZZLE=n): group n token tiles, used when the activations (ncols_y x ncols_x f16) do not fit in 8 MiB + static const int swizzle = getenv("GGML_CUDA_MMDQ_SWIZZLE") ? atoi(getenv("GGML_CUDA_MMDQ_SWIZZLE")) : 0; + mmdq_epi_args epi = epi_in; + epi.swizzle = !has_ids && swizzle > 0 && int64_t(ncols_y)*ncols_x*2 > (8 << 20) ? swizzle : 0; int nblocks_t = (ncols_y + BT - 1)/BT; if constexpr (has_ids) { // ncols_y is the number of compact rows, each expert adds at most one partial block From 0b17fe0d6ec6ab7131b8f077d3c689b6e049de42 Mon Sep 17 00:00:00 2001 From: lichang Date: Thu, 12 Nov 2026 19:19:33 -0700 Subject: [PATCH 3/3] tests: cover partial MMDQ token-tile groups Assisted-by: Codex --- tests/test-backend-ops.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index fa5be9dc9788..a4d5a48f9231 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10496,6 +10496,8 @@ static void add_rdna35_mmq_cases(std::vector> & test_ // Test cases for evaluation: should try to cover edge cases while using small input sizes to keep the runtime low static std::vector> make_test_cases_eval() { std::vector> test_cases; + // The input exceeds 8 MiB and the final token-tile group is partial. + test_cases.emplace_back(new test_mmdq_mat(GGML_TYPE_Q4_K, 4352, 128, 1025)); for (ggml_type type : {GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K}) { for (int64_t n : {65, 128, 258}) { for (ggml_glu_op op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) {