From a3fc880f5ae3fdf456e553f43133699288d1d5cb Mon Sep 17 00:00:00 2001 From: lichang Date: Thu, 12 Nov 2026 20:13:32 -0700 Subject: [PATCH] hip: fuse Gemma scalar and GELU multiplications on RDNA3.5 Preserve FP32 arithmetic and the existing numerical kernels. Fuse eligible GELU products and normalized residual scaling with consumer, layout, and alias checks. Retain fallbacks for unsupported graphs. Assisted-by: Codex --- ggml/src/ggml-cuda/ggml-cuda.cu | 59 +++++++++++++++++++++++++++++++++ ggml/src/ggml-cuda/norm.cu | 50 +++++++++++++++++++--------- ggml/src/ggml-cuda/norm.cuh | 2 +- ggml/src/ggml-cuda/unary.cu | 4 +++ tests/test-backend-ops.cpp | 28 ++++++++++++++++ 5 files changed, 127 insertions(+), 16 deletions(-) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 929353e00ec9..ea6dcf7eea5a 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -4087,6 +4087,65 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph ggml_tensor * node = cgraph->nodes[i]; + // Exact contiguous aliases preserve element ownership; partial overlaps do not. + const auto safe_alias = [](const ggml_tensor * src, const ggml_tensor * dst) { + const uintptr_t src_addr = (uintptr_t) src->data; + const uintptr_t dst_addr = (uintptr_t) dst->data; + const bool disjoint = src_addr <= dst_addr ? dst_addr - src_addr >= ggml_nbytes(src) : + src_addr - dst_addr >= ggml_nbytes(dst); + return disjoint || (src_addr == dst_addr && src->type == dst->type && + ggml_are_same_shape(src, dst) && ggml_is_contiguous(src)); + }; + static const bool gelu_mul = []() { + const char * env = getenv("GGML_CUDA_FUSE_GELU_MUL"); + return !env || atoi(env) != 0; + }(); + if (gelu_mul && node->op == GGML_OP_UNARY && ggml_get_unary_op(node) == GGML_UNARY_OP_GELU && + node->type == GGML_TYPE_F32 && GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[cuda_ctx->device].cc)) { + int next = i + 1; + if (next < cgraph->n_nodes && cgraph->nodes[next]->op == GGML_OP_VIEW) { + ++next; + } + const int nodes[] = { i, next }; + const ggml_op ops[] = { GGML_OP_UNARY, GGML_OP_MUL }; + if (ggml_can_fuse_subgraph_ext(cgraph, nodes, 2, ops, &next, 1)) { + ggml_tensor * mul = cgraph->nodes[next]; + const ggml_tensor * other = mul->src[mul->src[0] == node ? 1 : 0]; + if ((mul->src[0] == node || mul->src[1] == node) && mul->type == GGML_TYPE_F32 && other->type == GGML_TYPE_F32 && + node->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous_1(node->src[0]) && + ggml_is_contiguous_1(other) && ggml_is_contiguous(mul) && ggml_are_same_shape(node, other) && + ggml_are_same_shape(node, mul) && safe_alias(node->src[0], mul) && safe_alias(other, mul)) { + ggml_cuda_op_unary_mul(*cuda_ctx, node, mul); + return next - i; + } + } + } + + static const bool rms_post_scale = []() { + const char * env = getenv("GGML_CUDA_FUSE_RMS_POST_SCALE"); + return !env || atoi(env) != 0; + }(); + if (rms_post_scale && node->op == GGML_OP_RMS_NORM && + GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[cuda_ctx->device].cc)) { + const int output = i + 3; + if (ggml_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD, GGML_OP_MUL }) && + ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) { + ggml_tensor * add = cgraph->nodes[i + 2]; + ggml_tensor * mul = cgraph->nodes[output]; + const ggml_tensor * scale = mul->src[mul->src[0] == add ? 1 : 0]; + const ggml_tensor * weight_node = cgraph->nodes[i + 1]; + const ggml_tensor * weight = weight_node->src[weight_node->src[0] == node ? 1 : 0]; + const ggml_tensor * residual = add->src[add->src[0] == weight_node ? 1 : 0]; + if ((mul->src[0] == add || mul->src[1] == add) && mul->type == GGML_TYPE_F32 && + scale->type == GGML_TYPE_F32 && ggml_nelements(scale) == 1 && + ggml_are_same_shape(add, mul) && ggml_is_contiguous(mul) && + safe_alias(node->src[0], mul) && safe_alias(weight, mul) && safe_alias(residual, mul) && safe_alias(scale, mul)) { + ggml_cuda_op_rms_norm_fused_add(*cuda_ctx, node, cgraph->nodes[i + 1], add, mul); + return 3; + } + } + } + if (node->op == GGML_OP_MUL_MAT_ID && cuda_ctx->stream_context().concurrent_events.empty() && ggml_cuda_match_shared_expert(cgraph, i, i + 3) && ggml_cuda_should_fuse_mul_mat_vec_q(cgraph->nodes[i + 2]->src[1])) { diff --git a/ggml/src/ggml-cuda/norm.cu b/ggml/src/ggml-cuda/norm.cu index 3f0ea6f2a474..b6bb88deb86a 100644 --- a/ggml/src/ggml-cuda/norm.cu +++ b/ggml/src/ggml-cuda/norm.cu @@ -81,7 +81,7 @@ static __global__ void group_norm_f32(const float * x, float * dst, const int gr } } -template +template static __global__ void rms_norm_f32(const float * x, float * dst, const int ncols, @@ -107,7 +107,7 @@ static __global__ void rms_norm_f32(const float * x, const uint3 add_nrows_packed = make_uint3(0, 0, 0), const uint3 add_nchannels_packed = make_uint3(0, 0, 0), const uint3 add_nsamples_packed = make_uint3(0, 0, 0), - const float scale_out = 1.0f) { + const float scale_out = 1.0f, const float * post_scale = nullptr) { ggml_cuda_pdl_lc(); const int nrows = gridDim.x; const int row = blockIdx.x; @@ -188,6 +188,13 @@ static __global__ void rms_norm_f32(const float * x, r.z *= scale_out; r.w *= scale_out; } + if constexpr (do_post_scale) { + const float factor = post_scale[0]; + r.x *= factor; + r.y *= factor; + r.z *= factor; + r.w *= factor; + } dst4[i] = r; } } @@ -209,7 +216,8 @@ static __global__ void rms_norm_f32(const float * x, if constexpr (do_multiply && do_add) { const int mul_col = fastmodulo(col, mul_ncols_packed); const int add_col = fastmodulo(col, add_ncols_packed); - dstc[col] = scale * xc[col] * mulc[mul_col] + addc[add_col]; + const float value = scale * xc[col] * mulc[mul_col] + addc[add_col]; + dstc[col] = do_post_scale ? value * post_scale[0] : value; } else if constexpr (do_multiply) { const int mul_col = fastmodulo(col, mul_ncols_packed); dstc[col] = scale * xc[col] * mulc[mul_col]; @@ -387,15 +395,15 @@ static void group_norm_f32_cuda( do { \ const dim3 block_dims(BLOCK_SIZE, 1, 1); \ const ggml_cuda_kernel_launch_params launch_params{ blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float) : 0, stream }; \ - ggml_cuda_kernel_launch(rms_norm_f32, launch_params, \ + ggml_cuda_kernel_launch(rms_norm_f32, launch_params, \ x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, \ mul, mul_stride_row, mul_stride_channel, mul_stride_sample, \ mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, \ add, add_stride_row, add_stride_channel, add_stride_sample, \ - add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, scale_out); \ + add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, scale_out, post_scale); \ } while (0) -template +template static void rms_norm_f32_dispatch( const dim3 blocks_num, const ggml_tensor * x_t, float * dst, const int ncols, const int nchannels, const int nsamples, @@ -404,7 +412,7 @@ static void rms_norm_f32_dispatch( const uint3 mul_ncols_packed, const uint3 mul_nrows_packed, const uint3 mul_nchannels_packed, const uint3 mul_nsamples_packed, const ggml_tensor * add_t, const int64_t add_stride_row, const int64_t add_stride_channel, const int64_t add_stride_sample, const uint3 add_ncols_packed, const uint3 add_nrows_packed, const uint3 add_nchannels_packed, const uint3 add_nsamples_packed, - cudaStream_t stream, const float scale_out = 1.0f) { + cudaStream_t stream, const float scale_out = 1.0f, const float * post_scale = nullptr) { static_assert(!do_add || do_multiply, "fusing add is not supported without multiplying"); static_assert(!do_scale || !do_multiply, "fusing scale is not supported with multiplying"); @@ -462,6 +470,7 @@ static void rms_norm_f32_cuda( stream, scale_out); } +template static void rms_norm_mul_f32_cuda(const ggml_tensor * x_t, const ggml_tensor * mul_t, const ggml_tensor * add_t, @@ -488,7 +497,7 @@ static void rms_norm_mul_f32_cuda(const ggml_tensor * x_t, const uint32_t add_nchannels, const uint32_t add_nsamples, const float eps, - cudaStream_t stream) { + cudaStream_t stream, const float * post_scale = nullptr) { const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX)); if (mul_t == nullptr) { rms_norm_f32_cuda(x_t, dst, ncols, nrows, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, stream); @@ -511,12 +520,12 @@ static void rms_norm_mul_f32_cuda(const ggml_tensor * x_t, const uint3 add_nrows_packed = init_fastdiv_values(add_nrows); const uint3 add_nchannels_packed = init_fastdiv_values(add_nchannels); const uint3 add_nsamples_packed = init_fastdiv_values(add_nsamples); - rms_norm_f32_dispatch( + rms_norm_f32_dispatch( blocks_num, x_t, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, mul_t, mul_stride_row, mul_stride_channel, mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, add_t, add_stride_row, add_stride_channel, add_stride_sample, - add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, stream); + add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, stream, 1.0f, post_scale); } } @@ -693,10 +702,11 @@ void ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * eps, stream); } -void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, +template +static void rms_norm_fused_add_impl(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * mul_tensor, - ggml_tensor * add_tensor) { + ggml_tensor * add_tensor, ggml_tensor * scale_tensor) { const ggml_tensor * rms_norm_src = (ggml_tensor *) dst->src[0]; float eps = 0.0f; @@ -722,7 +732,7 @@ void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, GGML_ASSERT(false); } - float * dst_d = (float *) add_tensor->data; + float * dst_d = (float *) (scale_tensor ? scale_tensor->data : add_tensor->data); cudaStream_t stream = ctx.stream(); GGML_ASSERT(rms_norm_src->type == GGML_TYPE_F32); @@ -764,14 +774,24 @@ void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, const int add_nchannels = add_src->ne[2]; const int add_nsamples = add_src->ne[3]; - rms_norm_mul_f32_cuda(rms_norm_src, mul_src,add_src,dst_d, + const float * post_scale = scale_tensor ? (const float *) scale_tensor->src[scale_tensor->src[0] == add_tensor ? 1 : 0]->data : nullptr; + rms_norm_mul_f32_cuda(rms_norm_src, mul_src,add_src,dst_d, ne00,ne01, ne02, ne03, /*s00*/ s01, s02, s03, /*mul_s00*/ mul_s01, mul_s02, mul_s03, mul_ncols, mul_nrows, mul_nchannels, mul_nsamples, /*add_s00*/ add_s01, add_s02, add_s03, add_ncols, add_nrows, add_nchannels, add_nsamples, - eps, stream); + eps, stream, post_scale); +} + +void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst, + ggml_tensor * mul_tensor, ggml_tensor * add_tensor, ggml_tensor * scale_tensor) { + if (scale_tensor) { + rms_norm_fused_add_impl(ctx, dst, mul_tensor, add_tensor, scale_tensor); + } else { + rms_norm_fused_add_impl(ctx, dst, mul_tensor, add_tensor, nullptr); + } } void ggml_cuda_op_rms_norm_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { diff --git a/ggml/src/ggml-cuda/norm.cuh b/ggml/src/ggml-cuda/norm.cuh index 95618df5fd4d..a53d3db6715e 100644 --- a/ggml/src/ggml-cuda/norm.cuh +++ b/ggml/src/ggml-cuda/norm.cuh @@ -13,7 +13,7 @@ void ggml_cuda_op_rms_norm_scale_fused(ggml_backend_cuda_context & ctx, ggml_ten void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * mul_tensor, - ggml_tensor * add_tensor); + ggml_tensor * add_tensor, ggml_tensor * scale_tensor = nullptr); void ggml_cuda_op_rms_norm_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/unary.cu b/ggml/src/ggml-cuda/unary.cu index b3b6aa48d3ac..ec3f3e06380b 100644 --- a/ggml/src/ggml-cuda/unary.cu +++ b/ggml/src/ggml-cuda/unary.cu @@ -747,6 +747,10 @@ static void ggml_cuda_op_unary_mul_impl(ggml_backend_cuda_context & ctx, ggml_te void ggml_cuda_op_unary_mul(ggml_backend_cuda_context & ctx, ggml_tensor * unary_node, ggml_tensor * mul_node) { switch (ggml_get_unary_op(unary_node)) { + case GGML_UNARY_OP_GELU: + GGML_ASSERT(unary_node->type == GGML_TYPE_F32); + ggml_cuda_op_unary_mul_impl(ctx, unary_node, mul_node); + break; case GGML_UNARY_OP_SILU: ggml_cuda_op_unary_mul_impl(ctx, unary_node, mul_node); break; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 3e52e4abe10c..1a7d14abcfb7 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -2941,6 +2941,28 @@ struct test_rope_set_rows : public test_case { } }; +struct test_rms_post_scale : public test_case { + const int width; + const bool extra, inplace; + test_rms_post_scale(int width, bool extra, bool inplace = false) : width(width), extra(extra), inplace(inplace) {} + std::string vars() override { return VARS_TO_STR3(width, extra, inplace); } + std::string op_desc(ggml_tensor *) override { return "RMS_POST_SCALE"; } + bool run_whole_graph() override { return true; } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, width, 129); + ggml_tensor * weight = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, width); + ggml_tensor * residual = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, width, 129); + ggml_tensor * scalar = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); + if (inplace) { + residual = ggml_scale(ctx, residual, 0.5f); + } + ggml_tensor * normalized = ggml_mul(ctx, ggml_rms_norm(ctx, x, 1e-6f), weight); + ggml_tensor * add = inplace ? ggml_add_inplace(ctx, residual, normalized) : ggml_add(ctx, normalized, residual); + ggml_tensor * out = inplace ? ggml_mul_inplace(ctx, add, scalar) : ggml_mul(ctx, add, scalar); + return extra ? ggml_add(ctx, out, add) : out; + } +}; + // GGML_OP_RMS_NORM with optional GGML_OP_MUL, GGML_OP_ROPE, GGML_OP_VIEW and GGML_OP_SET_ROWS struct test_rms_norm_mul_rope : public test_case { const std::array ne; @@ -10337,6 +10359,12 @@ static void add_rdna35_mmq_cases(std::vector> & test_ // Test cases for evaluation: should try to cover edge cases while using small input sizes to keep the runtime low static std::vector> make_test_cases_eval() { std::vector> test_cases; + for (int width : {64, 193, 1536, 2048}) { + for (bool extra : {false, true}) { + test_cases.emplace_back(new test_rms_post_scale(width, extra)); + } + test_cases.emplace_back(new test_rms_post_scale(width, false, true)); + } std::default_random_engine rng(0); // unary ops