diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 929353e00ec9..ea6dcf7eea5a 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -4087,6 +4087,65 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph ggml_tensor * node = cgraph->nodes[i]; + // Exact contiguous aliases preserve element ownership; partial overlaps do not. + const auto safe_alias = [](const ggml_tensor * src, const ggml_tensor * dst) { + const uintptr_t src_addr = (uintptr_t) src->data; + const uintptr_t dst_addr = (uintptr_t) dst->data; + const bool disjoint = src_addr <= dst_addr ? dst_addr - src_addr >= ggml_nbytes(src) : + src_addr - dst_addr >= ggml_nbytes(dst); + return disjoint || (src_addr == dst_addr && src->type == dst->type && + ggml_are_same_shape(src, dst) && ggml_is_contiguous(src)); + }; + static const bool gelu_mul = []() { + const char * env = getenv("GGML_CUDA_FUSE_GELU_MUL"); + return !env || atoi(env) != 0; + }(); + if (gelu_mul && node->op == GGML_OP_UNARY && ggml_get_unary_op(node) == GGML_UNARY_OP_GELU && + node->type == GGML_TYPE_F32 && GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[cuda_ctx->device].cc)) { + int next = i + 1; + if (next < cgraph->n_nodes && cgraph->nodes[next]->op == GGML_OP_VIEW) { + ++next; + } + const int nodes[] = { i, next }; + const ggml_op ops[] = { GGML_OP_UNARY, GGML_OP_MUL }; + if (ggml_can_fuse_subgraph_ext(cgraph, nodes, 2, ops, &next, 1)) { + ggml_tensor * mul = cgraph->nodes[next]; + const ggml_tensor * other = mul->src[mul->src[0] == node ? 1 : 0]; + if ((mul->src[0] == node || mul->src[1] == node) && mul->type == GGML_TYPE_F32 && other->type == GGML_TYPE_F32 && + node->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous_1(node->src[0]) && + ggml_is_contiguous_1(other) && ggml_is_contiguous(mul) && ggml_are_same_shape(node, other) && + ggml_are_same_shape(node, mul) && safe_alias(node->src[0], mul) && safe_alias(other, mul)) { + ggml_cuda_op_unary_mul(*cuda_ctx, node, mul); + return next - i; + } + } + } + + static const bool rms_post_scale = []() { + const char * env = getenv("GGML_CUDA_FUSE_RMS_POST_SCALE"); + return !env || atoi(env) != 0; + }(); + if (rms_post_scale && node->op == GGML_OP_RMS_NORM && + GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[cuda_ctx->device].cc)) { + const int output = i + 3; + if (ggml_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD, GGML_OP_MUL }) && + ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) { + ggml_tensor * add = cgraph->nodes[i + 2]; + ggml_tensor * mul = cgraph->nodes[output]; + const ggml_tensor * scale = mul->src[mul->src[0] == add ? 1 : 0]; + const ggml_tensor * weight_node = cgraph->nodes[i + 1]; + const ggml_tensor * weight = weight_node->src[weight_node->src[0] == node ? 1 : 0]; + const ggml_tensor * residual = add->src[add->src[0] == weight_node ? 1 : 0]; + if ((mul->src[0] == add || mul->src[1] == add) && mul->type == GGML_TYPE_F32 && + scale->type == GGML_TYPE_F32 && ggml_nelements(scale) == 1 && + ggml_are_same_shape(add, mul) && ggml_is_contiguous(mul) && + safe_alias(node->src[0], mul) && safe_alias(weight, mul) && safe_alias(residual, mul) && safe_alias(scale, mul)) { + ggml_cuda_op_rms_norm_fused_add(*cuda_ctx, node, cgraph->nodes[i + 1], add, mul); + return 3; + } + } + } + if (node->op == GGML_OP_MUL_MAT_ID && cuda_ctx->stream_context().concurrent_events.empty() && ggml_cuda_match_shared_expert(cgraph, i, i + 3) && ggml_cuda_should_fuse_mul_mat_vec_q(cgraph->nodes[i + 2]->src[1])) { diff --git a/ggml/src/ggml-cuda/norm.cu b/ggml/src/ggml-cuda/norm.cu index 3f0ea6f2a474..b6bb88deb86a 100644 --- a/ggml/src/ggml-cuda/norm.cu +++ b/ggml/src/ggml-cuda/norm.cu @@ -81,7 +81,7 @@ static __global__ void group_norm_f32(const float * x, float * dst, const int gr } } -template +template static __global__ void rms_norm_f32(const float * x, float * dst, const int ncols, @@ -107,7 +107,7 @@ static __global__ void rms_norm_f32(const float * x, const uint3 add_nrows_packed = make_uint3(0, 0, 0), const uint3 add_nchannels_packed = make_uint3(0, 0, 0), const uint3 add_nsamples_packed = make_uint3(0, 0, 0), - const float scale_out = 1.0f) { + const float scale_out = 1.0f, const float * post_scale = nullptr) { ggml_cuda_pdl_lc(); const int nrows = gridDim.x; const int row = blockIdx.x; @@ -188,6 +188,13 @@ static __global__ void rms_norm_f32(const float * x, r.z *= scale_out; r.w *= scale_out; } + if constexpr (do_post_scale) { + const float factor = post_scale[0]; + r.x *= factor; + r.y *= factor; + r.z *= factor; + r.w *= factor; + } dst4[i] = r; } } @@ -209,7 +216,8 @@ static __global__ void rms_norm_f32(const float * x, if constexpr (do_multiply && do_add) { const int mul_col = fastmodulo(col, mul_ncols_packed); const int add_col = fastmodulo(col, add_ncols_packed); - dstc[col] = scale * xc[col] * mulc[mul_col] + addc[add_col]; + const float value = scale * xc[col] * mulc[mul_col] + addc[add_col]; + dstc[col] = do_post_scale ? value * post_scale[0] : value; } else if constexpr (do_multiply) { const int mul_col = fastmodulo(col, mul_ncols_packed); dstc[col] = scale * xc[col] * mulc[mul_col]; @@ -387,15 +395,15 @@ static void group_norm_f32_cuda( do { \ const dim3 block_dims(BLOCK_SIZE, 1, 1); \ const ggml_cuda_kernel_launch_params launch_params{ blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float) : 0, stream }; \ - ggml_cuda_kernel_launch(rms_norm_f32, launch_params, \ + ggml_cuda_kernel_launch(rms_norm_f32, launch_params, \ x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, \ mul, mul_stride_row, mul_stride_channel, mul_stride_sample, \ mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, \ add, add_stride_row, add_stride_channel, add_stride_sample, \ - add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, scale_out); \ + add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, scale_out, post_scale); \ } while (0) -template +template static void rms_norm_f32_dispatch( const dim3 blocks_num, const ggml_tensor * x_t, float * dst, const int ncols, const int nchannels, const int nsamples, @@ -404,7 +412,7 @@ static void rms_norm_f32_dispatch( const uint3 mul_ncols_packed, const uint3 mul_nrows_packed, const uint3 mul_nchannels_packed, const uint3 mul_nsamples_packed, const ggml_tensor * add_t, const int64_t add_stride_row, const int64_t add_stride_channel, const int64_t add_stride_sample, const uint3 add_ncols_packed, const uint3 add_nrows_packed, const uint3 add_nchannels_packed, const uint3 add_nsamples_packed, - cudaStream_t stream, const float scale_out = 1.0f) { + cudaStream_t stream, const float scale_out = 1.0f, const float * post_scale = nullptr) { static_assert(!do_add || do_multiply, "fusing add is not supported without multiplying"); static_assert(!do_scale || !do_multiply, "fusing scale is not supported with multiplying"); @@ -462,6 +470,7 @@ static void rms_norm_f32_cuda( stream, scale_out); } +template static void rms_norm_mul_f32_cuda(const ggml_tensor * x_t, const ggml_tensor * mul_t, const ggml_tensor * add_t, @@ -488,7 +497,7 @@ static void rms_norm_mul_f32_cuda(const ggml_tensor * x_t, const uint32_t add_nchannels, const uint32_t add_nsamples, const float eps, - cudaStream_t stream) { + cudaStream_t stream, const float * post_scale = nullptr) { const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX)); if (mul_t == nullptr) { rms_norm_f32_cuda(x_t, dst, ncols, nrows, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, stream); @@ -511,12 +520,12 @@ static void rms_norm_mul_f32_cuda(const ggml_tensor * x_t, const uint3 add_nrows_packed = init_fastdiv_values(add_nrows); const uint3 add_nchannels_packed = init_fastdiv_values(add_nchannels); const uint3 add_nsamples_packed = init_fastdiv_values(add_nsamples); - rms_norm_f32_dispatch( + rms_norm_f32_dispatch( blocks_num, x_t, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, mul_t, mul_stride_row, mul_stride_channel, mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, add_t, add_stride_row, add_stride_channel, add_stride_sample, - add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, stream); + add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, stream, 1.0f, post_scale); } } @@ -693,10 +702,11 @@ void ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * eps, stream); } -void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, +template +static void rms_norm_fused_add_impl(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * mul_tensor, - ggml_tensor * add_tensor) { + ggml_tensor * add_tensor, ggml_tensor * scale_tensor) { const ggml_tensor * rms_norm_src = (ggml_tensor *) dst->src[0]; float eps = 0.0f; @@ -722,7 +732,7 @@ void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, GGML_ASSERT(false); } - float * dst_d = (float *) add_tensor->data; + float * dst_d = (float *) (scale_tensor ? scale_tensor->data : add_tensor->data); cudaStream_t stream = ctx.stream(); GGML_ASSERT(rms_norm_src->type == GGML_TYPE_F32); @@ -764,14 +774,24 @@ void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, const int add_nchannels = add_src->ne[2]; const int add_nsamples = add_src->ne[3]; - rms_norm_mul_f32_cuda(rms_norm_src, mul_src,add_src,dst_d, + const float * post_scale = scale_tensor ? (const float *) scale_tensor->src[scale_tensor->src[0] == add_tensor ? 1 : 0]->data : nullptr; + rms_norm_mul_f32_cuda(rms_norm_src, mul_src,add_src,dst_d, ne00,ne01, ne02, ne03, /*s00*/ s01, s02, s03, /*mul_s00*/ mul_s01, mul_s02, mul_s03, mul_ncols, mul_nrows, mul_nchannels, mul_nsamples, /*add_s00*/ add_s01, add_s02, add_s03, add_ncols, add_nrows, add_nchannels, add_nsamples, - eps, stream); + eps, stream, post_scale); +} + +void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst, + ggml_tensor * mul_tensor, ggml_tensor * add_tensor, ggml_tensor * scale_tensor) { + if (scale_tensor) { + rms_norm_fused_add_impl(ctx, dst, mul_tensor, add_tensor, scale_tensor); + } else { + rms_norm_fused_add_impl(ctx, dst, mul_tensor, add_tensor, nullptr); + } } void ggml_cuda_op_rms_norm_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { diff --git a/ggml/src/ggml-cuda/norm.cuh b/ggml/src/ggml-cuda/norm.cuh index 95618df5fd4d..a53d3db6715e 100644 --- a/ggml/src/ggml-cuda/norm.cuh +++ b/ggml/src/ggml-cuda/norm.cuh @@ -13,7 +13,7 @@ void ggml_cuda_op_rms_norm_scale_fused(ggml_backend_cuda_context & ctx, ggml_ten void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * mul_tensor, - ggml_tensor * add_tensor); + ggml_tensor * add_tensor, ggml_tensor * scale_tensor = nullptr); void ggml_cuda_op_rms_norm_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/unary.cu b/ggml/src/ggml-cuda/unary.cu index b3b6aa48d3ac..ec3f3e06380b 100644 --- a/ggml/src/ggml-cuda/unary.cu +++ b/ggml/src/ggml-cuda/unary.cu @@ -747,6 +747,10 @@ static void ggml_cuda_op_unary_mul_impl(ggml_backend_cuda_context & ctx, ggml_te void ggml_cuda_op_unary_mul(ggml_backend_cuda_context & ctx, ggml_tensor * unary_node, ggml_tensor * mul_node) { switch (ggml_get_unary_op(unary_node)) { + case GGML_UNARY_OP_GELU: + GGML_ASSERT(unary_node->type == GGML_TYPE_F32); + ggml_cuda_op_unary_mul_impl(ctx, unary_node, mul_node); + break; case GGML_UNARY_OP_SILU: ggml_cuda_op_unary_mul_impl(ctx, unary_node, mul_node); break; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 3e52e4abe10c..1a7d14abcfb7 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -2941,6 +2941,28 @@ struct test_rope_set_rows : public test_case { } }; +struct test_rms_post_scale : public test_case { + const int width; + const bool extra, inplace; + test_rms_post_scale(int width, bool extra, bool inplace = false) : width(width), extra(extra), inplace(inplace) {} + std::string vars() override { return VARS_TO_STR3(width, extra, inplace); } + std::string op_desc(ggml_tensor *) override { return "RMS_POST_SCALE"; } + bool run_whole_graph() override { return true; } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, width, 129); + ggml_tensor * weight = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, width); + ggml_tensor * residual = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, width, 129); + ggml_tensor * scalar = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); + if (inplace) { + residual = ggml_scale(ctx, residual, 0.5f); + } + ggml_tensor * normalized = ggml_mul(ctx, ggml_rms_norm(ctx, x, 1e-6f), weight); + ggml_tensor * add = inplace ? ggml_add_inplace(ctx, residual, normalized) : ggml_add(ctx, normalized, residual); + ggml_tensor * out = inplace ? ggml_mul_inplace(ctx, add, scalar) : ggml_mul(ctx, add, scalar); + return extra ? ggml_add(ctx, out, add) : out; + } +}; + // GGML_OP_RMS_NORM with optional GGML_OP_MUL, GGML_OP_ROPE, GGML_OP_VIEW and GGML_OP_SET_ROWS struct test_rms_norm_mul_rope : public test_case { const std::array ne; @@ -10337,6 +10359,12 @@ static void add_rdna35_mmq_cases(std::vector> & test_ // Test cases for evaluation: should try to cover edge cases while using small input sizes to keep the runtime low static std::vector> make_test_cases_eval() { std::vector> test_cases; + for (int width : {64, 193, 1536, 2048}) { + for (bool extra : {false, true}) { + test_cases.emplace_back(new test_rms_post_scale(width, extra)); + } + test_cases.emplace_back(new test_rms_post_scale(width, false, true)); + } std::default_random_engine rng(0); // unary ops