Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions ggml/src/ggml-cuda/ggml-cuda.cu
Original file line number Diff line number Diff line change
Expand Up @@ -4087,6 +4087,65 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph

ggml_tensor * node = cgraph->nodes[i];

// Exact contiguous aliases preserve element ownership; partial overlaps do not.
const auto safe_alias = [](const ggml_tensor * src, const ggml_tensor * dst) {
const uintptr_t src_addr = (uintptr_t) src->data;
const uintptr_t dst_addr = (uintptr_t) dst->data;
const bool disjoint = src_addr <= dst_addr ? dst_addr - src_addr >= ggml_nbytes(src) :
src_addr - dst_addr >= ggml_nbytes(dst);
return disjoint || (src_addr == dst_addr && src->type == dst->type &&
ggml_are_same_shape(src, dst) && ggml_is_contiguous(src));
};
static const bool gelu_mul = []() {
const char * env = getenv("GGML_CUDA_FUSE_GELU_MUL");
return !env || atoi(env) != 0;
}();
if (gelu_mul && node->op == GGML_OP_UNARY && ggml_get_unary_op(node) == GGML_UNARY_OP_GELU &&
node->type == GGML_TYPE_F32 && GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[cuda_ctx->device].cc)) {
int next = i + 1;
if (next < cgraph->n_nodes && cgraph->nodes[next]->op == GGML_OP_VIEW) {
++next;
}
const int nodes[] = { i, next };
const ggml_op ops[] = { GGML_OP_UNARY, GGML_OP_MUL };
if (ggml_can_fuse_subgraph_ext(cgraph, nodes, 2, ops, &next, 1)) {
ggml_tensor * mul = cgraph->nodes[next];
const ggml_tensor * other = mul->src[mul->src[0] == node ? 1 : 0];
if ((mul->src[0] == node || mul->src[1] == node) && mul->type == GGML_TYPE_F32 && other->type == GGML_TYPE_F32 &&
node->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous_1(node->src[0]) &&
ggml_is_contiguous_1(other) && ggml_is_contiguous(mul) && ggml_are_same_shape(node, other) &&
ggml_are_same_shape(node, mul) && safe_alias(node->src[0], mul) && safe_alias(other, mul)) {
ggml_cuda_op_unary_mul(*cuda_ctx, node, mul);
return next - i;
}
}
}

static const bool rms_post_scale = []() {
const char * env = getenv("GGML_CUDA_FUSE_RMS_POST_SCALE");
return !env || atoi(env) != 0;
}();
if (rms_post_scale && node->op == GGML_OP_RMS_NORM &&
GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[cuda_ctx->device].cc)) {
const int output = i + 3;
if (ggml_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD, GGML_OP_MUL }) &&
ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) {
ggml_tensor * add = cgraph->nodes[i + 2];
ggml_tensor * mul = cgraph->nodes[output];
const ggml_tensor * scale = mul->src[mul->src[0] == add ? 1 : 0];
const ggml_tensor * weight_node = cgraph->nodes[i + 1];
const ggml_tensor * weight = weight_node->src[weight_node->src[0] == node ? 1 : 0];
const ggml_tensor * residual = add->src[add->src[0] == weight_node ? 1 : 0];
if ((mul->src[0] == add || mul->src[1] == add) && mul->type == GGML_TYPE_F32 &&
scale->type == GGML_TYPE_F32 && ggml_nelements(scale) == 1 &&
ggml_are_same_shape(add, mul) && ggml_is_contiguous(mul) &&
safe_alias(node->src[0], mul) && safe_alias(weight, mul) && safe_alias(residual, mul) && safe_alias(scale, mul)) {
ggml_cuda_op_rms_norm_fused_add(*cuda_ctx, node, cgraph->nodes[i + 1], add, mul);
return 3;
}
}
}

if (node->op == GGML_OP_MUL_MAT_ID && cuda_ctx->stream_context().concurrent_events.empty() &&
ggml_cuda_match_shared_expert(cgraph, i, i + 3) &&
ggml_cuda_should_fuse_mul_mat_vec_q(cgraph->nodes[i + 2]->src[1])) {
Expand Down
50 changes: 35 additions & 15 deletions ggml/src/ggml-cuda/norm.cu
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ static __global__ void group_norm_f32(const float * x, float * dst, const int gr
}
}

template <int block_size, int vpt, bool do_vec, bool do_multiply = false, bool do_add = false, bool do_scale = false>
template <int block_size, int vpt, bool do_vec, bool do_multiply = false, bool do_add = false, bool do_scale = false, bool do_post_scale = false>
static __global__ void rms_norm_f32(const float * x,
float * dst,
const int ncols,
Expand All @@ -107,7 +107,7 @@ static __global__ void rms_norm_f32(const float * x,
const uint3 add_nrows_packed = make_uint3(0, 0, 0),
const uint3 add_nchannels_packed = make_uint3(0, 0, 0),
const uint3 add_nsamples_packed = make_uint3(0, 0, 0),
const float scale_out = 1.0f) {
const float scale_out = 1.0f, const float * post_scale = nullptr) {
ggml_cuda_pdl_lc();
const int nrows = gridDim.x;
const int row = blockIdx.x;
Expand Down Expand Up @@ -188,6 +188,13 @@ static __global__ void rms_norm_f32(const float * x,
r.z *= scale_out;
r.w *= scale_out;
}
if constexpr (do_post_scale) {
const float factor = post_scale[0];
r.x *= factor;
r.y *= factor;
r.z *= factor;
r.w *= factor;
}
dst4[i] = r;
}
}
Expand All @@ -209,7 +216,8 @@ static __global__ void rms_norm_f32(const float * x,
if constexpr (do_multiply && do_add) {
const int mul_col = fastmodulo(col, mul_ncols_packed);
const int add_col = fastmodulo(col, add_ncols_packed);
dstc[col] = scale * xc[col] * mulc[mul_col] + addc[add_col];
const float value = scale * xc[col] * mulc[mul_col] + addc[add_col];
dstc[col] = do_post_scale ? value * post_scale[0] : value;
} else if constexpr (do_multiply) {
const int mul_col = fastmodulo(col, mul_ncols_packed);
dstc[col] = scale * xc[col] * mulc[mul_col];
Expand Down Expand Up @@ -387,15 +395,15 @@ static void group_norm_f32_cuda(
do { \
const dim3 block_dims(BLOCK_SIZE, 1, 1); \
const ggml_cuda_kernel_launch_params launch_params{ blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float) : 0, stream }; \
ggml_cuda_kernel_launch(rms_norm_f32<BLOCK_SIZE, VPT, VEC, do_multiply, do_add, do_scale>, launch_params, \
ggml_cuda_kernel_launch(rms_norm_f32<BLOCK_SIZE, VPT, VEC, do_multiply, do_add, do_scale, do_post_scale>, launch_params, \
x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, \
mul, mul_stride_row, mul_stride_channel, mul_stride_sample, \
mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, \
add, add_stride_row, add_stride_channel, add_stride_sample, \
add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, scale_out); \
add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, scale_out, post_scale); \
} while (0)

template <bool do_multiply, bool do_add, bool do_scale>
template <bool do_multiply, bool do_add, bool do_scale, bool do_post_scale = false>
static void rms_norm_f32_dispatch(
const dim3 blocks_num,
const ggml_tensor * x_t, float * dst, const int ncols, const int nchannels, const int nsamples,
Expand All @@ -404,7 +412,7 @@ static void rms_norm_f32_dispatch(
const uint3 mul_ncols_packed, const uint3 mul_nrows_packed, const uint3 mul_nchannels_packed, const uint3 mul_nsamples_packed,
const ggml_tensor * add_t, const int64_t add_stride_row, const int64_t add_stride_channel, const int64_t add_stride_sample,
const uint3 add_ncols_packed, const uint3 add_nrows_packed, const uint3 add_nchannels_packed, const uint3 add_nsamples_packed,
cudaStream_t stream, const float scale_out = 1.0f) {
cudaStream_t stream, const float scale_out = 1.0f, const float * post_scale = nullptr) {
static_assert(!do_add || do_multiply, "fusing add is not supported without multiplying");
static_assert(!do_scale || !do_multiply, "fusing scale is not supported with multiplying");

Expand Down Expand Up @@ -462,6 +470,7 @@ static void rms_norm_f32_cuda(
stream, scale_out);
}

template <bool do_post_scale = false>
static void rms_norm_mul_f32_cuda(const ggml_tensor * x_t,
const ggml_tensor * mul_t,
const ggml_tensor * add_t,
Expand All @@ -488,7 +497,7 @@ static void rms_norm_mul_f32_cuda(const ggml_tensor * x_t,
const uint32_t add_nchannels,
const uint32_t add_nsamples,
const float eps,
cudaStream_t stream) {
cudaStream_t stream, const float * post_scale = nullptr) {
const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX));
if (mul_t == nullptr) {
rms_norm_f32_cuda(x_t, dst, ncols, nrows, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, stream);
Expand All @@ -511,12 +520,12 @@ static void rms_norm_mul_f32_cuda(const ggml_tensor * x_t,
const uint3 add_nrows_packed = init_fastdiv_values(add_nrows);
const uint3 add_nchannels_packed = init_fastdiv_values(add_nchannels);
const uint3 add_nsamples_packed = init_fastdiv_values(add_nsamples);
rms_norm_f32_dispatch</*do_multiply*/true, /*do_add*/true, /*do_scale*/false>(
rms_norm_f32_dispatch</*do_multiply*/true, /*do_add*/true, /*do_scale*/false, do_post_scale>(
blocks_num, x_t, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps,
mul_t, mul_stride_row, mul_stride_channel, mul_stride_sample,
mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
add_t, add_stride_row, add_stride_channel, add_stride_sample,
add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, stream);
add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, stream, 1.0f, post_scale);
}
}

Expand Down Expand Up @@ -693,10 +702,11 @@ void ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor *
eps, stream);
}

void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx,
template <bool do_post_scale>
static void rms_norm_fused_add_impl(ggml_backend_cuda_context & ctx,
ggml_tensor * dst,
ggml_tensor * mul_tensor,
ggml_tensor * add_tensor) {
ggml_tensor * add_tensor, ggml_tensor * scale_tensor) {
const ggml_tensor * rms_norm_src = (ggml_tensor *) dst->src[0];
float eps = 0.0f;

Expand All @@ -722,7 +732,7 @@ void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx,
GGML_ASSERT(false);
}

float * dst_d = (float *) add_tensor->data;
float * dst_d = (float *) (scale_tensor ? scale_tensor->data : add_tensor->data);
cudaStream_t stream = ctx.stream();

GGML_ASSERT(rms_norm_src->type == GGML_TYPE_F32);
Expand Down Expand Up @@ -764,14 +774,24 @@ void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx,
const int add_nchannels = add_src->ne[2];
const int add_nsamples = add_src->ne[3];

rms_norm_mul_f32_cuda(rms_norm_src, mul_src,add_src,dst_d,
const float * post_scale = scale_tensor ? (const float *) scale_tensor->src[scale_tensor->src[0] == add_tensor ? 1 : 0]->data : nullptr;
rms_norm_mul_f32_cuda<do_post_scale>(rms_norm_src, mul_src,add_src,dst_d,
ne00,ne01, ne02, ne03,
/*s00*/ s01, s02, s03,
/*mul_s00*/ mul_s01, mul_s02, mul_s03,
mul_ncols, mul_nrows, mul_nchannels, mul_nsamples,
/*add_s00*/ add_s01, add_s02, add_s03,
add_ncols, add_nrows, add_nchannels, add_nsamples,
eps, stream);
eps, stream, post_scale);
}

void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst,
ggml_tensor * mul_tensor, ggml_tensor * add_tensor, ggml_tensor * scale_tensor) {
if (scale_tensor) {
rms_norm_fused_add_impl<true>(ctx, dst, mul_tensor, add_tensor, scale_tensor);
} else {
rms_norm_fused_add_impl<false>(ctx, dst, mul_tensor, add_tensor, nullptr);
}
}

void ggml_cuda_op_rms_norm_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
Expand Down
2 changes: 1 addition & 1 deletion ggml/src/ggml-cuda/norm.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ void ggml_cuda_op_rms_norm_scale_fused(ggml_backend_cuda_context & ctx, ggml_ten
void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx,
ggml_tensor * dst,
ggml_tensor * mul_tensor,
ggml_tensor * add_tensor);
ggml_tensor * add_tensor, ggml_tensor * scale_tensor = nullptr);

void ggml_cuda_op_rms_norm_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);

Expand Down
4 changes: 4 additions & 0 deletions ggml/src/ggml-cuda/unary.cu
Original file line number Diff line number Diff line change
Expand Up @@ -747,6 +747,10 @@ static void ggml_cuda_op_unary_mul_impl(ggml_backend_cuda_context & ctx, ggml_te

void ggml_cuda_op_unary_mul(ggml_backend_cuda_context & ctx, ggml_tensor * unary_node, ggml_tensor * mul_node) {
switch (ggml_get_unary_op(unary_node)) {
case GGML_UNARY_OP_GELU:
GGML_ASSERT(unary_node->type == GGML_TYPE_F32);
ggml_cuda_op_unary_mul_impl<op_gelu>(ctx, unary_node, mul_node);
break;
case GGML_UNARY_OP_SILU:
ggml_cuda_op_unary_mul_impl<op_silu>(ctx, unary_node, mul_node);
break;
Expand Down
28 changes: 28 additions & 0 deletions tests/test-backend-ops.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2941,6 +2941,28 @@ struct test_rope_set_rows : public test_case {
}
};

struct test_rms_post_scale : public test_case {
const int width;
const bool extra, inplace;
test_rms_post_scale(int width, bool extra, bool inplace = false) : width(width), extra(extra), inplace(inplace) {}
std::string vars() override { return VARS_TO_STR3(width, extra, inplace); }
std::string op_desc(ggml_tensor *) override { return "RMS_POST_SCALE"; }
bool run_whole_graph() override { return true; }
ggml_tensor * build_graph(ggml_context * ctx) override {
ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, width, 129);
ggml_tensor * weight = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, width);
ggml_tensor * residual = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, width, 129);
ggml_tensor * scalar = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
if (inplace) {
residual = ggml_scale(ctx, residual, 0.5f);
}
ggml_tensor * normalized = ggml_mul(ctx, ggml_rms_norm(ctx, x, 1e-6f), weight);
ggml_tensor * add = inplace ? ggml_add_inplace(ctx, residual, normalized) : ggml_add(ctx, normalized, residual);
ggml_tensor * out = inplace ? ggml_mul_inplace(ctx, add, scalar) : ggml_mul(ctx, add, scalar);
return extra ? ggml_add(ctx, out, add) : out;
}
};

// GGML_OP_RMS_NORM with optional GGML_OP_MUL, GGML_OP_ROPE, GGML_OP_VIEW and GGML_OP_SET_ROWS
struct test_rms_norm_mul_rope : public test_case {
const std::array<int64_t, 4> ne;
Expand Down Expand Up @@ -10337,6 +10359,12 @@ static void add_rdna35_mmq_cases(std::vector<std::unique_ptr<test_case>> & test_
// Test cases for evaluation: should try to cover edge cases while using small input sizes to keep the runtime low
static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
std::vector<std::unique_ptr<test_case>> test_cases;
for (int width : {64, 193, 1536, 2048}) {
for (bool extra : {false, true}) {
test_cases.emplace_back(new test_rms_post_scale(width, extra));
}
test_cases.emplace_back(new test_rms_post_scale(width, false, true));
}
std::default_random_engine rng(0);

// unary ops
Expand Down
Loading