Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 37 additions & 1 deletion ggml/src/ggml-cuda/cpy.cu
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#include "opt-traffic.cuh"
#include "cpy.cuh"
#include "dequantize.cuh"
#include "cpy-utils.cuh"
Expand All @@ -11,6 +12,29 @@ const int CUDA_CPY_TILE_DIM_2D = 32; // 2D tile dimension for transposed blocks
const int CUDA_CPY_BLOCK_NM = 8; // block size of 3rd dimension if available
const int CUDA_CPY_BLOCK_ROWS = 8; // block dimension for marching through rows

static __global__ void cpy_f32_f16_vec8(
const char * src, uint4 * dst, const uint32_t n, const uint3 ne0, const uint3 ne1, const uint3 ne2,
const size_t nb1, const size_t nb2, const size_t nb3) {
ggml_cuda_pdl_lc();
const uint32_t i = blockIdx.x*blockDim.x + threadIdx.x;
if (i >= n) {
return;
}
const uint32_t c = fastmodulo(i, ne0);
const uint32_t row = fastdiv(i, ne0);
const uint32_t r1 = fastmodulo(row, ne1);
const uint32_t r23 = fastdiv(row, ne1);
const uint32_t r2 = fastmodulo(r23, ne2);
const uint32_t r3 = fastdiv(r23, ne2);
ggml_cuda_pdl_sync();
const float4 * ptr = (const float4 *) (src + 32*size_t(c) + nb1*r1 + nb2*r2 + nb3*r3);
const float4 a = ptr[0], b = ptr[1];
alignas(16) const half values[8] = {ggml_cuda_cast<half>(a.x), ggml_cuda_cast<half>(a.y),
ggml_cuda_cast<half>(a.z), ggml_cuda_cast<half>(a.w), ggml_cuda_cast<half>(b.x),
ggml_cuda_cast<half>(b.y), ggml_cuda_cast<half>(b.z), ggml_cuda_cast<half>(b.w)};
ggml_cuda_memcpy_1<16>(dst + i, values);
}

template <cpy_kernel_t cpy_1>
static __global__ void cpy_scalar(const char * cx, char * cdst, const int64_t ne,
const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,
Expand Down Expand Up @@ -494,7 +518,19 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg
(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
}
} else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16) {
if (contiguous_srcs) {
if (GGML_CUDA_TRAFFIC_OPT_ENABLED(GGML_CUDA_CPY_F32_F16_VEC8, true) &&
GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[ctx.device].cc) && !contiguous_srcs &&
ggml_is_contiguous(src1) && ggml_are_same_shape(src0, src1) && nb00 == sizeof(float) &&
ne00 % 8 == 0 && ne >= 8192 && ne/8 <= INT32_MAX &&
ne00/8 <= UINT32_MAX && ne01 <= UINT32_MAX && ne02 <= UINT32_MAX &&
(uintptr_t) src0_ddc % 16 == 0 && (uintptr_t) src1_ddc % 16 == 0 &&
nb01 % 16 == 0 && nb02 % 16 == 0 && nb03 % 16 == 0 &&
((uintptr_t) src0_ddc <= (uintptr_t) src1_ddc ?
(uintptr_t) src1_ddc - (uintptr_t) src0_ddc >= ggml_nbytes(src0) :
(uintptr_t) src0_ddc - (uintptr_t) src1_ddc >= ggml_nbytes(src1))) {
cpy_f32_f16_vec8<<<(ne/8 + 255)/256, 256, 0, main_stream>>>(src0_ddc, (uint4 *) src1_ddc,
ne/8, init_fastdiv_values(ne00/8), init_fastdiv_values(ne01), init_fastdiv_values(ne02), nb01, nb02, nb03);
} else if (contiguous_srcs) {
ggml_cpy_scalar_contiguous_cuda<float, half>
(src0_ddc, src1_ddc, ne, main_stream);
} else {
Expand Down
23 changes: 23 additions & 0 deletions ggml/src/ggml-cuda/ggml-cuda.cu
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#include "opt-traffic.cuh"
#include "ggml-cuda.h"
#include "ggml-impl.h"
#include "ggml-backend-impl.h"
Expand Down Expand Up @@ -4087,6 +4088,28 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph

ggml_tensor * node = cgraph->nodes[i];

if (GGML_CUDA_TRAFFIC_OPT_ENABLED(GGML_CUDA_FUSE_NORM_MUL_ADD, true) && node->op == GGML_OP_NORM && GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[cuda_ctx->device].cc)) {
const ggml_op ops[] = { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD };
const int output_idx = i + 2;
if (ggml_can_fuse_subgraph(cgraph, i, 3, ops, &output_idx, 1)) {
ggml_tensor * mul = cgraph->nodes[i + 1];
ggml_tensor * add = cgraph->nodes[output_idx];
const ggml_tensor * weight = mul->src[mul->src[0] == node ? 1 : 0];
const ggml_tensor * bias = add->src[add->src[0] == mul ? 1 : 0];
if ((mul->src[0] == node || mul->src[1] == node) && (add->src[0] == mul || add->src[1] == mul) &&
node->src[0]->type == GGML_TYPE_F32 && weight->type == GGML_TYPE_F32 && bias->type == GGML_TYPE_F32 &&
node->type == GGML_TYPE_F32 && mul->type == GGML_TYPE_F32 && add->type == GGML_TYPE_F32 &&
ggml_are_same_shape(node, add) && ggml_are_same_shape(node, mul) &&
ggml_is_contiguous(node->src[0]) && ggml_is_contiguous(add) &&
ggml_is_contiguous(weight) && weight->ne[0] == node->ne[0] && ggml_nrows(weight) == 1 &&
ggml_is_contiguous(bias) && bias->ne[0] == node->ne[0] && ggml_nrows(bias) == 1 &&
ggml_cuda_check_fusion_memory_ranges(cgraph, i, 3, &output_idx, 1)) {
ggml_cuda_op_norm_fused(*cuda_ctx, node, mul, add);
return 2;
}
}
}

if (node->op == GGML_OP_MUL_MAT_ID && cuda_ctx->stream_context().concurrent_events.empty() &&
ggml_cuda_match_shared_expert(cgraph, i, i + 3) &&
ggml_cuda_should_fuse_mul_mat_vec_q(cgraph->nodes[i + 2]->src[1])) {
Expand Down
55 changes: 55 additions & 0 deletions ggml/src/ggml-cuda/moe-weighted-reduction.cu
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,47 @@ static __global__ void moe_weighted_reduction_f32(const float * __restrict__ exp
dst[token * n_embd + col] = sum;
}

// same sums as moe_weighted_reduction_f32, with all loads of a thread issued before the sum and 4 columns per thread
template <int n_expert_used>
static __global__ void moe_weighted_reduction_f32_x4(const float4 * __restrict__ experts,
const float * __restrict__ expert_scale,
const float * __restrict__ weights,
float4 * __restrict__ dst,
const int64_t n_embd4) {
const int64_t token = blockIdx.x;
const int64_t col = (int64_t) blockIdx.y * blockDim.x + threadIdx.x;
if (col >= n_embd4) {
return;
}

const uint64_t first_row = (uint64_t) token * n_expert_used;
float4 x[n_expert_used];
float w[n_expert_used];
#pragma unroll
for (int expert = 0; expert < n_expert_used; ++expert) {
x[expert] = experts[(first_row + expert) * n_embd4 + col];
w[expert] = weights[first_row + expert];
}
float s[n_expert_used];
#pragma unroll
for (int expert = 0; expert < n_expert_used; ++expert) {
s[expert] = expert_scale != nullptr ? expert_scale[first_row + expert] : 1.0f;
}
float4 sum;
sum.x = (x[0].x * s[0]) * w[0];
sum.y = (x[0].y * s[0]) * w[0];
sum.z = (x[0].z * s[0]) * w[0];
sum.w = (x[0].w * s[0]) * w[0];
#pragma unroll
for (int expert = 1; expert < n_expert_used; ++expert) {
sum.x += (x[expert].x * s[expert]) * w[expert];
sum.y += (x[expert].y * s[expert]) * w[expert];
sum.z += (x[expert].z * s[expert]) * w[expert];
sum.w += (x[expert].w * s[expert]) * w[expert];
}
dst[token * n_embd4 + col] = sum;
}

static void launch_moe_weighted_reduction(const float * experts,
const float * expert_scale,
const float * weights,
Expand All @@ -32,6 +73,20 @@ static void launch_moe_weighted_reduction(const float * experts,
int64_t n_tokens,
int n_expert_used,
cudaStream_t stream) {
// opt-in: one kernel per common expert count, the generic kernel is latency bound with large n_tokens
static const bool use_x4 = getenv("GGML_CUDA_MOE_REDUCE_X4") != nullptr && atoi(getenv("GGML_CUDA_MOE_REDUCE_X4")) != 0;
if (use_x4 && n_embd % 4 == 0 && (n_expert_used == 8 || n_expert_used == 4) &&
(uintptr_t) experts % 16 == 0 && (uintptr_t) dst % 16 == 0) {
constexpr int threads = 128;
const int64_t n_embd4 = n_embd / 4;
const dim3 blocks(n_tokens, (n_embd4 + threads - 1) / threads, 1);
if (n_expert_used == 8) {
moe_weighted_reduction_f32_x4<8><<<blocks, threads, 0, stream>>>((const float4 *) experts, expert_scale, weights, (float4 *) dst, n_embd4);
} else {
moe_weighted_reduction_f32_x4<4><<<blocks, threads, 0, stream>>>((const float4 *) experts, expert_scale, weights, (float4 *) dst, n_embd4);
}
return;
}
constexpr int threads = 256;
const dim3 blocks(n_tokens, (n_embd + threads - 1) / threads, 1);
moe_weighted_reduction_f32
Expand Down
39 changes: 36 additions & 3 deletions ggml/src/ggml-cuda/norm.cu
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
#include "norm.cuh"
#include <cstdint>

template <int block_size>
template <int block_size, bool affine = false>
static __global__ void norm_f32(
const float * x, float * dst, const int ncols, const int nchannels, const int nsamples,
const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps) {
const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps,
const float * weight = nullptr, const float * bias = nullptr) {
const int nrows = gridDim.x;
const int row = blockIdx.x;
const int tid = threadIdx.x;
Expand Down Expand Up @@ -35,7 +36,22 @@ static __global__ void norm_f32(
const float inv_std = rsqrtf(var + eps);

for (int col = tid; col < ncols; col += block_size) {
dstc[col] = (xc[col] - mean) * inv_std;
float value = (xc[col] - mean) * inv_std;
if constexpr (affine) {
#if defined(GGML_USE_HIP)
asm volatile("" : "+v"(value));
#else
asm volatile("" : "+f"(value));
#endif
value *= weight[col];
#if defined(GGML_USE_HIP)
asm volatile("" : "+v"(value));
#else
asm volatile("" : "+f"(value));
#endif
value += bias[col];
}
dstc[col] = value;
}

if constexpr (block_size > WARP_SIZE) {
Expand Down Expand Up @@ -569,6 +585,23 @@ void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream);
}

void ggml_cuda_op_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * norm, ggml_tensor * mul, ggml_tensor * add) {
const ggml_tensor * x = norm->src[0];
const ggml_tensor * w = mul->src[mul->src[0] == norm ? 1 : 0];
const ggml_tensor * b = add->src[add->src[0] == mul ? 1 : 0];
const dim3 grid(x->ne[1], MIN(x->ne[2], int64_t(UINT16_MAX)), MIN(x->ne[3], int64_t(UINT16_MAX)));
const float eps = ggml_get_op_params_f32(norm, 0);
if (x->ne[0] < 1024) {
norm_f32<WARP_SIZE, true><<<grid, WARP_SIZE, 0, ctx.stream()>>>((const float *) x->data, (float *) add->data,
x->ne[0], x->ne[2], x->ne[3], x->nb[1]/sizeof(float), x->nb[2]/sizeof(float), x->nb[3]/sizeof(float), eps,
(const float *) w->data, (const float *) b->data);
} else {
norm_f32<1024, true><<<grid, 1024, 32*sizeof(float2), ctx.stream()>>>((const float *) x->data, (float *) add->data,
x->ne[0], x->ne[2], x->ne[3], x->nb[1]/sizeof(float), x->nb[2]/sizeof(float), x->nb[3]/sizeof(float), eps,
(const float *) w->data, (const float *) b->data);
}
}

void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
const ggml_tensor * src0 = dst->src[0];
const float * src0_d = (const float *)src0->data;
Expand Down
2 changes: 2 additions & 0 deletions ggml/src/ggml-cuda/norm.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,5 @@ void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx,
void ggml_cuda_op_rms_norm_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);

void ggml_cuda_op_l2_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);

void ggml_cuda_op_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * norm, ggml_tensor * mul, ggml_tensor * add);
6 changes: 6 additions & 0 deletions ggml/src/ggml-cuda/opt-traffic.cuh
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#pragma once

#include <cstdlib>

#define GGML_CUDA_TRAFFIC_OPT_ENABLED(name, fallback) \
([] { static const bool enabled = [] { const char * v = std::getenv(#name); return v ? std::atoi(v) != 0 : (fallback); }(); return enabled; }())
68 changes: 57 additions & 11 deletions ggml/src/ggml-cuda/rope.cu
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#include "opt-traffic.cuh"
#include "convert.cuh"
#include "ggml-cuda/common.cuh"
#include "ggml.h"
Expand Down Expand Up @@ -391,8 +392,14 @@ static void rope_norm_cuda(const T * x,
const bool inplace,
cudaStream_t stream) {
GGML_ASSERT(ne00 % 2 == 0);
const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);
const int n_blocks_x = (ne00 + 2 * CUDA_ROPE_BLOCK_SIZE - 1) / (2 * CUDA_ROPE_BLOCK_SIZE);
// opt-in (GGML_CUDA_ROPE_BLOCK): smaller blocks for short rows (head dim 64/128 leaves most of a 256-thread block idle)
static const int block_size = []() {
const char * env = getenv("GGML_CUDA_ROPE_BLOCK");
const int bs = env ? atoi(env) : 0;
return bs >= 32 && bs <= CUDA_ROPE_BLOCK_SIZE ? bs : CUDA_ROPE_BLOCK_SIZE;
}();
const dim3 block_dims(1, block_size, 1);
const int n_blocks_x = (ne00 + 2 * block_size - 1) / (2 * block_size);
const dim3 block_nums(nr, n_blocks_x, 1);

const float theta_scale = powf(freq_base, -2.0f / n_dims);
Expand Down Expand Up @@ -435,8 +442,23 @@ static void rope_neox_cuda(const T * x,
const bool inplace,
cudaStream_t stream) {
GGML_ASSERT(ne00 % 2 == 0);
const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);
const int n_blocks_x = (ne00 + 2 * CUDA_ROPE_BLOCK_SIZE - 1) / (2 * CUDA_ROPE_BLOCK_SIZE);
int block_size = CUDA_ROPE_BLOCK_SIZE;
#if defined(GGML_USE_HIP)
const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
if (GGML_CUDA_TRAFFIC_OPT_ENABLED(GGML_CUDA_ROPE_SMALL_BLOCK, true) && GGML_CUDA_CC_IS_RDNA3_5(cc) && ne00 == 128) {
block_size = 64;
}
#endif
static const int block_override = []() {
const char * env = getenv("GGML_CUDA_ROPE_BLOCK");
const int bs = env ? atoi(env) : 0;
return bs >= 32 && bs <= CUDA_ROPE_BLOCK_SIZE ? bs : 0;
}();
if (block_override) {
block_size = block_override;
}
const dim3 block_dims(1, block_size, 1);
const int n_blocks_x = (ne00 + 2 * block_size - 1) / (2 * block_size);
const dim3 block_nums(nr, n_blocks_x, 1);

const float theta_scale = powf(freq_base, -2.0f / n_dims);
Expand Down Expand Up @@ -482,8 +504,15 @@ static void rope_multi_cuda(const T * x,
const bool inplace,
cudaStream_t stream) {
GGML_ASSERT(ne00 % 2 == 0);
const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);
const int n_blocks_x = (ne00 + 2 * CUDA_ROPE_BLOCK_SIZE - 1) / (2 * CUDA_ROPE_BLOCK_SIZE);
int block_size = CUDA_ROPE_BLOCK_SIZE;
#if defined(GGML_USE_HIP)
const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
if (GGML_CUDA_TRAFFIC_OPT_ENABLED(GGML_CUDA_ROPE_SMALL_BLOCK, true) && GGML_CUDA_CC_IS_RDNA3_5(cc) && (ne00 == 72 || ne00 == 80 || ne00 == 128)) {
block_size = 64;
}
#endif
const dim3 block_dims(1, block_size, 1);
const int n_blocks_x = (ne00 + 2 * block_size - 1) / (2 * block_size);
const dim3 block_nums(nr, n_blocks_x, 1);

const float theta_scale = powf(freq_base, -2.0f / n_dims);
Expand Down Expand Up @@ -525,8 +554,15 @@ static void rope_vision_cuda(const T * x,
const mrope_sections sections,
cudaStream_t stream) {
GGML_ASSERT(ne00 % 2 == 0);
const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);
const int n_blocks_x = (ne00 + 2 * CUDA_ROPE_BLOCK_SIZE - 1) / (2 * CUDA_ROPE_BLOCK_SIZE);
int block_size = CUDA_ROPE_BLOCK_SIZE;
#if defined(GGML_USE_HIP)
const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
if (GGML_CUDA_TRAFFIC_OPT_ENABLED(GGML_CUDA_ROPE_SMALL_BLOCK, true) && GGML_CUDA_CC_IS_RDNA3_5(cc) && (ne00 == 72 || ne00 == 80 || ne00 == 128)) {
block_size = 64;
}
#endif
const dim3 block_dims(1, block_size, 1);
const int n_blocks_x = (ne00 + 2 * block_size - 1) / (2 * block_size);
const dim3 block_nums(nr, n_blocks_x, 1);
// break down (head_dim, heads, seq) into (CUDA_ROPE_BLOCK_SIZE, x, heads * seq)
// where x ~= ceil(head_dim / CUDA_ROPE_BLOCK_SIZE);
Expand Down Expand Up @@ -844,17 +880,27 @@ static void rms_norm_mul_rope_cuda(
const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels);
const uint3 mul_nsamples_packed = init_fastdiv_values(mul_nsamples);

static const bool norm_rope_warp = []() {
const char * env = getenv("GGML_CUDA_NORM_ROPE_WARP");
return env != nullptr && atoi(env) != 0;
}();
if (ncols < 1024) {
const dim3 block_dims(256, 1, 1);
const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
const bool one_warp = norm_rope_warp && GGML_CUDA_CC_IS_RDNA3_5(cc) && ncols <= 256;
const dim3 block_dims(one_warp ? WARP_SIZE : 256, 1, 1);
const ggml_cuda_kernel_launch_params launch_params = {blocks_num, block_dims, 32*sizeof(float), stream};
if (freq_factors == nullptr) {
ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<256, false, D>, launch_params,
const auto kernel = one_warp ? rms_norm_mul_rope_f32<WARP_SIZE, false, D> :
rms_norm_mul_rope_f32<256, false, D>;
ggml_cuda_kernel_launch(kernel, launch_params,
x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03,
mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale,
freq_factors, row_indices, set_rows_stride, is_neox);
} else {
ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<256, true, D>, launch_params,
const auto kernel = one_warp ? rms_norm_mul_rope_f32<WARP_SIZE, true, D> :
rms_norm_mul_rope_f32<256, true, D>;
ggml_cuda_kernel_launch(kernel, launch_params,
x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03,
mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale,
Expand Down
Loading
Loading