Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion ggml/src/ggml-cuda/fattn-mma-f16.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -322,10 +322,18 @@ static constexpr __device__ int ggml_cuda_fattn_mma_get_nstages_target(const int
}

static __host__ bool ggml_cuda_fattn_mma_get_Q_in_reg(const int DKQ, const int DV, const int ncols, const int cc) {
if (GGML_CUDA_CC_IS_RDNA3_5(cc) && DKQ == 512 && DV == 512 && ncols == 32) {
return false;
}
return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).Q_in_reg;
}

static constexpr __device__ bool ggml_cuda_fattn_mma_get_Q_in_reg(const int DKQ, const int DV, const int ncols) {
#if defined(RDNA3_5)
if (DKQ == 512 && DV == 512 && ncols == 32) {
return false;
}
#endif
return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).Q_in_reg;
}

Expand Down Expand Up @@ -1866,7 +1874,12 @@ static __global__ void flash_attn_ext_f16(
#endif // __CUDA_ARCH__ == GGML_CUDA_CC_TURING

#if defined(AMD_WMMA_AVAILABLE)
if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 256) {
#if defined(RDNA3_5)
constexpr bool language_d512 = DKQ == 512 && DV == 512 && ncols1 == 4 && ncols2 == 8;
#else
constexpr bool language_d512 = false;
#endif
if (ncols1*ncols2 < 16 || ncols2 == 1 || (DKQ > 256 && !language_d512)) {
NO_DEVICE_CODE;
return;
}
Expand Down
14 changes: 14 additions & 0 deletions ggml/src/ggml-cuda/fattn.cu
Original file line number Diff line number Diff line change
Expand Up @@ -732,6 +732,20 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
}
}

static const bool wmma_d512 = []() {
const char * env = getenv("GGML_CUDA_FATTN_WMMA_D512");
return env == nullptr || atoi(env) != 0;
}();
float logit_softcap = 0.0f;
memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));
// D512 uses the existing 4 x 8 tile, which fits RDNA shared memory.
if (wmma_d512 &&
GGML_CUDA_CC_IS_RDNA3_5(cc) && amd_wmma_available(cc) && gqa_opt_applies &&
Q->ne[0] == 512 && V->ne[0] == 512 && gqa_ratio == 8 && Q->ne[1] >= 32 &&
logit_softcap == 0.0f && K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16) {
return BEST_FATTN_KERNEL_MMA_F16;
}

// AMD WMMA is faster than the tile kernel if the wide tiles with high arithmetic intensity can be utilized.
if ((amd_wmma_available(cc) && gqa_opt_applies && Q->ne[0] <= 256) && Q->ne[0] != 40 && Q->ne[0] != 72 &&
Q->ne[1] * gqa_ratio_eff > (Q->ne[0] <= 128 ? 8 : 16)) {
Expand Down
29 changes: 29 additions & 0 deletions ggml/src/ggml-cuda/ggml-cuda.cu
Original file line number Diff line number Diff line change
Expand Up @@ -4087,6 +4087,35 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph

ggml_tensor * node = cgraph->nodes[i];

static const bool fuse_norm_rope_2axis = []() {
const char * env = getenv("GGML_CUDA_FUSE_NORM_ROPE_2AXIS");
return !env || atoi(env) != 0;
}();
if (fuse_norm_rope_2axis && node->op == GGML_OP_RMS_NORM && GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[cuda_ctx->device].cc)) {
const ggml_op ops[] = { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE, GGML_OP_ROPE };
const int output_idx = i + 3;
if (ggml_can_fuse_subgraph(cgraph, i, 4, ops, &output_idx, 1)) {
ggml_tensor * mul = cgraph->nodes[i + 1];
ggml_tensor * first = cgraph->nodes[i + 2];
ggml_tensor * second = cgraph->nodes[output_idx];
const int32_t * fp = (const int32_t *) first->op_params;
const int32_t * sp = (const int32_t *) second->op_params;
if (ggml_cuda_should_fuse_rms_norm_mul_rope(node, mul, first) &&
(mul->src[0] == node || mul->src[1] == node) &&
first->src[2] == nullptr && second->src[2] == nullptr && second->src[0] == first &&
fp[2] == GGML_ROPE_TYPE_NEOX && fp[1] > 0 && 2*fp[1] == node->ne[0] && node->ne[0] < 1024 &&
sp[15] == fp[1] && memcmp(fp, sp, 15*sizeof(int32_t)) == 0 &&
second->type == GGML_TYPE_F32 && ggml_are_same_shape(first, second) && ggml_is_contiguous(second) &&
second->src[1]->type == GGML_TYPE_I32 && ggml_is_contiguous(second->src[1]) &&
ggml_nelements(second->src[1]) >= node->ne[2] &&
ggml_cuda_check_fusion_memory_ranges(cgraph, i, 4, &output_idx, 1)) {
ggml_cuda_op_rms_norm_mul_rope_fused(*cuda_ctx, node, mul, first, nullptr, second);
return 3;
}
}
}


if (node->op == GGML_OP_MUL_MAT_ID && cuda_ctx->stream_context().concurrent_events.empty() &&
ggml_cuda_match_shared_expert(cgraph, i, i + 3) &&
ggml_cuda_should_fuse_mul_mat_vec_q(cgraph->nodes[i + 2]->src[1])) {
Expand Down
97 changes: 69 additions & 28 deletions ggml/src/ggml-cuda/rope.cu
Original file line number Diff line number Diff line change
Expand Up @@ -726,7 +726,7 @@ void ggml_cuda_op_rope_fused(ggml_backend_cuda_context & ctx, ggml_tensor * rope

// fused RMS_NORM + MUL + ROPE (+ VIEW + SET_ROWS)
// one block per row: block_reduce gives the norm scale, then each thread applies mul and rope to the elements it owns
template <int block_size, bool has_ff, typename D>
template <int block_size, bool has_ff, typename D, bool two_axis = false, bool loop_grid = true>
static __global__ void rms_norm_mul_rope_f32(
const float * x, D * dst, const int ncols, const int nchannels, const int nsamples,
const int64_t s01, const int64_t s02, const int64_t s03,
Expand All @@ -741,7 +741,7 @@ static __global__ void rms_norm_mul_rope_f32(
const rope_corr_dims corr_dims, const float theta_scale,
const float * freq_factors,
const int64_t * row_indices, const int set_rows_stride,
const bool is_neox) {
const bool is_neox, const int32_t * pos_second) {
ggml_cuda_pdl_lc();
const int row = blockIdx.x;
const int tid = threadIdx.x;
Expand All @@ -751,8 +751,8 @@ static __global__ void rms_norm_mul_rope_f32(
ggml_cuda_pdl_sync();

// grid.y and grid.z are clamped to the CUDA limit, iterate over the excess channels/samples
for (int sample = blockIdx.z; sample < nsamples; sample += gridDim.z) {
for (int channel = blockIdx.y; channel < nchannels; channel += gridDim.y) {
for (int sample = blockIdx.z; !loop_grid || sample < nsamples; sample += gridDim.z) {
for (int channel = blockIdx.y; !loop_grid || channel < nchannels; channel += gridDim.y) {
const float * xc = x + sample*s03 + channel*s02 + row*s01;

const uint32_t mul_row = fastmodulo(row, mul_nrows_packed);
Expand All @@ -778,40 +778,57 @@ static __global__ void rms_norm_mul_rope_f32(
D * dstc = dst + idst;

for (int i0 = 2*tid; i0 < ncols; i0 += 2*block_size) {
const int offset = two_axis && i0 >= n_dims ? n_dims : 0;
const int ir = i0 - offset;
int ix0;
int ix1;
if (is_neox && i0 < n_dims) {
ix0 = i0/2;
ix1 = i0/2 + n_dims/2;
if (is_neox && ir < n_dims) {
ix0 = offset + ir/2;
ix1 = offset + ir/2 + n_dims/2;
} else {
ix0 = i0 + 0;
ix1 = i0 + 1;
}

const float x0 = scale * xc[ix0] * mulc[fastmodulo(ix0, mul_ncols_packed)];
const float x1 = scale * xc[ix1] * mulc[fastmodulo(ix1, mul_ncols_packed)];
float x0 = scale * xc[ix0] * mulc[fastmodulo(ix0, mul_ncols_packed)];
float x1 = scale * xc[ix1] * mulc[fastmodulo(ix1, mul_ncols_packed)];
if constexpr (two_axis) {
if (offset != 0) {
#if defined(GGML_USE_HIP)
asm volatile("" : "+v"(x0), "+v"(x1));
#else
asm volatile("" : "+f"(x0), "+f"(x1));
#endif
}
}

if (i0 >= n_dims) {
if (ir >= n_dims) {
dstc[ix0] = ggml_cuda_cast<D>(x0);
dstc[ix1] = ggml_cuda_cast<D>(x1);
continue;
}

const float theta_base = pos[channel]*powf(theta_scale, i0/2.0f);
const float freq_factor = has_ff ? freq_factors[i0/2] : 1.0f;
const float theta_base = (two_axis && offset != 0 ? pos_second[channel] : pos[channel])*powf(theta_scale, ir/2.0f);
const float freq_factor = has_ff ? freq_factors[ir/2] : 1.0f;

float cos_theta;
float sin_theta;
rope_yarn<true>(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta);
rope_yarn<true>(theta_base/freq_factor, freq_scale, corr_dims, ir, ext_factor, attn_factor, cos_theta, sin_theta);

dstc[ix0] = ggml_cuda_cast<D>(x0*cos_theta - x1*sin_theta);
dstc[ix1] = ggml_cuda_cast<D>(x0*sin_theta + x1*cos_theta);
}

if constexpr (block_size > WARP_SIZE) {
if constexpr (block_size > WARP_SIZE && loop_grid) {
// sync is needed as we reuse s_sum across block_reduce invocations, see #26385
__syncthreads();
}
if constexpr (!loop_grid) {
break;
}
}
if constexpr (!loop_grid) {
break;
}
}
}
Expand All @@ -832,7 +849,7 @@ static void rms_norm_mul_rope_cuda(
const rope_corr_dims corr_dims,
const float * freq_factors,
const int64_t * row_indices, const int set_rows_stride,
const bool is_neox, cudaStream_t stream) {
const bool is_neox, cudaStream_t stream, const int32_t * pos_second = nullptr) {
GGML_ASSERT(ncols % 2 == 0);

const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX));
Expand All @@ -844,6 +861,27 @@ static void rms_norm_mul_rope_cuda(
const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels);
const uint3 mul_nsamples_packed = init_fastdiv_values(mul_nsamples);

if (pos_second != nullptr) {
static const bool norm_rope_small_block = []() {
const char * env = getenv("GGML_CUDA_NORM_ROPE_SMALL_BLOCK");
return !env || atoi(env) != 0;
}();
GGML_ASSERT(ncols < 1024 && ncols == 2*n_dims && is_neox && !freq_factors && !set_rows_stride);
const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
const bool small_block = norm_rope_small_block && GGML_CUDA_CC_IS_RDNA3_5(cc) && ncols == 128 && nchannels >= 128;
const ggml_cuda_kernel_launch_params params = {blocks_num, dim3(small_block ? 128 : 256), 32*sizeof(float), stream};
const bool loop_grid = nchannels > UINT16_MAX || nsamples > UINT16_MAX;
const auto kernel = small_block ?
(loop_grid ? rms_norm_mul_rope_f32<128, false, D, true> : rms_norm_mul_rope_f32<128, false, D, true, false>) :
(loop_grid ? rms_norm_mul_rope_f32<256, false, D, true> : rms_norm_mul_rope_f32<256, false, D, true, false>);
ggml_cuda_kernel_launch(kernel, params,
x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03,
mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale,
freq_factors, row_indices, set_rows_stride, is_neox, pos_second);
return;
}

if (ncols < 1024) {
const dim3 block_dims(256, 1, 1);
const ggml_cuda_kernel_launch_params launch_params = {blocks_num, block_dims, 32*sizeof(float), stream};
Expand All @@ -852,13 +890,13 @@ static void rms_norm_mul_rope_cuda(
x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03,
mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale,
freq_factors, row_indices, set_rows_stride, is_neox);
freq_factors, row_indices, set_rows_stride, is_neox, nullptr);
} else {
ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<256, true, D>, launch_params,
x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03,
mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale,
freq_factors, row_indices, set_rows_stride, is_neox);
freq_factors, row_indices, set_rows_stride, is_neox, nullptr);
}
} else {
const dim3 block_dims(1024, 1, 1);
Expand All @@ -868,19 +906,19 @@ static void rms_norm_mul_rope_cuda(
x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03,
mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale,
freq_factors, row_indices, set_rows_stride, is_neox);
freq_factors, row_indices, set_rows_stride, is_neox, nullptr);
} else {
ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<1024, true, D>, launch_params,
x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03,
mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale,
freq_factors, row_indices, set_rows_stride, is_neox);
freq_factors, row_indices, set_rows_stride, is_neox, nullptr);
}
}
}

void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx,
ggml_tensor * rms_norm, ggml_tensor * mul, ggml_tensor * rope, ggml_tensor * set_rows) {
ggml_tensor * rms_norm, ggml_tensor * mul, ggml_tensor * rope, ggml_tensor * set_rows, ggml_tensor * rope_second) {
const ggml_tensor * x = rms_norm->src[0];
const ggml_tensor * mul_src = mul->src[0] == rms_norm ? mul->src[1] : mul->src[0];

Expand All @@ -892,8 +930,9 @@ void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx,
GGML_ASSERT(mul_src->type == GGML_TYPE_F32);
GGML_ASSERT(rope->type == GGML_TYPE_F32);

void * dst_d = rope->data;
ggml_type dst_type = rope->type;
const ggml_tensor * output = rope_second ? rope_second : rope;
void * dst_d = output->data;
ggml_type dst_type = output->type;
const int64_t * row_indices = nullptr;
int set_rows_stride = 0;

Expand Down Expand Up @@ -943,10 +982,10 @@ void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx,
const int64_t mul_s02 = mul_src->nb[2] / ts_mul;
const int64_t mul_s03 = mul_src->nb[3] / ts_mul;

const size_t ts_dst = ggml_type_size(rope->type);
const int64_t s1 = rope->nb[1] / ts_dst;
const int64_t s2 = rope->nb[2] / ts_dst;
const int64_t s3 = rope->nb[3] / ts_dst;
const size_t ts_dst = ggml_type_size(output->type);
const int64_t s1 = output->nb[1] / ts_dst;
const int64_t s2 = output->nb[2] / ts_dst;
const int64_t s3 = output->nb[3] / ts_dst;

cudaStream_t stream = ctx.stream();

Expand All @@ -956,14 +995,16 @@ void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx,
(const float *) mul_src->data, mul_s01, mul_s02, mul_s03,
mul_src->ne[0], mul_src->ne[1], mul_src->ne[2], mul_src->ne[3],
n_dims, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims,
freq_factors, row_indices, set_rows_stride, is_neox, stream);
freq_factors, row_indices, set_rows_stride, is_neox, stream,
rope_second ? (const int32_t *) rope_second->src[1]->data : nullptr);
} else if (dst_type == GGML_TYPE_F16) {
rms_norm_mul_rope_cuda((const float *) x->data, (half *) dst_d,
x->ne[0], x->ne[1], x->ne[2], x->ne[3], s01, s02, s03, s1, s2, s3, eps,
(const float *) mul_src->data, mul_s01, mul_s02, mul_s03,
mul_src->ne[0], mul_src->ne[1], mul_src->ne[2], mul_src->ne[3],
n_dims, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims,
freq_factors, row_indices, set_rows_stride, is_neox, stream);
freq_factors, row_indices, set_rows_stride, is_neox, stream,
rope_second ? (const int32_t *) rope_second->src[1]->data : nullptr);
} else {
GGML_ABORT("fatal error");
}
Expand Down
2 changes: 1 addition & 1 deletion ggml/src/ggml-cuda/rope.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@ void ggml_cuda_op_rope_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);

void ggml_cuda_op_rope_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * set_rows);

void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx, ggml_tensor * rms_norm, ggml_tensor * mul, ggml_tensor * rope, ggml_tensor * set_rows);
void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx, ggml_tensor * rms_norm, ggml_tensor * mul, ggml_tensor * rope, ggml_tensor * set_rows, ggml_tensor * rope_second = nullptr);
Loading
Loading