Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 25 additions & 2 deletions ggml/src/ggml-cuda/fattn-mma-f16.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,16 @@
#include "mma.cuh"
#include "fattn-common.cuh"

#ifndef GGML_CUDA_FATTN_WMMA_NO_GQA
#define GGML_CUDA_FATTN_WMMA_NO_GQA 1
#endif
#ifndef GGML_CUDA_FATTN_D64_FULL_TILE
#define GGML_CUDA_FATTN_D64_FULL_TILE 1
#endif
#ifndef GGML_CUDA_FATTN_D80_FULL_TILE
#define GGML_CUDA_FATTN_D80_FULL_TILE 1
#endif

using namespace ggml_cuda_mma;

// Config options for the MMA kernel.
Expand Down Expand Up @@ -1355,11 +1365,19 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
// kb0_start is always < kb0_stop so the last iter can be executed unconditionally.
if constexpr (ncols2 == 1 || use_sparse) {
constexpr bool oob_check = true;
#if defined(RDNA3_5)
// Only the last dense KV tile can be partial.
constexpr bool full_tile_oob_check = !(((DKQ == 80 && GGML_CUDA_FATTN_D80_FULL_TILE) ||
(DKQ == 64 && GGML_CUDA_FATTN_D64_FULL_TILE)) &&
DV == DKQ && ncols1 == 64 && ncols2 == 1 && !use_sparse);
#else
constexpr bool full_tile_oob_check = true;
#endif
for (; kb0 < kb0_stop-1; ++kb0) {
constexpr bool last_iter = false;
constexpr int k_VKQ_sup = nbatch_fa;
flash_attn_ext_f16_iter
<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,
<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, full_tile_oob_check,
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
Expand Down Expand Up @@ -1866,7 +1884,12 @@ static __global__ void flash_attn_ext_f16(
#endif // __CUDA_ARCH__ == GGML_CUDA_CC_TURING

#if defined(AMD_WMMA_AVAILABLE)
if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 256) {
#if defined(RDNA3_5)
constexpr bool vision_mha = (DKQ == 80 || (DKQ == 64 && GGML_CUDA_FATTN_WMMA_NO_GQA)) && DV == DKQ && ncols1 == 64;
#else
constexpr bool vision_mha = false;
#endif
if (ncols1*ncols2 < 16 || (ncols2 == 1 && !vision_mha) || DKQ > 256) {
NO_DEVICE_CODE;
return;
}
Expand Down
112 changes: 112 additions & 0 deletions ggml/src/ggml-cuda/fattn.cu
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@
#include "fattn-vec.cuh"
#include "fattn.cuh"

#ifndef GGML_CUDA_FATTN_WMMA_D72
#define GGML_CUDA_FATTN_WMMA_D72 1
#endif

#if !defined(GGML_USE_MUSA)

// one list per group of ncols1 queries: a column is selected if any query of the group can see it
Expand Down Expand Up @@ -305,6 +309,68 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_con
}
}

#if GGML_CUDA_FATTN_WMMA_D72
template <typename T, int Dsrc, int Ddst>
static __global__ void flash_attn_pad_head(
const char * src, T * dst, const int64_t n, const uint3 ne1, const uint3 ne2,
const size_t nb1, const size_t nb2, const size_t nb3) {
const int64_t i = int64_t(blockIdx.x)*blockDim.x + threadIdx.x;
if (i >= n) {
return;
}
const int col = i % Ddst;
const int row = i / Ddst;
const int i1 = fastmodulo(row, ne1);
const int i23 = fastdiv(row, ne1);
const int i2 = fastmodulo(i23, ne2);
const int i3 = fastdiv(i23, ne2);
dst[i] = col < Dsrc ? ((const T *) (src + i1*nb1 + i2*nb2 + i3*nb3))[col] : T(0.0f);
}

template <typename T, int Dsrc, int Ddst>
static void ggml_cuda_flash_attn_pad_head(const ggml_tensor * src, ggml_tensor * dst, cudaStream_t stream) {
const int64_t n = ggml_nelements(dst);
const ggml_cuda_kernel_launch_params params(dim3((n + 255)/256), dim3(256), 0, stream);
ggml_cuda_kernel_launch(flash_attn_pad_head<T, Dsrc, Ddst>, params,
(const char *) src->data, (T *) dst->data, n, init_fastdiv_values(src->ne[1]), init_fastdiv_values(src->ne[2]),
src->nb[1], src->nb[2], src->nb[3]);
CUDA_CHECK(cudaGetLastError());
}

static void ggml_cuda_flash_attn_ext_mma_f16_pad72(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
ggml_tensor q = *dst->src[0];
ggml_tensor k = *dst->src[1];
ggml_tensor v = *dst->src[2];
ggml_tensor out = *dst;
const int64_t rows = ggml_nrows(&q);
ggml_cuda_pool_alloc<float> q_out(ctx.pool(), 2*80*rows);
ggml_cuda_pool_alloc<half> kv(ctx.pool(), 2*80*rows);
q.data = q_out.get();
k.data = kv.get();
v.data = kv.get() + 80*rows;
out.data = q_out.get() + 80*rows;
for (ggml_tensor * t : {&q, &k, &v, &out}) {
t->ne[0] = 80;
for (int d = 1; d < GGML_MAX_DIMS; ++d) {
t->nb[d] = t->nb[d - 1]*t->ne[d - 1];
}
t->view_src = nullptr;
t->view_offs = 0;
}
out.src[0] = &q;
out.src[1] = &k;
out.src[2] = &v;
const cudaStream_t stream = ctx.stream();
ggml_cuda_flash_attn_pad_head<float, 72, 80>(dst->src[0], &q, stream);
ggml_cuda_flash_attn_pad_head<half, 72, 80>(dst->src[1], &k, stream);
ggml_cuda_flash_attn_pad_head<half, 72, 80>(dst->src[2], &v, stream);
// Keep the original scale: padding changes the storage width, not the attention width.
ggml_cuda_flash_attn_ext_mma_f16_case<80, 80, 64, 1>(ctx, &out);
ggml_cuda_flash_attn_pad_head<float, 80, 72>(&out, dst, stream);
}

#endif

static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
const ggml_tensor * KQV = dst;
Expand All @@ -318,6 +384,11 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg
GGML_ASSERT(V->ne[0] == 64);
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2< 64, 64>(ctx, dst);
break;
#if GGML_CUDA_FATTN_WMMA_D72
case 72:
ggml_cuda_flash_attn_ext_mma_f16_pad72(ctx, dst);
break;
#endif
case 80:
GGML_ASSERT(V->ne[0] == 80);
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2< 80, 80>(ctx, dst);
Expand Down Expand Up @@ -732,6 +803,47 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
}
}

#if GGML_CUDA_FATTN_WMMA_D72 || GGML_CUDA_FATTN_WMMA_NO_GQA
float logit_softcap = 0.0f;
memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));
#endif

#if GGML_CUDA_FATTN_WMMA_D72
static const bool wmma_d72 = []() {
const char * env = getenv("GGML_CUDA_FATTN_WMMA_D72");
return env == nullptr || atoi(env) != 0;
}();
// Pad sufficiently large unmasked D72 self-attention to the existing D80 WMMA tile.
if (wmma_d72 && GGML_CUDA_CC_IS_RDNA3_5(cc) && amd_wmma_available(cc) && Q->ne[0] == 72 && V->ne[0] == 72 &&
gqa_ratio == 1 && !mask && !KQV->src[4] && max_bias == 0.0f && logit_softcap == 0.0f &&
Q->ne[1] >= 256 && ggml_nrows(Q) >= 4096 && ggml_nrows(Q) <= INT_MAX/80 &&
Q->ne[1] == K->ne[1] && K->ne[1] == V->ne[1] && Q->ne[2] == K->ne[2] && Q->ne[2] == V->ne[2] &&
Q->ne[3] == K->ne[3] && Q->ne[3] == V->ne[3] &&
Q->type == GGML_TYPE_F32 && K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16 &&
Q->nb[0] == sizeof(float) && K->nb[0] == sizeof(half) && V->nb[0] == sizeof(half) &&
KQV->type == GGML_TYPE_F32 && ggml_is_contiguous(KQV)) {
return BEST_FATTN_KERNEL_MMA_F16;
}
#endif

#if GGML_CUDA_FATTN_WMMA_NO_GQA
static const bool wmma_no_gqa = []() {
const char * env = getenv("GGML_CUDA_FATTN_WMMA_NO_GQA");
return env == nullptr || atoi(env) != 0;
}();
// Unmasked D64/D80 self-attention uses one head per WMMA tile.
if (wmma_no_gqa && GGML_CUDA_CC_IS_RDNA3_5(cc) && amd_wmma_available(cc) &&
(Q->ne[0] == 64 || Q->ne[0] == 80) && V->ne[0] == Q->ne[0] &&
gqa_ratio == 1 && !mask && !KQV->src[4] && !KQV->src[5] && max_bias == 0.0f && logit_softcap == 0.0f &&
Q->ne[1] >= 64 && Q->ne[1] == K->ne[1] && K->ne[1] == V->ne[1] &&
Q->ne[2] == K->ne[2] && K->ne[2] == V->ne[2] && Q->ne[3] == K->ne[3] && K->ne[3] == V->ne[3] &&
Q->type == GGML_TYPE_F32 && K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16 &&
Q->nb[0] == sizeof(float) && K->nb[0] == sizeof(half) && V->nb[0] == sizeof(half) &&
KQV->type == GGML_TYPE_F32 && ggml_is_contiguous(KQV)) {
return BEST_FATTN_KERNEL_MMA_F16;
}
#endif

// AMD WMMA is faster than the tile kernel if the wide tiles with high arithmetic intensity can be utilized.
if ((amd_wmma_available(cc) && gqa_opt_applies && Q->ne[0] <= 256) && Q->ne[0] != 40 && Q->ne[0] != 72 &&
Q->ne[1] * gqa_ratio_eff > (Q->ne[0] <= 128 ? 8 : 16)) {
Expand Down
28 changes: 28 additions & 0 deletions tests/test-backend-ops.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -12727,6 +12727,30 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_flash_attn_ext(512, 512, 4, { 4, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16));
test_cases.emplace_back(new test_flash_attn_ext(576, 512, 4, { 4, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16, {0, 1, 2, 3}, true, true));

// Vision WMMA dispatch boundaries, partial KV tiles, and strided batches.
for (int head_size : {64, 80}) {
for (int tokens : {63, 64, 65, 257}) {
test_cases.emplace_back(new test_flash_attn_ext(head_size, head_size, 4, {1, 1}, tokens, tokens, false));
}
test_cases.emplace_back(new test_flash_attn_ext(head_size, head_size, 4, {1, 2}, 129, 129, false, false, 0, 0,
GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3}));
}

// D72 padding dispatch boundaries, strided batches, and feature fallbacks.
for (int tokens : {255, 256, 257}) {
test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, tokens, tokens, false));
}
test_cases.emplace_back(new test_flash_attn_ext(72, 72, 15, {1, 1}, 256, 256, false));
test_cases.emplace_back(new test_flash_attn_ext(72, 72, 4, {1, 2}, 513, 513, false, false, 0, 0, GGML_PREC_F32,
GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3}));
test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 257, 257, true));
test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 257, 257, false, true));
test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 257, 257, true, false, 8.0f));
test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 257, 257, false, false, 0, 0, GGML_PREC_F32,
GGML_TYPE_F32, GGML_TYPE_F32));
test_cases.emplace_back(new test_flash_attn_ext(72, 72, 8, {2, 1}, 257, 257, false));
test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 513, 257, false));

// FLASH_ATTN_EXT: large logits
test_cases.emplace_back(new test_flash_attn_ext_large_logits( 64, 64, 16, {4, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
test_cases.emplace_back(new test_flash_attn_ext_large_logits(128, 128, 8, {4, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
Expand Down Expand Up @@ -13267,6 +13291,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
}
}

// Gemma-3 vision attention, including the D72 padding and output copy.
test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 4096, 4096, false, false, 0, 0, GGML_PREC_F32,
GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3}, false));

// Qwen3-VL-8B https://github.com/ggml-org/llama.cpp/issues/17012
test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 5776, 5776, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));

Expand Down
Loading