diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index 96b596ea6fb5..b3d603564893 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -3,6 +3,16 @@ #include "mma.cuh" #include "fattn-common.cuh" +#ifndef GGML_CUDA_FATTN_WMMA_NO_GQA +#define GGML_CUDA_FATTN_WMMA_NO_GQA 1 +#endif +#ifndef GGML_CUDA_FATTN_D64_FULL_TILE +#define GGML_CUDA_FATTN_D64_FULL_TILE 1 +#endif +#ifndef GGML_CUDA_FATTN_D80_FULL_TILE +#define GGML_CUDA_FATTN_D80_FULL_TILE 1 +#endif + using namespace ggml_cuda_mma; // Config options for the MMA kernel. @@ -1355,11 +1365,19 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( // kb0_start is always < kb0_stop so the last iter can be executed unconditionally. if constexpr (ncols2 == 1 || use_sparse) { constexpr bool oob_check = true; +#if defined(RDNA3_5) + // Only the last dense KV tile can be partial. + constexpr bool full_tile_oob_check = !(((DKQ == 80 && GGML_CUDA_FATTN_D80_FULL_TILE) || + (DKQ == 64 && GGML_CUDA_FATTN_D64_FULL_TILE)) && + DV == DKQ && ncols1 == 64 && ncols2 == 1 && !use_sparse); +#else + constexpr bool full_tile_oob_check = true; +#endif for (; kb0 < kb0_stop-1; ++kb0) { constexpr bool last_iter = false; constexpr int k_VKQ_sup = nbatch_fa; flash_attn_ext_f16_iter - (Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap, ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, @@ -1866,7 +1884,12 @@ static __global__ void flash_attn_ext_f16( #endif // __CUDA_ARCH__ == GGML_CUDA_CC_TURING #if defined(AMD_WMMA_AVAILABLE) - if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 256) { +#if defined(RDNA3_5) + constexpr bool vision_mha = (DKQ == 80 || (DKQ == 64 && GGML_CUDA_FATTN_WMMA_NO_GQA)) && DV == DKQ && ncols1 == 64; +#else + constexpr bool vision_mha = false; +#endif + if (ncols1*ncols2 < 16 || (ncols2 == 1 && !vision_mha) || DKQ > 256) { NO_DEVICE_CODE; return; } diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index 5d7ebf966378..ead21e286c72 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -5,6 +5,10 @@ #include "fattn-vec.cuh" #include "fattn.cuh" +#ifndef GGML_CUDA_FATTN_WMMA_D72 +#define GGML_CUDA_FATTN_WMMA_D72 1 +#endif + #if !defined(GGML_USE_MUSA) // one list per group of ncols1 queries: a column is selected if any query of the group can see it @@ -305,6 +309,68 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_con } } +#if GGML_CUDA_FATTN_WMMA_D72 +template +static __global__ void flash_attn_pad_head( + const char * src, T * dst, const int64_t n, const uint3 ne1, const uint3 ne2, + const size_t nb1, const size_t nb2, const size_t nb3) { + const int64_t i = int64_t(blockIdx.x)*blockDim.x + threadIdx.x; + if (i >= n) { + return; + } + const int col = i % Ddst; + const int row = i / Ddst; + const int i1 = fastmodulo(row, ne1); + const int i23 = fastdiv(row, ne1); + const int i2 = fastmodulo(i23, ne2); + const int i3 = fastdiv(i23, ne2); + dst[i] = col < Dsrc ? ((const T *) (src + i1*nb1 + i2*nb2 + i3*nb3))[col] : T(0.0f); +} + +template +static void ggml_cuda_flash_attn_pad_head(const ggml_tensor * src, ggml_tensor * dst, cudaStream_t stream) { + const int64_t n = ggml_nelements(dst); + const ggml_cuda_kernel_launch_params params(dim3((n + 255)/256), dim3(256), 0, stream); + ggml_cuda_kernel_launch(flash_attn_pad_head, params, + (const char *) src->data, (T *) dst->data, n, init_fastdiv_values(src->ne[1]), init_fastdiv_values(src->ne[2]), + src->nb[1], src->nb[2], src->nb[3]); + CUDA_CHECK(cudaGetLastError()); +} + +static void ggml_cuda_flash_attn_ext_mma_f16_pad72(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + ggml_tensor q = *dst->src[0]; + ggml_tensor k = *dst->src[1]; + ggml_tensor v = *dst->src[2]; + ggml_tensor out = *dst; + const int64_t rows = ggml_nrows(&q); + ggml_cuda_pool_alloc q_out(ctx.pool(), 2*80*rows); + ggml_cuda_pool_alloc kv(ctx.pool(), 2*80*rows); + q.data = q_out.get(); + k.data = kv.get(); + v.data = kv.get() + 80*rows; + out.data = q_out.get() + 80*rows; + for (ggml_tensor * t : {&q, &k, &v, &out}) { + t->ne[0] = 80; + for (int d = 1; d < GGML_MAX_DIMS; ++d) { + t->nb[d] = t->nb[d - 1]*t->ne[d - 1]; + } + t->view_src = nullptr; + t->view_offs = 0; + } + out.src[0] = &q; + out.src[1] = &k; + out.src[2] = &v; + const cudaStream_t stream = ctx.stream(); + ggml_cuda_flash_attn_pad_head(dst->src[0], &q, stream); + ggml_cuda_flash_attn_pad_head(dst->src[1], &k, stream); + ggml_cuda_flash_attn_pad_head(dst->src[2], &v, stream); + // Keep the original scale: padding changes the storage width, not the attention width. + ggml_cuda_flash_attn_ext_mma_f16_case<80, 80, 64, 1>(ctx, &out); + ggml_cuda_flash_attn_pad_head(&out, dst, stream); +} + +#endif + static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; const ggml_tensor * KQV = dst; @@ -318,6 +384,11 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg GGML_ASSERT(V->ne[0] == 64); ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2< 64, 64>(ctx, dst); break; +#if GGML_CUDA_FATTN_WMMA_D72 + case 72: + ggml_cuda_flash_attn_ext_mma_f16_pad72(ctx, dst); + break; +#endif case 80: GGML_ASSERT(V->ne[0] == 80); ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2< 80, 80>(ctx, dst); @@ -732,6 +803,47 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const } } +#if GGML_CUDA_FATTN_WMMA_D72 || GGML_CUDA_FATTN_WMMA_NO_GQA + float logit_softcap = 0.0f; + memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float)); +#endif + +#if GGML_CUDA_FATTN_WMMA_D72 + static const bool wmma_d72 = []() { + const char * env = getenv("GGML_CUDA_FATTN_WMMA_D72"); + return env == nullptr || atoi(env) != 0; + }(); + // Pad sufficiently large unmasked D72 self-attention to the existing D80 WMMA tile. + if (wmma_d72 && GGML_CUDA_CC_IS_RDNA3_5(cc) && amd_wmma_available(cc) && Q->ne[0] == 72 && V->ne[0] == 72 && + gqa_ratio == 1 && !mask && !KQV->src[4] && max_bias == 0.0f && logit_softcap == 0.0f && + Q->ne[1] >= 256 && ggml_nrows(Q) >= 4096 && ggml_nrows(Q) <= INT_MAX/80 && + Q->ne[1] == K->ne[1] && K->ne[1] == V->ne[1] && Q->ne[2] == K->ne[2] && Q->ne[2] == V->ne[2] && + Q->ne[3] == K->ne[3] && Q->ne[3] == V->ne[3] && + Q->type == GGML_TYPE_F32 && K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16 && + Q->nb[0] == sizeof(float) && K->nb[0] == sizeof(half) && V->nb[0] == sizeof(half) && + KQV->type == GGML_TYPE_F32 && ggml_is_contiguous(KQV)) { + return BEST_FATTN_KERNEL_MMA_F16; + } +#endif + +#if GGML_CUDA_FATTN_WMMA_NO_GQA + static const bool wmma_no_gqa = []() { + const char * env = getenv("GGML_CUDA_FATTN_WMMA_NO_GQA"); + return env == nullptr || atoi(env) != 0; + }(); + // Unmasked D64/D80 self-attention uses one head per WMMA tile. + if (wmma_no_gqa && GGML_CUDA_CC_IS_RDNA3_5(cc) && amd_wmma_available(cc) && + (Q->ne[0] == 64 || Q->ne[0] == 80) && V->ne[0] == Q->ne[0] && + gqa_ratio == 1 && !mask && !KQV->src[4] && !KQV->src[5] && max_bias == 0.0f && logit_softcap == 0.0f && + Q->ne[1] >= 64 && Q->ne[1] == K->ne[1] && K->ne[1] == V->ne[1] && + Q->ne[2] == K->ne[2] && K->ne[2] == V->ne[2] && Q->ne[3] == K->ne[3] && K->ne[3] == V->ne[3] && + Q->type == GGML_TYPE_F32 && K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16 && + Q->nb[0] == sizeof(float) && K->nb[0] == sizeof(half) && V->nb[0] == sizeof(half) && + KQV->type == GGML_TYPE_F32 && ggml_is_contiguous(KQV)) { + return BEST_FATTN_KERNEL_MMA_F16; + } +#endif + // AMD WMMA is faster than the tile kernel if the wide tiles with high arithmetic intensity can be utilized. if ((amd_wmma_available(cc) && gqa_opt_applies && Q->ne[0] <= 256) && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[1] * gqa_ratio_eff > (Q->ne[0] <= 128 ? 8 : 16)) { diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index a44ee04f3baa..08ba065db216 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -12727,6 +12727,30 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(512, 512, 4, { 4, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16)); test_cases.emplace_back(new test_flash_attn_ext(576, 512, 4, { 4, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16, {0, 1, 2, 3}, true, true)); + // Vision WMMA dispatch boundaries, partial KV tiles, and strided batches. + for (int head_size : {64, 80}) { + for (int tokens : {63, 64, 65, 257}) { + test_cases.emplace_back(new test_flash_attn_ext(head_size, head_size, 4, {1, 1}, tokens, tokens, false)); + } + test_cases.emplace_back(new test_flash_attn_ext(head_size, head_size, 4, {1, 2}, 129, 129, false, false, 0, 0, + GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3})); + } + + // D72 padding dispatch boundaries, strided batches, and feature fallbacks. + for (int tokens : {255, 256, 257}) { + test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, tokens, tokens, false)); + } + test_cases.emplace_back(new test_flash_attn_ext(72, 72, 15, {1, 1}, 256, 256, false)); + test_cases.emplace_back(new test_flash_attn_ext(72, 72, 4, {1, 2}, 513, 513, false, false, 0, 0, GGML_PREC_F32, + GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3})); + test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 257, 257, true)); + test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 257, 257, false, true)); + test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 257, 257, true, false, 8.0f)); + test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 257, 257, false, false, 0, 0, GGML_PREC_F32, + GGML_TYPE_F32, GGML_TYPE_F32)); + test_cases.emplace_back(new test_flash_attn_ext(72, 72, 8, {2, 1}, 257, 257, false)); + test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 513, 257, false)); + // FLASH_ATTN_EXT: large logits test_cases.emplace_back(new test_flash_attn_ext_large_logits( 64, 64, 16, {4, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext_large_logits(128, 128, 8, {4, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); @@ -13267,6 +13291,10 @@ static std::vector> make_test_cases_perf() { } } + // Gemma-3 vision attention, including the D72 padding and output copy. + test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 4096, 4096, false, false, 0, 0, GGML_PREC_F32, + GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3}, false)); + // Qwen3-VL-8B https://github.com/ggml-org/llama.cpp/issues/17012 test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 5776, 5776, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));