From 4e2780eda63bc9da21f7c6146bb3f2788f47715d Mon Sep 17 00:00:00 2001 From: lichang Date: Thu, 12 Nov 2026 16:34:10 -0700 Subject: [PATCH 1/4] hip: add dedicated RDNA3.5 prefill and vision attention Classify mask tiles and specialize D256 query groups. Keep the new paths opt-in with conservative shape and precision guards. Assisted-by: Codex --- ggml/src/ggml-cuda/fattn-vision.cu | 729 ++++++++++++++++++++++++++++ ggml/src/ggml-cuda/fattn-vision.cuh | 14 + ggml/src/ggml-cuda/fattn.cu | 18 + tests/test-backend-ops.cpp | 45 ++ 4 files changed, 806 insertions(+) create mode 100644 ggml/src/ggml-cuda/fattn-vision.cu create mode 100644 ggml/src/ggml-cuda/fattn-vision.cuh diff --git a/ggml/src/ggml-cuda/fattn-vision.cu b/ggml/src/ggml-cuda/fattn-vision.cu new file mode 100644 index 000000000000..a6a4bfe632dd --- /dev/null +++ b/ggml/src/ggml-cuda/fattn-vision.cu @@ -0,0 +1,729 @@ +#include "fattn-vision.cuh" + +#if defined(GGML_USE_HIP) && defined(RDNA3) +#define FATTN_VISION_DEVICE +#endif + +#ifdef FATTN_VISION_DEVICE +typedef _Float16 fv_h16 __attribute__((ext_vector_type(16))); +typedef float fv_f8 __attribute__((ext_vector_type(8))); +#endif + +static constexpr int FV_NW = 8; // waves per block, 16 queries per wave +static constexpr int FV_BQ = 16*FV_NW; // queries per block +static constexpr int FV_BK = 64; // keys per K/V tile + +#ifdef FATTN_VISION_DEVICE +static __device__ __forceinline__ fv_h16 fv_load16(const _Float16 * p) { + fv_h16 r; + ((uint4 *) &r)[0] = ((const uint4 *) p)[0]; + ((uint4 *) &r)[1] = ((const uint4 *) p)[1]; + return r; +} +#endif // FATTN_VISION_DEVICE + +// mask tile classes for the prefill path +enum : uint8_t { + FV_TILE_ZERO = 0, // all mask values are 0 + FV_TILE_INF = 1, // all mask values are -inf, the tile is skipped + FV_TILE_MIX = 2, // mask is added to the scores +}; + +// one block per (KV tile, query block): class of the mask values of the queries < nq and keys < nkv +template +static __global__ void __launch_bounds__(FV_NW*32, 1) flash_attn_ext_prefill_classify( + const char * __restrict__ mask, uint8_t * __restrict__ cls, const int nq, const int nkv, + const int64_t nbm1, const int64_t nbm3) { + const int t = blockIdx.x; + const int qb = blockIdx.y; + const int b = blockIdx.z; + mask += b*nbm3; + + int any_nonzero = 0; + int any_finite = 0; + constexpr int NCH = BK/8; // 16 byte chunks per mask row + for (int c = threadIdx.x; c < BQ*NCH; c += FV_NW*32) { + const int q = qb*BQ + c / NCH; + const int k = t*BK + (c % NCH)*8; + if (q >= nq || k >= nkv) { + continue; + } + const uint16_t * row = (const uint16_t *) (mask + q*nbm1); + uint16_t x[8]; + if (k + 8 <= nkv) { + *(uint4 *) x = *(const uint4 *) (row + k); + } else { + for (int j = 0; j < 8; ++j) { + x[j] = k + j < nkv ? row[k + j] : 0xFC00; + } + } +#pragma unroll + for (int j = 0; j < 8; ++j) { + any_nonzero |= (x[j] & 0x7FFF) != 0; + any_finite |= x[j] != 0xFC00; + } + } + any_nonzero = __syncthreads_or(any_nonzero); + any_finite = __syncthreads_or(any_finite); + if (threadIdx.x == 0) { + cls[(int64_t(b)*gridDim.y + qb)*gridDim.x + t] = !any_finite ? FV_TILE_INF : (any_nonzero ? FV_TILE_MIX : FV_TILE_ZERO); + } +} + +// Each wave computes S^T = K*Q^T and O^T += V^T*P^T for 16 queries, so one lane owns one query column. +// K/V tiles go through registers to LDS (V transposed); the next tile is loaded while the current one is used. +// MASK: f16 mask + GQA (prefill); tiles are classified by flash_attn_ext_prefill_classify, -inf tiles are skipped. +template +static __global__ void __launch_bounds__(FV_NW*32, 1) flash_attn_ext_vision( + const char * __restrict__ Q, const char * __restrict__ K, const char * __restrict__ V, float * __restrict__ dst, + const int nq, const int nkv, const int nh, const float qscale, + const int64_t nbq1, const int64_t nbq2, const int64_t nbq3, + const int64_t nbk1, const int64_t nbk2, const int64_t nbk3, + const int64_t nbv1, const int64_t nbv2, const int64_t nbv3, + const char * __restrict__ mask, const uint8_t * __restrict__ cls, const int gqa, + const int64_t nbm1, const int64_t nbm3, const int nbm) { +#ifdef FATTN_VISION_DEVICE + constexpr int NCH = D/8; // 16 byte chunks per K/V row + constexpr int NKD = DP/16; // WMMA steps over the head dim + constexpr int NT = FV_NW*32; + constexpr int NLD = (FV_BK/2*NCH + NT - 1)/NT; + constexpr int SK = DP + 8; // Ks row stride + constexpr int SV = FV_BK + 8; // Vt row stride + + __shared__ __align__(16) _Float16 Ks[FV_BK*SK]; + __shared__ __align__(16) _Float16 Vt[DP*SV]; + + const int tid = threadIdx.x; + const int lane = tid % 32; + const int wave = tid / 32; + const int lr = lane % 16; + const int hl = lane / 16; + + // MASK: heads in x and the longest (last) causal query blocks first + const int h = MASK ? blockIdx.x : blockIdx.y; + const int qblk = MASK ? gridDim.y - 1 - blockIdx.y : blockIdx.x; + const int b = blockIdx.z; + const int hk = MASK ? h / gqa : h; + + Q += h*nbq2 + b*nbq3; + K += hk*nbk2 + b*nbk3; + V += hk*nbv2 + b*nbv3; + + if constexpr (D < DP) { + for (int i = tid; i < FV_BK*(DP - D); i += NT) { + Ks[(i / (DP - D))*SK + D + i % (DP - D)] = 0; + } + for (int i = tid; i < (DP - D)*SV; i += NT) { + Vt[D*SV + i] = 0; + } + } + + const int q = qblk*FV_BQ + wave*16 + lr; + fv_h16 qb[NKD]; +#pragma unroll + for (int c = 0; c < NKD; ++c) { +#pragma unroll + for (int j = 0; j < 16; ++j) { + const int d = c*16 + j; + float x = 0.0f; + if (d < D && q < nq) { + x = ((const float *) (Q + q*nbq1))[d] * qscale; + } + qb[c][j] = (_Float16) x; + } + } + + // each load item is one 16 byte chunk of two consecutive K/V rows, so V can be written transposed as 32 bit pairs + uint4 kr[NLD][2]; + uint4 vr[NLD][2]; + auto load_regs = [&](const int k0) { +#pragma unroll + for (int i = 0; i < NLD; ++i) { + const int c = tid + i*NT; + const int row = 2*(c / NCH); + const int ch = c % NCH; +#pragma unroll + for (int r = 0; r < 2; ++r) { + kr[i][r] = make_uint4(0, 0, 0, 0); + vr[i][r] = make_uint4(0, 0, 0, 0); + if (c < FV_BK/2*NCH && k0 + row + r < nkv) { + kr[i][r] = *(const uint4 *) (K + (k0 + row + r)*nbk1 + ch*16); + vr[i][r] = *(const uint4 *) (V + (k0 + row + r)*nbv1 + ch*16); + } + } + } + }; + auto store_lds = [&]() { +#pragma unroll + for (int i = 0; i < NLD; ++i) { + const int c = tid + i*NT; + const int row = 2*(c / NCH); + const int ch = c % NCH; + if (c < FV_BK/2*NCH) { + *(uint4 *) (Ks + (row + 0)*SK + ch*8) = kr[i][0]; + *(uint4 *) (Ks + (row + 1)*SK + ch*8) = kr[i][1]; + const uint32_t * v0 = (const uint32_t *) &vr[i][0]; + const uint32_t * v1 = (const uint32_t *) &vr[i][1]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + *(uint32_t *) (Vt + (ch*8 + 2*j + 0)*SV + row) = (v0[j] & 0x0000FFFFu) | (v1[j] << 16); + *(uint32_t *) (Vt + (ch*8 + 2*j + 1)*SV + row) = (v0[j] >> 16) | (v1[j] & 0xFFFF0000u); + } + } + } + }; + + float m = -INFINITY; // running max of the query column (log2 domain) + float l = 0.0f; // partial row sum, the other half-wave holds the rest + fv_f8 o[NKD]; +#pragma unroll + for (int dt = 0; dt < NKD; ++dt) { + o[dt] = fv_f8{0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + } + + const int ntiles = (nkv + FV_BK - 1)/FV_BK; + const uint8_t * cl = nullptr; + const half * mrow = nullptr; + if constexpr (MASK) { + cl = cls + (int64_t(nbm == 1 ? 0 : b)*gridDim.y + qblk)*ntiles; + mrow = (const half *) (mask + (nbm == 1 ? 0 : b)*nbm3 + int64_t(q < nq ? q : 0)*nbm1); + } + auto next_tile = [&](int t) { + if constexpr (MASK) { + while (t < ntiles && cl[t] == FV_TILE_INF) { + ++t; + } + } + return t; + }; + + int t = next_tile(0); + if (t < ntiles) { + load_regs(t*FV_BK); + } + while (t < ntiles) { + __syncthreads(); + store_lds(); + __syncthreads(); + const int t_next = next_tile(t + 1); + if (t_next < ntiles) { + load_regs(t_next*FV_BK); + } + + // s[st][v] = S^T[key st*16 + 2*v + hl][query lr] + fv_f8 s[4]; +#pragma unroll + for (int st = 0; st < 4; ++st) { + s[st] = fv_f8{0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; +#pragma unroll + for (int c = 0; c < NKD; ++c) { + const fv_h16 a = fv_load16(Ks + (st*16 + lr)*SK + c*16); + s[st] = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a, qb[c], s[st]); + } + } + + const int kb = t*FV_BK; + if (kb + FV_BK > nkv) { +#pragma unroll + for (int st = 0; st < 4; ++st) { +#pragma unroll + for (int v = 0; v < 8; ++v) { + if (kb + st*16 + 2*v + hl >= nkv) { + s[st][v] = -INFINITY; + } + } + } + } + if constexpr (MASK) { + if (cl[t] == FV_TILE_MIX && q < nq) { +#pragma unroll + for (int st = 0; st < 4; ++st) { + const int k0 = kb + st*16; // nkv % 16 == 0 + if (k0 < nkv) { + uint32_t mw[8]; + ((uint4 *) mw)[0] = ((const uint4 *) (mrow + k0))[0]; + ((uint4 *) mw)[1] = ((const uint4 *) (mrow + k0))[1]; +#pragma unroll + for (int v = 0; v < 8; ++v) { + const uint16_t bits = hl ? mw[v] >> 16 : mw[v] & 0xFFFF; + s[st][v] += 1.44269504088896341f*__half2float(__ushort_as_half(bits)); + } + } + } + } + } + + float m_new = m; +#pragma unroll + for (int st = 0; st < 4; ++st) { +#pragma unroll + for (int v = 0; v < 8; ++v) { + m_new = fmaxf(m_new, s[st][v]); + } + } + m_new = fmaxf(m_new, __shfl_xor(m_new, 16, 32)); + // a query column with only -inf so far keeps o = l = 0 + const float m_exp = MASK && m_new == -INFINITY ? 0.0f : m_new; + const float alpha = __builtin_amdgcn_exp2f(m - m_exp); + m = m_new; + l *= alpha; +#pragma unroll + for (int dt = 0; dt < NKD; ++dt) { + o[dt] *= alpha; + } + + // P^T as WMMA B operand: both half-waves need all 16 keys of the query column, keys 2*v + hl come from this lane +#pragma unroll + for (int st = 0; st < 4; ++st) { + union { + fv_h16 h; + uint32_t u[8]; + } pb; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const float p0 = __builtin_amdgcn_exp2f(s[st][2*j + 0] - m_exp); + const float p1 = __builtin_amdgcn_exp2f(s[st][2*j + 1] - m_exp); + l += p0 + p1; + half2 w2 = __floats2half2_rn(p0, p1); + const uint32_t w = *(const uint32_t *) &w2; + const uint32_t x = __shfl_xor(w, 16, 32); + const uint32_t lo = hl ? x : w; + const uint32_t hi = hl ? w : x; + pb.u[2*j + 0] = (lo & 0x0000FFFFu) | (hi << 16); + pb.u[2*j + 1] = (lo >> 16) | (hi & 0xFFFF0000u); + } + + // o[dt][v] = O^T[dim dt*16 + 2*v + hl][query lr] +#pragma unroll + for (int dt = 0; dt < NKD; ++dt) { + const fv_h16 a = fv_load16(Vt + (dt*16 + lr)*SV + st*16); + o[dt] = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a, pb.h, o[dt]); + } + } + t = t_next; + } + + l += __shfl_xor(l, 16, 32); + if (q < nq) { + const float il = MASK && l == 0.0f ? 0.0f : 1.0f/l; + float * dst_q = dst + ((int64_t(b)*nq + q)*nh + h)*D; +#pragma unroll + for (int dt = 0; dt < NKD; ++dt) { +#pragma unroll + for (int v = 0; v < 8; ++v) { + const int d = dt*16 + 2*v + hl; + if (d < D) { + dst_q[d] = o[dt][v]*il; + } + } + } + } +#else + GGML_UNUSED_VARS(Q, K, V, dst, nq, nkv, nh, qscale, nbq1, nbq2, nbq3, nbk1, nbk2, nbk3, nbv1, nbv2, nbv3, mask, cls, gqa, nbm1, nbm3, nbm); + NO_DEVICE_CODE; +#endif // FATTN_VISION_DEVICE +} + +// D=256 prefill: the 16 queries of a group are handled by two waves, each owns half of the head dim of Q and O. +// The partial S of the two waves are summed through LDS, so both compute the same softmax. +static constexpr int FS_BK = 32; // keys per K/V tile + +// NG query groups of 16 per block; NG = 8 puts the S exchange buffer over the K tile to fit in LDS +// SXH: exchange the partial S in f16 (both waves round both halves the same way), so the buffer fits next to the K tile and one barrier per tile is saved +template +static __global__ void __launch_bounds__(NG*2*32, 1) flash_attn_ext_prefill_split( + const char * __restrict__ Q, const char * __restrict__ K, const char * __restrict__ V, float * __restrict__ dst, + const int nq, const int nkv, const int nh, const float qscale, + const int64_t nbq1, const int64_t nbq2, const int64_t nbq3, + const int64_t nbk1, const int64_t nbk2, const int64_t nbk3, + const int64_t nbv1, const int64_t nbv2, const int64_t nbv3, + const char * __restrict__ mask, const uint8_t * __restrict__ cls, const int gqa, + const int64_t nbm1, const int64_t nbm3, const int nbm) { +#ifdef FATTN_VISION_DEVICE + constexpr int DH = D/2; + constexpr int NCH = D/8; + constexpr int NKD = DH/16; + constexpr int NT = NG*2*32; + constexpr bool SX_OVER_K = NG > 4 && !SXH; + constexpr int NLD = (FS_BK/2*NCH + NT - 1)/NT; + constexpr int SK = D + 8; + constexpr int SV = FS_BK + 8; + + constexpr int KS_BYTES = FS_BK*SK*sizeof(_Float16); + constexpr int SX_BYTES = 2*NG*32*16*(SXH ? sizeof(_Float16) : sizeof(float)); + __shared__ __align__(16) char smem[(SX_OVER_K ? (KS_BYTES > SX_BYTES ? KS_BYTES : SX_BYTES) : KS_BYTES + SX_BYTES) + D*SV*sizeof(_Float16)]; + _Float16 * Ks = (_Float16 *) smem; + float (*Sx)[32][16] = (float (*)[32][16]) (smem + (SX_OVER_K ? 0 : KS_BYTES)); + _Float16 (*Sxh)[32][16] = (_Float16 (*)[32][16]) (smem + KS_BYTES); + _Float16 * Vt = (_Float16 *) (smem + (SX_OVER_K ? (KS_BYTES > SX_BYTES ? KS_BYTES : SX_BYTES) : KS_BYTES + SX_BYTES)); + + const int tid = threadIdx.x; + const int lane = tid % 32; + const int wave = tid / 32; + const int lr = lane % 16; + const int hl = lane / 16; + const int hd = wave % 2; + + const int h = blockIdx.x; + const int qblk = gridDim.y - 1 - blockIdx.y; + const int b = blockIdx.z; + const int hk = h / gqa; + + Q += h*nbq2 + b*nbq3; + K += hk*nbk2 + b*nbk3; + V += hk*nbv2 + b*nbv3; + + const int q = qblk*16*NG + (wave/2)*16 + lr; + fv_h16 qb[NKD]; +#pragma unroll + for (int c = 0; c < NKD; ++c) { +#pragma unroll + for (int j = 0; j < 16; ++j) { + const float x = q < nq ? ((const float *) (Q + q*nbq1))[hd*DH + c*16 + j] * qscale : 0.0f; + qb[c][j] = (_Float16) x; + } + } + + uint4 kr[NLD][2]; + uint4 vr[NLD][2]; + auto load_regs = [&](const int k0) { +#pragma unroll + for (int i = 0; i < NLD; ++i) { + const int c = tid + i*NT; + const int row = 2*(c / NCH); + const int ch = c % NCH; +#pragma unroll + for (int r = 0; r < 2; ++r) { + kr[i][r] = make_uint4(0, 0, 0, 0); + vr[i][r] = make_uint4(0, 0, 0, 0); + if (c < FS_BK/2*NCH && k0 + row + r < nkv) { + kr[i][r] = *(const uint4 *) (K + (k0 + row + r)*nbk1 + ch*16); + vr[i][r] = *(const uint4 *) (V + (k0 + row + r)*nbv1 + ch*16); + } + } + } + }; + auto store_lds = [&]() { +#pragma unroll + for (int i = 0; i < NLD; ++i) { + const int c = tid + i*NT; + const int row = 2*(c / NCH); + const int ch = c % NCH; + if (c < FS_BK/2*NCH) { + *(uint4 *) (Ks + (row + 0)*SK + ch*8) = kr[i][0]; + *(uint4 *) (Ks + (row + 1)*SK + ch*8) = kr[i][1]; + const uint32_t * v0 = (const uint32_t *) &vr[i][0]; + const uint32_t * v1 = (const uint32_t *) &vr[i][1]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + *(uint32_t *) (Vt + (ch*8 + 2*j + 0)*SV + row) = (v0[j] & 0x0000FFFFu) | (v1[j] << 16); + *(uint32_t *) (Vt + (ch*8 + 2*j + 1)*SV + row) = (v0[j] >> 16) | (v1[j] & 0xFFFF0000u); + } + } + } + }; + + float m = -INFINITY; + float l = 0.0f; + fv_f8 o[NKD]; +#pragma unroll + for (int dt = 0; dt < NKD; ++dt) { + o[dt] = fv_f8{0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + } + + const int ntiles = (nkv + FS_BK - 1)/FS_BK; + const uint8_t * cl = cls + (int64_t(nbm == 1 ? 0 : b)*gridDim.y + qblk)*ntiles; + const half * mrow = (const half *) (mask + (nbm == 1 ? 0 : b)*nbm3 + int64_t(q < nq ? q : 0)*nbm1); + auto next_tile = [&](int t) { + while (t < ntiles && cl[t] == FV_TILE_INF) { + ++t; + } + return t; + }; + + int t = next_tile(0); + if (t < ntiles) { + load_regs(t*FS_BK); + } + while (t < ntiles) { + __syncthreads(); + store_lds(); + __syncthreads(); + const int t_next = next_tile(t + 1); + if (t_next < ntiles) { + load_regs(t_next*FS_BK); + } + + fv_f8 s[2]; +#pragma unroll + for (int st = 0; st < 2; ++st) { + s[st] = fv_f8{0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; +#pragma unroll + for (int c = 0; c < NKD; ++c) { + const fv_h16 a = fv_load16(Ks + (st*16 + lr)*SK + hd*DH + c*16); + s[st] = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a, qb[c], s[st]); + } + } + if constexpr (SX_OVER_K) { + __syncthreads(); + } + if constexpr (SXH) { + typedef _Float16 fv_h8 __attribute__((ext_vector_type(8))); +#pragma unroll + for (int st = 0; st < 2; ++st) { + const fv_h8 own = __builtin_convertvector(s[st], fv_h8); + *(fv_h8 *) &Sxh[wave][lane][st*8] = own; + s[st] = __builtin_convertvector(own, fv_f8); + } + __syncthreads(); +#pragma unroll + for (int st = 0; st < 2; ++st) { + typedef _Float16 fv_h8b __attribute__((ext_vector_type(8))); + s[st] += __builtin_convertvector(*(const fv_h8b *) &Sxh[wave ^ 1][lane][st*8], fv_f8); + } + } else { +#pragma unroll + for (int st = 0; st < 2; ++st) { + *(fv_f8 *) &Sx[wave][lane][st*8] = s[st]; + } + __syncthreads(); +#pragma unroll + for (int st = 0; st < 2; ++st) { + s[st] += *(const fv_f8 *) &Sx[wave ^ 1][lane][st*8]; + } + } + + const int kb = t*FS_BK; +#pragma unroll + for (int st = 0; st < 2; ++st) { + const int k0 = kb + st*16; // nkv % 16 == 0 + if (k0 >= nkv) { + s[st] = fv_f8{-INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY}; + } else if (cl[t] == FV_TILE_MIX && q < nq) { + uint32_t mw[8]; + ((uint4 *) mw)[0] = ((const uint4 *) (mrow + k0))[0]; + ((uint4 *) mw)[1] = ((const uint4 *) (mrow + k0))[1]; +#pragma unroll + for (int v = 0; v < 8; ++v) { + const uint16_t bits = hl ? mw[v] >> 16 : mw[v] & 0xFFFF; + s[st][v] += 1.44269504088896341f*__half2float(__ushort_as_half(bits)); + } + } + } + + float m_new = m; +#pragma unroll + for (int st = 0; st < 2; ++st) { +#pragma unroll + for (int v = 0; v < 8; ++v) { + m_new = fmaxf(m_new, s[st][v]); + } + } + m_new = fmaxf(m_new, __shfl_xor(m_new, 16, 32)); + const float m_exp = m_new == -INFINITY ? 0.0f : m_new; + const float alpha = __builtin_amdgcn_exp2f(m - m_exp); + m = m_new; + l *= alpha; +#pragma unroll + for (int dt = 0; dt < NKD; ++dt) { + o[dt] *= alpha; + } + +#pragma unroll + for (int st = 0; st < 2; ++st) { + union { + fv_h16 h; + uint32_t u[8]; + } pb; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const float p0 = __builtin_amdgcn_exp2f(s[st][2*j + 0] - m_exp); + const float p1 = __builtin_amdgcn_exp2f(s[st][2*j + 1] - m_exp); + l += p0 + p1; + half2 w2 = __floats2half2_rn(p0, p1); + const uint32_t w = *(const uint32_t *) &w2; + const uint32_t x = __shfl_xor(w, 16, 32); + const uint32_t lo = hl ? x : w; + const uint32_t hi = hl ? w : x; + pb.u[2*j + 0] = (lo & 0x0000FFFFu) | (hi << 16); + pb.u[2*j + 1] = (lo >> 16) | (hi & 0xFFFF0000u); + } +#pragma unroll + for (int dt = 0; dt < NKD; ++dt) { + const fv_h16 a = fv_load16(Vt + (hd*DH + dt*16 + lr)*SV + st*16); + o[dt] = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a, pb.h, o[dt]); + } + } + t = t_next; + } + + l += __shfl_xor(l, 16, 32); + if (q < nq) { + const float il = l == 0.0f ? 0.0f : 1.0f/l; + float * dst_q = dst + ((int64_t(b)*nq + q)*nh + h)*D + hd*DH; +#pragma unroll + for (int dt = 0; dt < NKD; ++dt) { +#pragma unroll + for (int v = 0; v < 8; ++v) { + dst_q[dt*16 + 2*v + hl] = o[dt][v]*il; + } + } + } +#else + GGML_UNUSED_VARS(Q, K, V, dst, nq, nkv, nh, qscale, nbq1, nbq2, nbq3, nbk1, nbk2, nbk3, nbv1, nbv2, nbv3, mask, cls, gqa, nbm1, nbm3, nbm); + NO_DEVICE_CODE; +#endif // FATTN_VISION_DEVICE +} + +static bool fattn_vision_common_supported(int cc, const ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + const ggml_tensor * sinks = dst->src[4]; + + if (!GGML_CUDA_CC_IS_RDNA3_5(cc) || sinks || dst->src[5] || + (ggml_prec_get_acc(dst) != GGML_PREC_DEFAULT && ggml_prec_get_acc(dst) != GGML_PREC_F32) || K->ne[2] <= 0) { + return false; + } + float max_bias = 0.0f; + float logit_softcap = 0.0f; + memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float)); + memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float)); + if (max_bias != 0.0f || logit_softcap != 0.0f) { + return false; + } + if (Q->type != GGML_TYPE_F32 || K->type != GGML_TYPE_F16 || V->type != GGML_TYPE_F16 || dst->type != GGML_TYPE_F32) { + return false; + } + if (K->ne[0] != Q->ne[0] || V->ne[0] != Q->ne[0] || K->ne[3] != Q->ne[3] || V->ne[3] != Q->ne[3] || K->ne[1] != V->ne[1] || K->ne[2] != V->ne[2]) { + return false; + } + if (Q->nb[0] != sizeof(float) || K->nb[0] != sizeof(half) || V->nb[0] != sizeof(half) || !ggml_is_contiguous(dst)) { + return false; + } + for (const ggml_tensor * t : {K, V}) { + if ((uintptr_t) t->data % 16 != 0 || t->nb[1] % 16 != 0 || t->nb[2] % 16 != 0 || t->nb[3] % 16 != 0) { + return false; + } + } + return Q->ne[1] >= FV_BQ/2 && K->ne[1] >= 1 && Q->ne[1] <= INT_MAX && K->ne[1] <= INT_MAX; +} + +bool ggml_cuda_flash_attn_ext_vision_supported(int cc, const ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const int64_t D = Q->ne[0]; + return !dst->src[3] && (D == 64 || D == 72 || D == 80) && K->ne[2] == Q->ne[2] && fattn_vision_common_supported(cc, dst); +} + +bool ggml_cuda_flash_attn_ext_prefill_supported(int cc, const ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * mask = dst->src[3]; + const int64_t D = Q->ne[0]; + if (!mask || mask->type != GGML_TYPE_F16 || mask->ne[2] != 1 || (mask->ne[3] != 1 && mask->ne[3] != Q->ne[3])) { + return false; + } + if (K->ne[1] % 16 != 0 || mask->ne[0] < K->ne[1] || mask->ne[1] < Q->ne[1] || mask->nb[0] != sizeof(half) || + (uintptr_t) mask->data % 16 != 0 || mask->nb[1] % 16 != 0 || (mask->ne[3] > 1 && mask->nb[3] % 16 != 0)) { + return false; + } + return (D == 64 || D == 128 || D == 256) && Q->ne[2] % K->ne[2] == 0 && fattn_vision_common_supported(cc, dst); +} + +static void fattn_vision_launch(ggml_backend_cuda_context & ctx, ggml_tensor * dst, const bool use_mask) { + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + const ggml_tensor * mask = use_mask ? dst->src[3] : nullptr; + + float scale = 1.0f; + memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float)); + const float qscale = scale * 1.44269504088896341f; // exp -> exp2 + + const int nq = Q->ne[1]; + const int nkv = K->ne[1]; + const int nh = Q->ne[2]; + const bool split = mask && Q->ne[0] == 256; + // D256 uses 128 queries per block by default; 64 queries remain available for comparison. + static const int split_ng = getenv("GGML_CUDA_FATTN_PREFILL_D256_NG") && atoi(getenv("GGML_CUDA_FATTN_PREFILL_D256_NG")) == 4 ? 4 : 8; + const int bq = split ? 16*split_ng : FV_BQ; + const int bk = split ? FS_BK : FV_BK; + const int nqb = (nq + bq - 1)/bq; + const int ntk = (nkv + bk - 1)/bk; + const int nbm = mask ? mask->ne[3] : 0; + + const dim3 block(FV_NW*32, 1, 1); + cudaStream_t stream = ctx.stream(); + + ggml_cuda_pool_alloc cls_local(ctx.pool()); + uint8_t * cls = nullptr; + if (mask) { + cls = cls_local.alloc(size_t(nbm)*nqb*ntk); + if (split && split_ng == 8) { + flash_attn_ext_prefill_classify<128, FS_BK><<>>( + (const char *) mask->data, cls, nq, nkv, mask->nb[1], mask->nb[3]); + } else if (split) { + flash_attn_ext_prefill_classify<64, FS_BK><<>>( + (const char *) mask->data, cls, nq, nkv, mask->nb[1], mask->nb[3]); + } else { + flash_attn_ext_prefill_classify<<>>( + (const char *) mask->data, cls, nq, nkv, mask->nb[1], mask->nb[3]); + } + CUDA_CHECK(cudaGetLastError()); + } + const dim3 grid = mask ? dim3(nh, nqb, Q->ne[3]) : dim3(nqb, nh, Q->ne[3]); + + if (split) { +#define FS_LAUNCH(NG_, SXH_) \ + flash_attn_ext_prefill_split<256, NG_, SXH_><<>>( \ + (const char *) Q->data, (const char *) K->data, (const char *) V->data, (float *) dst->data, \ + nq, nkv, nh, qscale, Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], V->nb[3], \ + (const char *) mask->data, cls, nh / K->ne[2], mask->nb[1], mask->nb[3], nbm) + // opt-in (GGML_CUDA_FATTN_PREFILL_D256_SXH): f16 exchange of the partial scores, one barrier less per tile + static const bool sxh = getenv("GGML_CUDA_FATTN_PREFILL_D256_SXH") && atoi(getenv("GGML_CUDA_FATTN_PREFILL_D256_SXH")) != 0; + if (split_ng == 8) { + if (sxh) { + FS_LAUNCH(8, true); + } else { + FS_LAUNCH(8, false); + } + } else { + FS_LAUNCH(4, false); + } +#undef FS_LAUNCH + CUDA_CHECK(cudaGetLastError()); + return; + } + +#define FV_LAUNCH(D_, DP_, M_) \ + flash_attn_ext_vision<<>>( \ + (const char *) Q->data, (const char *) K->data, (const char *) V->data, (float *) dst->data, \ + nq, nkv, nh, qscale, Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], V->nb[3], \ + mask ? (const char *) mask->data : nullptr, cls, nh / K->ne[2], mask ? mask->nb[1] : 0, mask ? mask->nb[3] : 0, nbm) + + if (mask) { + switch (Q->ne[0]) { + case 64: FV_LAUNCH( 64, 64, true); break; + case 128: FV_LAUNCH(128, 128, true); break; + default: GGML_ABORT("fatal error"); + } + } else { + switch (Q->ne[0]) { + case 64: FV_LAUNCH(64, 64, false); break; + case 72: FV_LAUNCH(72, 80, false); break; + case 80: FV_LAUNCH(80, 80, false); break; + default: GGML_ABORT("fatal error"); + } + } +#undef FV_LAUNCH + CUDA_CHECK(cudaGetLastError()); +} + +void ggml_cuda_flash_attn_ext_vision(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + fattn_vision_launch(ctx, dst, false); +} + +void ggml_cuda_flash_attn_ext_prefill(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + fattn_vision_launch(ctx, dst, true); +} diff --git a/ggml/src/ggml-cuda/fattn-vision.cuh b/ggml/src/ggml-cuda/fattn-vision.cuh new file mode 100644 index 000000000000..4313f528e059 --- /dev/null +++ b/ggml/src/ggml-cuda/fattn-vision.cuh @@ -0,0 +1,14 @@ +#pragma once + +#include "common.cuh" + +// opt-in (GGML_CUDA_FATTN_VISION=1): RDNA3.5 WMMA flash attention for vision encoders (no mask, no GQA, D 64/72/80) +bool ggml_cuda_flash_attn_ext_vision_supported(int cc, const ggml_tensor * dst); + +void ggml_cuda_flash_attn_ext_vision(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +// opt-in (GGML_CUDA_FATTN_PREFILL=1): same kernel with an f16 mask and GQA for LLM prefill (D 64/128/256) +bool ggml_cuda_flash_attn_ext_prefill_supported(int cc, const ggml_tensor * dst); + +void ggml_cuda_flash_attn_ext_prefill(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index 5d7ebf966378..32eaa388c9cc 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -4,6 +4,7 @@ #include "fattn-tile.cuh" #include "fattn-vec.cuh" #include "fattn.cuh" +#include "fattn-vision.cuh" #if !defined(GGML_USE_MUSA) @@ -793,6 +794,23 @@ size_t ggml_cuda_flash_attn_ext_get_alloc_size(int device, const ggml_tensor * d void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ggml_cuda_set_device(ctx.device); + static const bool fattn_vision = []() { + const char * env = getenv("GGML_CUDA_FATTN_VISION"); + return env != nullptr && atoi(env) != 0; + }(); + if (fattn_vision && ggml_cuda_flash_attn_ext_vision_supported(ggml_cuda_info().devices[ctx.device].cc, dst)) { + ggml_cuda_flash_attn_ext_vision(ctx, dst); + return; + } + static const bool fattn_prefill = []() { + const char * env = getenv("GGML_CUDA_FATTN_PREFILL"); + return env != nullptr && atoi(env) != 0; + }(); + if (fattn_prefill && ggml_cuda_flash_attn_ext_prefill_supported(ggml_cuda_info().devices[ctx.device].cc, dst)) { + ggml_cuda_flash_attn_ext_prefill(ctx, dst); + return; + } + switch (ggml_cuda_get_best_fattn_kernel(ggml_cuda_get_device(), dst)) { case BEST_FATTN_KERNEL_NONE: GGML_ABORT("fatal error"); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 3e52e4abe10c..2a62234f20c6 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -8814,6 +8814,37 @@ struct test_flash_attn_ext : public test_case { } }; +// Exercise all mask classes, query tails, GQA, and batch-specific masks. +struct test_rdna_prefill_attention : public test_flash_attn_ext { + const int mask_kind; + test_rdna_prefill_attention(int d, int queries, int keys, int gqa, int batches, int mask_kind) + : test_flash_attn_ext(d, d, 2, {gqa, batches}, keys, queries, mask_kind >= 0, false, + 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16), mask_kind(mask_kind) {} + std::string op_desc(ggml_tensor *) override { return "RDNA_PREFILL_ATTN"; } + std::string vars() override { return test_flash_attn_ext::vars() + "," + VAR_TO_STR(mask_kind); } + void initialize_tensors(ggml_context * ctx) override { + test_flash_attn_ext::initialize_tensors(ctx); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t; t = ggml_get_next_tensor(ctx, t)) { + if (strcmp(t->name, "m") != 0) { continue; } + std::vector values(ggml_nelements(t)); + for (int64_t b = 0; b < t->ne[3]; ++b) { + for (int64_t q = 0; q < t->ne[1]; ++q) { + for (int64_t k = 0; k < t->ne[0]; ++k) { + float v = 0.0f; + const int64_t past = kv - nb; + if (mask_kind == 1 && k > past + q) { v = -INFINITY; } + if (mask_kind == 2 && (k > past + q || k + 64 < past + q)) { v = -INFINITY; } + if (mask_kind == 3) { v = (k + q + b) % 7 == 0 ? -INFINITY : -0.125f * ((k + b) % 4); } + if (mask_kind == 4) { v = k < 64 ? -0.0f : -INFINITY; } + values[(b*t->ne[1] + q)*t->ne[0] + k] = ggml_fp32_to_fp16(v); + } + } + } + ggml_backend_tensor_set(t, values.data(), 0, values.size()*sizeof(values[0])); + } + } +}; + // large Q values, so the online softmax has to rescale the partial results struct test_flash_attn_ext_large_logits : public test_flash_attn_ext { static constexpr int q_range = 20; @@ -10337,6 +10368,20 @@ static void add_rdna35_mmq_cases(std::vector> & test_ // Test cases for evaluation: should try to cover edge cases while using small input sizes to keep the runtime low static std::vector> make_test_cases_eval() { std::vector> test_cases; + + for (int d : {64, 128, 256}) { + for (int mask_kind : {0, 1, 2, 3, 4}) { + test_cases.emplace_back(new test_rdna_prefill_attention(d, 129, 272, 4, 1, mask_kind)); + test_cases.emplace_back(new test_rdna_prefill_attention(d, 64, 128, 1, 2, mask_kind)); + } + } + for (int d : {64, 72, 80}) { + test_cases.emplace_back(new test_rdna_prefill_attention(d, 129, 197, 1, 1, -1)); + test_cases.emplace_back(new test_rdna_prefill_attention(d, 64, 64, 1, 2, -1)); + } + // Small query counts retain the existing dispatch. + test_cases.emplace_back(new test_rdna_prefill_attention(128, 1, 128, 4, 1, 1)); + std::default_random_engine rng(0); // unary ops From d4ec6e12276270203021f2d3ee2f6aa1d957bfa3 Mon Sep 17 00:00:00 2001 From: lichang Date: Thu, 12 Nov 2026 19:04:31 -0700 Subject: [PATCH 2/4] hip: reuse read-only mask classification within a graph Reserve classification scratch before node temporaries and release it after graph evaluation. Reuse leaf masks only when no graph node writes overlapping storage, and keep reuse on the main stream. Assisted-by: Codex --- ggml/src/ggml-cuda/common.cuh | 7 +++ ggml/src/ggml-cuda/fattn-vision.cu | 86 ++++++++++++++++++++++++++++- ggml/src/ggml-cuda/fattn-vision.cuh | 3 + ggml/src/ggml-cuda/ggml-cuda.cu | 4 ++ tests/test-backend-ops.cpp | 26 +++++++++ 5 files changed, 124 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 4d7f945673ac..4af84d1dfb45 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1579,6 +1579,13 @@ struct ggml_backend_cuda_context { ggml_cuda_stream_context concurrent_stream_context; + struct { + const ggml_tensor * mask = nullptr; + int64_t key[7] = {}; + std::vector inputs; + std::unique_ptr> buf; + } fattn_cls; + // dedicated buffer for the branches of overlapped concurrent regions (attention QKV, MoE shared // expert), reused across layers so their scratch never aliases tensors read across the region ggml_backend_buffer_t concurrent_scratch = nullptr; diff --git a/ggml/src/ggml-cuda/fattn-vision.cu b/ggml/src/ggml-cuda/fattn-vision.cu index a6a4bfe632dd..7063422cd7f0 100644 --- a/ggml/src/ggml-cuda/fattn-vision.cu +++ b/ggml/src/ggml-cuda/fattn-vision.cu @@ -631,6 +631,76 @@ bool ggml_cuda_flash_attn_ext_prefill_supported(int cc, const ggml_tensor * dst) return (D == 64 || D == 128 || D == 256) && Q->ne[2] % K->ne[2] == 0 && fattn_vision_common_supported(cc, dst); } +#ifndef GGML_CUDA_FATTN_PREFILL_MASK_CACHE +#define GGML_CUDA_FATTN_PREFILL_MASK_CACHE 1 +#endif + +void ggml_cuda_flash_attn_ext_prefill_prepare(ggml_backend_cuda_context & ctx, const ggml_cgraph * cgraph) { + auto & cache = ctx.fattn_cls; + GGML_ASSERT(!cache.buf); + cache.mask = nullptr; + cache.inputs.clear(); + GGML_UNUSED(cgraph); +#if GGML_CUDA_FATTN_PREFILL_MASK_CACHE + static const bool enabled = []() { + const char * env = getenv("GGML_CUDA_FATTN_PREFILL_MASK_CACHE"); + return !env || atoi(env) != 0; + }(); + static const bool prefill = []() { + const char * env = getenv("GGML_CUDA_FATTN_PREFILL"); + return env && atoi(env) != 0; + }(); + const int cc = ggml_cuda_info().devices[ctx.device].cc; + if (!enabled || !prefill || !GGML_CUDA_CC_IS_RDNA3_5(cc) || ctx.curr_stream_no != 0) { + return; + } + size_t size = 0; + for (int i = 0; i < cgraph->n_nodes; ++i) { + const ggml_tensor * node = cgraph->nodes[i]; + if (node->op != GGML_OP_FLASH_ATTN_EXT || !ggml_cuda_flash_attn_ext_prefill_supported(cc, node)) { + continue; + } + const ggml_tensor * mask = node->src[3]; + if (!mask || mask->op != GGML_OP_NONE || mask->view_src) { + continue; + } + if (std::find(cache.inputs.begin(), cache.inputs.end(), mask) == cache.inputs.end()) { + bool written = false; + const uintptr_t base = (uintptr_t) mask->data; + for (int j = 0; j < cgraph->n_nodes; ++j) { + const ggml_tensor * other = cgraph->nodes[j]; + if (other->op == GGML_OP_NONE || other->op == GGML_OP_VIEW || other->op == GGML_OP_RESHAPE || + other->op == GGML_OP_PERMUTE || other->op == GGML_OP_TRANSPOSE) { + continue; + } + const uintptr_t addr = (uintptr_t) other->data; + written = written || (base <= addr ? addr - base < ggml_nbytes(mask) : base - addr < ggml_nbytes(other)); + } + if (written) { + continue; + } + cache.inputs.push_back(mask); + } + // D256 can use 64 query rows per tile; reserve for that smaller tile too. + const size_t nq = (node->src[0]->ne[1] + 63)/64; + const size_t nk = (node->src[1]->ne[1] + FS_BK - 1)/FS_BK; + const size_t nb = mask->ne[3]; + GGML_ASSERT(nk > 0 && nb > 0 && nq <= SIZE_MAX/nk && nq*nk <= SIZE_MAX/nb); + size = std::max(size, nq*nk*nb); + } + // Reserve before any node scratch so this allocation is released last. + if (size) { + cache.buf = std::make_unique>(ctx.pool(), size); + } +#endif +} + +void ggml_cuda_flash_attn_ext_prefill_release(ggml_backend_cuda_context & ctx) { + ctx.fattn_cls.buf.reset(); + ctx.fattn_cls.mask = nullptr; + ctx.fattn_cls.inputs.clear(); +} + static void fattn_vision_launch(ggml_backend_cuda_context & ctx, ggml_tensor * dst, const bool use_mask) { const ggml_tensor * Q = dst->src[0]; const ggml_tensor * K = dst->src[1]; @@ -658,8 +728,20 @@ static void fattn_vision_launch(ggml_backend_cuda_context & ctx, ggml_tensor * d ggml_cuda_pool_alloc cls_local(ctx.pool()); uint8_t * cls = nullptr; - if (mask) { - cls = cls_local.alloc(size_t(nbm)*nqb*ntk); + auto & cache = ctx.fattn_cls; + const int64_t key[7] = {nq, nkv, bq, bk, nbm, mask ? int64_t(mask->nb[1]) : 0, mask ? int64_t(mask->nb[3]) : 0}; + const bool reuse = mask && ctx.curr_stream_no == 0 && cache.buf && + std::find(cache.inputs.begin(), cache.inputs.end(), mask) != cache.inputs.end(); + if (reuse && cache.mask == mask && memcmp(cache.key, key, sizeof(key)) == 0) { + cls = cache.buf->get(); + } else if (mask) { + if (reuse) { + cache.mask = mask; + memcpy(cache.key, key, sizeof(key)); + cls = cache.buf->get(); + } else { + cls = cls_local.alloc(size_t(nbm)*nqb*ntk); + } if (split && split_ng == 8) { flash_attn_ext_prefill_classify<128, FS_BK><<>>( (const char *) mask->data, cls, nq, nkv, mask->nb[1], mask->nb[3]); diff --git a/ggml/src/ggml-cuda/fattn-vision.cuh b/ggml/src/ggml-cuda/fattn-vision.cuh index 4313f528e059..67b63cdd57df 100644 --- a/ggml/src/ggml-cuda/fattn-vision.cuh +++ b/ggml/src/ggml-cuda/fattn-vision.cuh @@ -12,3 +12,6 @@ bool ggml_cuda_flash_attn_ext_prefill_supported(int cc, const ggml_tensor * dst) void ggml_cuda_flash_attn_ext_prefill(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +void ggml_cuda_flash_attn_ext_prefill_prepare(ggml_backend_cuda_context & ctx, const ggml_cgraph * cgraph); +void ggml_cuda_flash_attn_ext_prefill_release(ggml_backend_cuda_context & ctx); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 929353e00ec9..17f96b5f3f35 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -30,6 +30,7 @@ #include "ggml-cuda/diagmask.cuh" #include "ggml-cuda/diag.cuh" #include "ggml-cuda/fattn.cuh" +#include "ggml-cuda/fattn-vision.cuh" #include "ggml-cuda/fwht.cuh" #include "ggml-cuda/getrows.cuh" #include "ggml-cuda/im2col.cuh" @@ -4932,6 +4933,7 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud // Only perform the graph execution if CUDA graphs are not enabled, or we are capturing the graph. // With the use of CUDA graphs, the execution will be performed by the graph launch. if (!use_cuda_graph || cuda_graph_update_required) { + ggml_cuda_flash_attn_ext_prefill_prepare(*cuda_ctx, cgraph); [[maybe_unused]] int prev_i = 0; if (stream_ctx.concurrent_events.size() > 0) { @@ -5141,6 +5143,8 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud } } + ggml_cuda_flash_attn_ext_prefill_release(*cuda_ctx); + #ifdef USE_CUDA_GRAPH if (use_cuda_graph && cuda_graph_update_required) { // End CUDA graph capture ggml_cuda_graph_launch_chunk(cuda_ctx, graph, n_chunks++); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 2a62234f20c6..0d5e788fd5c9 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -8845,6 +8845,26 @@ struct test_rdna_prefill_attention : public test_flash_attn_ext { } }; +struct test_rdna_prefill_mask_reuse : public test_rdna_prefill_attention { + const bool computed; + test_rdna_prefill_mask_reuse(int d, int queries, int batches, bool computed) + : test_rdna_prefill_attention(d, queries, 272, 4, batches, 3), computed(computed) {} + bool run_whole_graph() override { return true; } + std::string op_desc(ggml_tensor *) override { return "RDNA_PREFILL_MASK_REUSE"; } + std::string vars() override { return test_rdna_prefill_attention::vars() + "," + VAR_TO_STR(computed); } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * first = test_rdna_prefill_attention::build_graph(ctx); + ggml_tensor * mask = first->src[3]; + if (computed) { + first->src[3] = ggml_cont(ctx, mask); + } + ggml_tensor * second = ggml_flash_attn_ext(ctx, first->src[0], first->src[1], first->src[2], first->src[3], + 1.0f/sqrtf(hsk), 0.0f, 0.0f); + ggml_flash_attn_ext_set_prec(second, GGML_PREC_F32); + return ggml_add(ctx, first, second); + } +}; + // large Q values, so the online softmax has to rescale the partial results struct test_flash_attn_ext_large_logits : public test_flash_attn_ext { static constexpr int q_range = 20; @@ -10369,6 +10389,12 @@ static void add_rdna35_mmq_cases(std::vector> & test_ static std::vector> make_test_cases_eval() { std::vector> test_cases; + for (int d : {64, 128, 256}) { + for (bool computed : {false, true}) { + test_cases.emplace_back(new test_rdna_prefill_mask_reuse(d, 129, 1, computed)); + test_cases.emplace_back(new test_rdna_prefill_mask_reuse(d, 64, 2, computed)); + } + } for (int d : {64, 128, 256}) { for (int mask_kind : {0, 1, 2, 3, 4}) { test_cases.emplace_back(new test_rdna_prefill_attention(d, 129, 272, 4, 1, mask_kind)); From 130bf5c352e3410624c5d143847a74b994664b12 Mon Sep 17 00:00:00 2001 From: lichang Date: Tue, 17 Nov 2026 11:59:40 -0700 Subject: [PATCH 3/4] ci: avoid prebuilt UI downloads in behavior checks Disable UI provisioning in the Ubuntu and Windows behavior builds so a changing external asset bundle cannot prevent the C++ and Python tests from running. Assisted-by: Codex --- .github/workflows/behavior-check.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/behavior-check.yml b/.github/workflows/behavior-check.yml index 73c3b8fe223a..d57ac5601191 100644 --- a/.github/workflows/behavior-check.yml +++ b/.github/workflows/behavior-check.yml @@ -90,6 +90,8 @@ jobs: cmake -B build \ -DGGML_NATIVE=OFF \ -DGGML_BACKEND_DL=OFF \ + -DLLAMA_BUILD_UI=OFF \ + -DLLAMA_USE_PREBUILT_UI=OFF \ -DLLAMA_FATAL_WARNINGS=ON time cmake --build build --config Release -j $(nproc) @@ -149,6 +151,8 @@ jobs: -DCMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake ^ -DCMAKE_BUILD_TYPE=Release ^ -DGGML_NATIVE=OFF ^ + -DLLAMA_BUILD_UI=OFF ^ + -DLLAMA_USE_PREBUILT_UI=OFF ^ -DGGML_BACKEND_DL=OFF set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1 cmake --build build --config Release -j %NINJA_JOBS% From 921b4afe1a85e1978168b395b3962ef1387c5234 Mon Sep 17 00:00:00 2001 From: lichang Date: Tue, 17 Nov 2026 12:09:21 -0700 Subject: [PATCH 4/4] tests: use current attention precision API Replace the deprecated setter in the prefill mask reuse test with ggml_prec_set_acc. Both set the same FP32 accumulation parameter; the supported API also builds with fatal deprecation warnings. Assisted-by: Codex --- tests/test-backend-ops.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 29e5cfb05d67..90f7ec6b9213 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -8860,7 +8860,7 @@ struct test_rdna_prefill_mask_reuse : public test_rdna_prefill_attention { } ggml_tensor * second = ggml_flash_attn_ext(ctx, first->src[0], first->src[1], first->src[2], first->src[3], 1.0f/sqrtf(hsk), 0.0f, 0.0f); - ggml_flash_attn_ext_set_prec(second, GGML_PREC_F32); + ggml_prec_set_acc(second, GGML_PREC_F32); return ggml_add(ctx, first, second); } };