Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ggml/src/ggml-vulkan/ggml-vulkan-common.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ bool ggml_vk_intel_windows_driver_in_range(uint32_t driver_version, uint32_t low
// shaders
void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline);
vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc);
vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool aligned, bool f32acc, bool use_mask, bool use_mask_opt, bool use_logit_softcap, bool use_sparse, ggml_type k_type, ggml_type v_type);
vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool aligned, bool f32acc, bool use_mask, bool use_mask_opt, bool use_logit_softcap, bool use_sparse, bool gqa_multi_token, ggml_type k_type, ggml_type v_type);
uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_device_architecture &arch);
void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested = nullptr);
bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type);
Expand Down
1 change: 1 addition & 0 deletions ggml/src/ggml-vulkan/ggml-vulkan-types.h
Original file line number Diff line number Diff line change
Expand Up @@ -750,6 +750,7 @@ struct vk_device_struct {
bool subgroup_size_control;
uint32_t subgroup_min_size;
uint32_t subgroup_max_size;
uint32_t max_compute_workgroup_subgroups = 0;
bool subgroup_require_full_support;

// floor(log2(maxComputeWorkGroupInvocations))
Expand Down
125 changes: 106 additions & 19 deletions ggml/src/ggml-vulkan/ggml-vulkan.cpp

Large diffs are not rendered by default.

20 changes: 12 additions & 8 deletions ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp
Original file line number Diff line number Diff line change
Expand Up @@ -112,16 +112,16 @@ void main() {
[[unroll]] for (uint32_t idx = 0; idx < Br * HSK / 4; idx += gl_WorkGroupSize.x) {
uint32_t d = (idx + tid) % (HSK / 4);
uint32_t r = (idx + tid) / (HSK / 4);
const bool is_in_bounds = r < Br && d < HSK / 4 && i * Br + r < N;
const bool is_in_bounds = r < Br && d < HSK / 4 && i * Br + r < N && gqa_row_valid(i * Br + r);
#ifndef MMQ
if (is_in_bounds) {
Qf[r * qf_stride + d] = FLOAT_TYPEV4(data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale);
Qf[r * qf_stride + d] = FLOAT_TYPEV4(data_qv4[q_offset / 4 + q_row_offset(i * Br + r) / 4 + d] * p.scale);
}
#else
const uint buf_ib = r * qf_stride + d / 8;
const uint buf_iqs = d % 8;

vec4 vals = is_in_bounds ? data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale : vec4(0.0f);
vec4 vals = is_in_bounds ? data_qv4[q_offset / 4 + q_row_offset(i * Br + r) / 4 + d] * p.scale : vec4(0.0f);
const vec4 abs_vals = abs(vals);

const float thread_max = max(max(abs_vals.x, abs_vals.y), max(abs_vals.z, abs_vals.w));
Expand Down Expand Up @@ -187,6 +187,10 @@ void main() {
mo_offset += ((iq3 % p.nem3) * p.nem2 + (iq2 % p.nem2)) * CEIL_DIV(p.nem1, Br) * mo_stride;
}

if (USE_MASK_OPT && !USE_SPARSE && p.k_num > 1) {
mask_opt_balance_split_k(mo_offset, mo_stride);
}

uint32_t mask_opt = 0;
uint32_t mask_opt_idx = ~0;
uint32_t mask_opt_bits = 0;
Expand Down Expand Up @@ -216,7 +220,7 @@ void main() {
uint32_t kcol;
bool kv_active = fa_kv_index(j * Bc + c, kcol);
if (kv_active && (!nem1_bounds_check || i * Br + r < p.nem1)) {
FLOAT_TYPE m = FLOAT_TYPE(data_m[m_offset + (i * Br + r) * m_stride + kcol]);
FLOAT_TYPE m = FLOAT_TYPE(data_m[m_offset + mask_row_offset(i * Br + r) + kcol]);
masksh[c * masksh_stride + r] = m;
max_mask = max(max_mask, float(m));
} else {
Expand Down Expand Up @@ -672,7 +676,7 @@ void main() {
const uint row = tile_row(r);
if (row < N) {
[[unroll]] for (uint32_t d = 0; d < HSV_per_thread / 4; ++d) {
gqaStore(row, d * D_split + d_tid, Of[r][d], o_offset, iq2, N);
gqa_store_row(row, d * D_split + d_tid, Of[r][d], o_offset, HSV * p.ne1 * p.k_num / 4);
}
}
}
Expand All @@ -681,8 +685,8 @@ void main() {
[[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) {
const uint row = tile_row(r);
if (row < N) {
perElemOpStoreCol0(row, 0u, ACC_TYPE(Lf[r]), o_offset, iq2, N);
perElemOpStoreCol0(row, 0u, ACC_TYPE(Mf[r]), o_offset + p.ne1, iq2, N);
gqa_store_col0_row(row, ACC_TYPE(Lf[r]), o_offset, p.ne1 * 2 * p.k_num);
gqa_store_col0_row(row, ACC_TYPE(Mf[r]), o_offset + p.ne1, p.ne1 * 2 * p.k_num);
}
}
} else {
Expand Down Expand Up @@ -750,7 +754,7 @@ void main() {
const uint row = tile_row(r);
if (row < N) {
[[unroll]] for (uint32_t d = 0; d < HSV_per_thread / 4; ++d) {
gqaStore(row, d * D_split + d_tid, Of[r][d], o_offset, iq2, N);
gqa_store_row(row, d * D_split + d_tid, Of[r][d], o_offset, p.ne1 * HSV / 4);
}
}
}
Expand Down
117 changes: 111 additions & 6 deletions ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ const bool LOGIT_SOFTCAP = (Flags & 4) != 0;
const bool OLD_AMD_WINDOWS = (Flags & 8) != 0;
// Sparse: gather binding-7 indices instead of scanning [0,KV); p.split_kv = n_kv_max.
const bool USE_SPARSE = (Flags & 16) != 0;
// Rows are token * gqa_ratio + head; N / gqa_ratio query tokens share one K/V head per workgroup.
const bool GQA_MULTI_TOKEN = (Flags & 32) != 0;

// Round up head sizes to a multiple of 16, for coopmat1/coopmat2 paths
const uint32_t HSK_pad = (HSK + 15) & ~15;
Expand Down Expand Up @@ -148,7 +150,7 @@ ACC_TYPE perElemOpGetSink(const in uint32_t r, const in uint32_t c, const in ACC

uint32_t i, N, KV, split_k_index, Tr, start_j, end_j,
gqa_iq1, iq2, iq3, rk2, rk3, rv2, rv3, ik2, ik3, iv2, iv3,
q_stride, k_stride, v_stride, m_stride, sparse_base;
q_stride, k_stride, v_stride, m_stride, sparse_base, gqa_tokens, gqa_valid_tokens;

void init_indices()
{
Expand Down Expand Up @@ -176,6 +178,16 @@ void init_indices()
split_k_index = 0;
}

// gqa_iq1 is the first query token of the workgroup.
gqa_tokens = 1;
gqa_valid_tokens = 1;
if (GQA_MULTI_TOKEN) {
gqa_tokens = p.N / p.gqa_ratio;
gqa_iq1 *= gqa_tokens;
// the last token group may be partial (p.ne2 is the number of query tokens)
gqa_valid_tokens = min(gqa_tokens, p.ne2 - gqa_iq1);
}

Tr = CEIL_DIV(N, Br);

start_j = split_k_index * p.split_kv / Bc;
Expand Down Expand Up @@ -226,6 +238,57 @@ void init_indices()
}
}

// Select KV blocks that are not fully masked. Only code 1 (ALL_NEG_INF) has its low bit set.
uint32_t mask_opt_active_bits(const in uint32_t word, const in uint32_t w, const in uint32_t n_blocks)
{
const uint32_t n_valid = min(16u, n_blocks - w * 16);
const uint32_t valid = n_valid == 16 ? 0x55555555u : (((1u << (2 * n_valid)) - 1u) & 0x55555555u);
return valid & ~(word & 0x55555555u);
}

// Balance splits over visible KV blocks; splitting the full cache leaves most sliding-window splits idle.
void mask_opt_balance_split_k(const in uint32_t mo_offset, const in uint32_t mo_stride)
{
const uint32_t n_blocks = CEIL_DIV(KV, Bc);
uint32_t n_active = 0;
for (uint32_t w = 0; w < mo_stride; ++w) {
n_active += bitCount(mask_opt_active_bits(data_mask_opt[mo_offset + w], w, n_blocks));
}
const uint32_t per_split = CEIL_DIV(n_active, p.k_num);
const uint32_t first = split_k_index * per_split;
const uint32_t last = min(first + per_split, n_active);

start_j = 0;
end_j = 0;
if (first >= last) {
return;
}
uint32_t seen = 0;
for (uint32_t w = 0; w < mo_stride && seen < last; ++w) {
const uint32_t bits = mask_opt_active_bits(data_mask_opt[mo_offset + w], w, n_blocks);
const uint32_t count = bitCount(bits);
const bool has_first = seen <= first && first < seen + count;
const bool has_last = seen < last && last <= seen + count;
if (!has_first && !has_last) {
seen += count;
continue;
}
for (uint32_t b = 0; b < 16; ++b) {
if ((bits & (1u << (2 * b))) == 0) {
continue;
}
if (seen == first) {
start_j = w * 16 + b;
}
seen++;
if (seen == last) {
end_j = w * 16 + b + 1;
break;
}
}
}
}

// Resolve a linear KV slot to a real column; false for inactive (sparse padding/-1, or dense OOB).
bool fa_kv_index(uint lin, out uint kv_col) {
if (USE_SPARSE) {
Expand All @@ -245,10 +308,52 @@ bool fa_kv_index(uint lin, out uint kv_col) {
// Based on ggml-cuda issue https://github.com/ggml-org/llama.cpp/issues/18606
const float FATTN_KQ_MAX_OFFSET = 3.0f*0.6931f;

// Store the output when doing grouped query attention.
// Rows index by Q's dimension 2, and the first N rows are valid.
void gqaStore(const in uint32_t r, const in uint32_t c, const in O_TYPEV4 elems, const in uint32_t o_offset, const in uint32_t iq2, const in uint32_t N)
// Query token (relative to gqa_iq1) and head (relative to iq2) of a GQA row.
uint32_t gqa_row_token(const in uint32_t r)
{
return GQA_MULTI_TOKEN ? r / p.gqa_ratio : 0;
}

uint32_t gqa_row_head(const in uint32_t r)
{
return GQA_MULTI_TOKEN ? r % p.gqa_ratio : r;
}

// False for rows of a partial last token group that have no query token.
bool gqa_row_valid(const in uint32_t r)
{
return !GQA_MULTI_TOKEN || gqa_row_token(r) < gqa_valid_tokens;
}

// Offset of a Q row from q_offset, in floats.
uint32_t q_row_offset(const in uint32_t r)
{
uint32_t offset = (iq2 + r) * HSV / 4 + c;
data_ov4[o_offset + offset] = D_TYPEV4(elems);
return gqa_row_token(r) * p.nb01 + gqa_row_head(r) * q_stride;
}

// Offset of the mask row used by Q row r, from m_offset. Padding rows reuse the last token.
uint32_t mask_row_offset(const in uint32_t r)
{
if (GQA_MULTI_TOKEN) {
return min(gqa_row_token(r), gqa_valid_tokens - 1) * KV;
}
return r * m_stride;
}

// token_stride is the query-token stride in vec4 units for O, or elements for split_k L/M.
void gqa_store_row(const in uint32_t r, const in uint32_t c, const in O_TYPEV4 elems, const in uint32_t o_offset, const in uint32_t token_stride)
{
if (!gqa_row_valid(r)) {
return;
}
const uint32_t offset = (iq2 + gqa_row_head(r)) * HSV / 4 + c;
data_ov4[o_offset + gqa_row_token(r) * token_stride + offset] = D_TYPEV4(elems);
}

void gqa_store_col0_row(const in uint32_t r, const in ACC_TYPE elem, const in uint32_t o_offset, const in uint32_t token_stride)
{
if (!gqa_row_valid(r)) {
return;
}
data_o[o_offset + gqa_row_token(r) * token_stride + iq2 + gqa_row_head(r)] = D_TYPE(elem);
}
Loading
Loading