diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-common.h b/ggml/src/ggml-vulkan/ggml-vulkan-common.h index 4e05fb81839b..75185d2b4b1e 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-common.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-common.h @@ -35,7 +35,7 @@ bool ggml_vk_intel_windows_driver_in_range(uint32_t driver_version, uint32_t low // shaders void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline); vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc); -vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool aligned, bool f32acc, bool use_mask, bool use_mask_opt, bool use_logit_softcap, bool use_sparse, ggml_type k_type, ggml_type v_type); +vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool aligned, bool f32acc, bool use_mask, bool use_mask_opt, bool use_logit_softcap, bool use_sparse, bool gqa_multi_token, ggml_type k_type, ggml_type v_type); uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_device_architecture &arch); void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested = nullptr); bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type); diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h index 81fd5b8a84ba..e47c5dab6405 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h @@ -750,6 +750,7 @@ struct vk_device_struct { bool subgroup_size_control; uint32_t subgroup_min_size; uint32_t subgroup_max_size; + uint32_t max_compute_workgroup_subgroups = 0; bool subgroup_require_full_support; // floor(log2(maxComputeWorkGroupInvocations)) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 22641f5fbdb4..51798f6e9099 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1391,7 +1391,8 @@ vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_t hsk, } vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool aligned, bool f32acc, - bool use_mask, bool use_mask_opt, bool use_logit_softcap, bool use_sparse, ggml_type k_type, ggml_type v_type) { + bool use_mask, bool use_mask_opt, bool use_logit_softcap, bool use_sparse, bool gqa_multi_token, + ggml_type k_type, ggml_type v_type) { const bool old_amd_windows = device->vendor_id == VK_VENDOR_ID_AMD && device->driver_id == vk::DriverId::eAmdProprietary && (device->architecture == AMD_GCN || device->architecture == AMD_RDNA1 || device->architecture == AMD_RDNA2); @@ -1399,7 +1400,8 @@ vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const vk_fa_ (use_mask ? 2 : 0) | (use_logit_softcap ? 4 : 0) | (old_amd_windows ? 8 : 0) | - (use_sparse ? 16 : 0); + (use_sparse ? 16 : 0) | + (gqa_multi_token ? 32 : 0); const uint32_t subgroup_size = params.disable_subgroups ? 0 : params.subgroup_size; @@ -1877,6 +1879,24 @@ static bool ggml_vk_amd_large_tile_supported(const vk_device& device) { return major > 2 || (major == 2 && build >= 416); } +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) +// These driver-specific schedules were measured on Strix Halo only. +static bool ggml_vk_is_strix_halo(const vk_device& device) { + return device->vendor_id == VK_VENDOR_ID_AMD && device->properties.deviceID == 0x1586 && + device->uma && device->architecture == AMD_RDNA3; +} + +static bool ggml_vk_is_strix_halo_radv(const vk_device& device) { + return ggml_vk_is_strix_halo(device) && device->driver_id == vk::DriverId::eMesaRadv; +} + +// AMD's proprietary compiler uses more MMVQ registers at wave64; prefer its minimum subgroup size. +static bool ggml_vk_use_strix_halo_amd_mmvq(const vk_device& device) { + return ggml_vk_is_strix_halo(device) && device->driver_id == vk::DriverId::eAmdProprietary && + device->subgroup_size_control && device->subgroup_min_size <= 32; +} +#endif // GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT + void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { VK_LOG_DEBUG("ggml_vk_load_shaders(" << device->name << ")"); @@ -3113,6 +3133,11 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { static constexpr uint32_t mul_mat_vec_num_bindings = 5; static constexpr uint32_t mul_mat_vec_id_num_bindings = 6; +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + const bool amd_mmvq = ggml_vk_use_strix_halo_amd_mmvq(device); + const uint32_t subgroup_size_int = ((device->vendor_id == VK_VENDOR_ID_INTEL && device->subgroup_size_control) || amd_mmvq) ? device->subgroup_min_size : device->subgroup_size; +#endif + #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) #define OCP_DMMV_LEN(NAME, REDUC) (device->ocp_fp4 ? NAME ## _ocp_len[REDUC] : NAME ## _len[REDUC]) #define OCP_DMMV_DATA(NAME, REDUC) (device->ocp_fp4 ? NAME ## _ocp_data[REDUC] : NAME ## _data[REDUC]) @@ -3124,6 +3149,9 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { for (uint32_t w = 0; w < DMMV_WG_SIZE_COUNT; ++w) { const uint32_t wg_size_subgroup = (w == DMMV_WG_SIZE_SUBGROUP) ? subgroup_size : (subgroup_size * 4); const uint32_t wg_size_subgroup16 = (w == DMMV_WG_SIZE_SUBGROUP) ? subgroup_size16 : (subgroup_size16 * 4); +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + const uint32_t wg_size_subgroup_int = (w == DMMV_WG_SIZE_SUBGROUP) ? subgroup_size_int : (subgroup_size_int * 4); +#endif const shader_reduction_mode reduc = (use_subgroups && w == DMMV_WG_SIZE_SUBGROUP) ? SHADER_REDUCTION_MODE_SUBGROUP : (use_subgroups && w == DMMV_WG_SIZE_LARGE) ? SHADER_REDUCTION_MODE_HYBRID : @@ -3194,9 +3222,6 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) if (device->integer_dot_product) { - const uint32_t subgroup_size_int = (device->vendor_id == VK_VENDOR_ID_INTEL && device->subgroup_size_control) ? device->subgroup_min_size : device->subgroup_size; - const uint32_t wg_size_subgroup_int = (w == DMMV_WG_SIZE_SUBGROUP) ? subgroup_size_int : (subgroup_size_int * 4); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_q8_1_f32", arr_dmmv_q2_0_q8_1_f32_len[reduc], arr_dmmv_q2_0_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(2*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(2*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_q8_1_f32", arr_dmmv_q4_0_q8_1_f32_len[reduc], arr_dmmv_q4_0_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_stdq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_stdq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_q8_1_f32", arr_dmmv_q4_1_q8_1_f32_len[reduc], arr_dmmv_q4_1_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_stdq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_stdq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); @@ -3206,9 +3231,13 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_MXFP4][i], "mul_mat_vec_mxfp4_q8_1_f32", arr_dmmv_mxfp4_q8_1_f32_len[reduc], arr_dmmv_mxfp4_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(2*rm_stdq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(2*rm_stdq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_q8_1_f32", arr_dmmv_q2_k_q8_1_f32_len[reduc], arr_dmmv_q2_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(2*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(2*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_q8_1_f32", arr_dmmv_q3_k_q8_1_f32_len[reduc], arr_dmmv_q3_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_q8_1_f32", arr_dmmv_q4_k_q8_1_f32_len[reduc], arr_dmmv_q4_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); + // i + 1 is the token count; at eight tokens these Strix Halo paths were faster with two rows. + const uint32_t q2_rows = amd_mmvq && i == 7 ? 2u : rm_int_n(2*rm_kq_int, i); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_q8_1_f32", arr_dmmv_q2_k_q8_1_f32_len[reduc], arr_dmmv_q2_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {q2_rows, 1, 1}, {wg_size_subgroup_int, q2_rows, i+1}, 1, true, use_subgroups, subgroup_size_int); + const uint32_t q3_rows = (ggml_vk_is_strix_halo_radv(device) || amd_mmvq) && i == 7 ? 2u : rm_int_n(rm_kq_int, i); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_q8_1_f32", arr_dmmv_q3_k_q8_1_f32_len[reduc], arr_dmmv_q3_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {q3_rows, 1, 1}, {wg_size_subgroup_int, q3_rows, i+1}, 1, true, use_subgroups, subgroup_size_int); + const uint32_t q4_rows = amd_mmvq && i == 7 ? 2u : rm_int_n(1*rm_kq_int, i); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_q8_1_f32", arr_dmmv_q4_k_q8_1_f32_len[reduc], arr_dmmv_q4_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {q4_rows, 1, 1}, {wg_size_subgroup_int, q4_rows, i+1}, 1, true, use_subgroups, subgroup_size_int); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_q8_1_f32", arr_dmmv_q5_k_q8_1_f32_len[reduc], arr_dmmv_q5_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_q8_1_f32", arr_dmmv_q6_k_q8_1_f32_len[reduc], arr_dmmv_q6_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); @@ -3251,9 +3280,6 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) if (device->integer_dot_product) { - const uint32_t subgroup_size_int = (device->vendor_id == VK_VENDOR_ID_INTEL && device->subgroup_size_control) ? device->subgroup_min_size : device->subgroup_size; - const uint32_t wg_size_subgroup_int = (w == DMMV_WG_SIZE_SUBGROUP) ? subgroup_size_int : (subgroup_size_int * 4); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q2_0], "mul_mat_vec_id_q2_0_q8_1_f32", arr_dmmv_id_q2_0_q8_1_f32_len[reduc], arr_dmmv_id_q2_0_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(2*rm_kq_int), 1, 1}, {wg_size_subgroup_int, rm_id(2*rm_kq_int)}, 1, true, use_subgroups, subgroup_size_int); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q4_0], "mul_mat_vec_id_q4_0_q8_1_f32", arr_dmmv_id_q4_0_q8_1_f32_len[reduc], arr_dmmv_id_q4_0_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_stdq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_stdq_int)}, 1, true, use_subgroups, subgroup_size_int); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q4_1], "mul_mat_vec_id_q4_1_q8_1_f32", arr_dmmv_id_q4_1_q8_1_f32_len[reduc], arr_dmmv_id_q4_1_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_stdq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_stdq_int)}, 1, true, use_subgroups, subgroup_size_int); @@ -4798,6 +4824,7 @@ vk_device ggml_vk_get_device(size_t idx) { if (device->subgroup_size_control) { device->subgroup_min_size = subgroup_size_control_props.minSubgroupSize; device->subgroup_max_size = subgroup_size_control_props.maxSubgroupSize; + device->max_compute_workgroup_subgroups = subgroup_size_control_props.maxComputeWorkgroupSubgroups; device_extensions.push_back("VK_EXT_subgroup_size_control"); } @@ -8369,7 +8396,21 @@ bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_ const uint32_t MatBr = 16, MatBc = 16; - const uint32_t row_split = Bc / MatBc; + // Cooperative tiles and subgroup reductions must use the same row/column partition. + if (params.path != FA_COOPMAT1 || Br < MatBr || Br % MatBr != 0 || + params.row_split == 0 || params.subgroup_size == 0) { + return false; + } + const uint32_t query_tiles = Br / MatBr; + if (params.row_split % query_tiles != 0) { + return false; + } + const uint32_t column_subgroups = params.row_split / query_tiles; + if (Bc != MatBc * column_subgroups || Br % params.row_split != 0 || (Br / params.row_split) % 4 != 0 || + params.workgroup_size != params.subgroup_size * params.row_split || + Bc < params.subgroup_size || Bc % params.subgroup_size != 0) { + return false; + } const uint32_t hsk_pad = ROUNDUP_POW2(hsk, 16); const uint32_t hsv_pad = ROUNDUP_POW2(hsv, 16); @@ -8377,7 +8418,7 @@ bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_ const uint32_t acctype = f32acc ? 4 : 2; const uint32_t f16vec4 = 8; - const uint32_t tmpsh = (Bc / MatBc) * sizeof(float); + const uint32_t tmpsh = params.row_split * sizeof(float); // DATA_A_IQ4_NL is compiled into the FA shaders unconditionally, so its shared table is always allocated. const uint32_t iq_shmem = 16 * sizeof(ggml_fp16_t); @@ -8391,13 +8432,13 @@ bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_ const uint32_t sfsh = Bc * sfshstride * acctype; const uint32_t kvshstride = (params.shmem_staging ? std::max(hsk_pad, hsv_pad) : MatBr) / 4 + 2; - const uint32_t vsh_stride = MatBc / 4 * row_split; + const uint32_t vsh_stride = MatBc / 4 * column_subgroups; const uint32_t ksh = ((kvshstride >= vsh_stride) ? (Bc * kvshstride) : (Bc * vsh_stride)) * f16vec4; // BF16 PVMat accumulator is f32 (no bf16 accumulator support), so pvsh is vec4 (16 bytes) const uint32_t pvsh_elem_size = (k_type == GGML_TYPE_BF16) ? 16u : f16vec4; - const uint32_t osh_stride = params.row_split * MatBr / 4; - const uint32_t pvsh = MatBc * osh_stride * pvsh_elem_size; + const uint32_t osh_stride = column_subgroups * MatBr / 4; + const uint32_t pvsh = Br * osh_stride * pvsh_elem_size; const uint32_t slope = Br * acctype; @@ -8526,6 +8567,48 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const tuning_params = get_fa_tuning_params(ctx->device, HSK, HSV, N, KV, k_type_eff, v_type_eff, f32acc); + // Fold query tokens into GQA rows to share K/V loads. Sparse lists remain per query token. + uint32_t gqa_tokens = 1; + // Four tokens at GQA > 4 need more than 16 rows. Wider tiles are tuned for 8K+ KV. + const bool use_gqa_query_tiles = gqa_ratio > 4 && f32acc && + ctx->device->vendor_id == VK_VENDOR_ID_AMD && + ctx->device->subgroup_size_control && + ctx->device->subgroup_min_size <= 32 && ctx->device->subgroup_max_size >= 32 && + tuning_params.path == FA_COOPMAT1 && HSK == 128 && HSV == 128 && KV >= 8192 && + k_type_eff == GGML_TYPE_F16 && v_type_eff == GGML_TYPE_F16; + const int32_t n_kv_max = mask ? ggml_get_op_params_i32(dst, 4) : 0; + if (gqa_ratio > 1 && neq1 > 1 && n_kv_max == 0 && + tuning_params.path != FA_COOPMAT2 && ctx->device->vendor_id != VK_VENDOR_ID_INTEL) { + const uint32_t token_limit = use_gqa_query_tiles ? 4 : max_gqa / gqa_ratio; + for (uint32_t tokens = std::min((uint32_t)neq1, token_limit); tokens > 1; --tokens) { + vk_fa_tuning_params folded_params = + get_fa_tuning_params(ctx->device, HSK, HSV, tokens * gqa_ratio, KV, k_type_eff, v_type_eff, f32acc); + if (use_gqa_query_tiles && tokens * gqa_ratio > 16) { + folded_params.block_rows = ROUNDUP_POW2(tokens * gqa_ratio, 16); + folded_params.subgroup_size = 32; + folded_params.block_cols = 32; + folded_params.row_split = 2 * (folded_params.block_rows / 16); + folded_params.workgroup_size = folded_params.row_split * folded_params.subgroup_size; + folded_params.shmem_staging = 1; + if (folded_params.workgroup_size > ctx->device->properties.limits.maxComputeWorkGroupInvocations || + folded_params.workgroup_size > ctx->device->properties.limits.maxComputeWorkGroupSize[0] || + (ctx->device->subgroup_size_control && folded_params.row_split > ctx->device->max_compute_workgroup_subgroups) || + !ggml_vk_flash_attn_coopmat_shmem_support(ctx->device, folded_params, HSK, HSV, f32acc, k_type_eff, v_type_eff)) { + continue; + } + } + if (folded_params.path == FA_COOPMAT2 || folded_params.block_rows < tokens * gqa_ratio) { + continue; + } + gqa_tokens = tokens; + N = tokens * gqa_ratio; + // the last group may be partial; its missing tokens are masked out in the shader + workgroups_x = CEIL_DIV((uint32_t)neq1, tokens); + tuning_params = folded_params; + break; + } + } + float scale = 1.0f; float max_bias = 0.0f; float logit_softcap = 0.0f; @@ -8539,7 +8622,6 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const } // Sparse mask hint (op_params[4]): compact the <= n_kv_max finite positions and gather only those. - const int32_t n_kv_max = mask ? ggml_get_op_params_i32(dst, 4) : 0; static const bool disable_sparse = getenv("GGML_VK_FA_SPARSE_DISABLE") != nullptr; const bool kv_f16 = k_type_eff == GGML_TYPE_F16 && v_type_eff == GGML_TYPE_F16; // cm2 dense is fast, so it needs a larger reduction to win. @@ -8595,10 +8677,15 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const } // Only use mask opt when the mask is fairly large. This hasn't been tuned extensively. - bool use_mask_opt = mask && !use_sparse && nem1 >= 32 && nem0 * nem1 > 32768 && nem0 >= tuning_params.block_cols * 16 + const bool mask_opt_large_batch = nem1 >= 32 && nem0 * nem1 > 32768; + // Limit the extra pre-pass to long caches (8K+ KV). GQA mask rows must fit in the first tile. + const bool mask_opt_long_kv = nem0 >= 8192 && + (gqa_ratio > 1 ? nem1 <= tuning_params.block_rows : nem1 < 32); + bool use_mask_opt = mask && !use_sparse && (mask_opt_large_batch || mask_opt_long_kv) && nem0 >= tuning_params.block_cols * 16 && (ctx->device->architecture != vk_device_architecture::AMD_GCN || HSK > 256 || HSV > 256); vk_fa_pipeline_state fa_pipeline_state = get_fa_pipeline_state(ctx->device, tuning_params, HSK, HSV, aligned, f32acc, - mask != nullptr, use_mask_opt, logit_softcap != 0, use_sparse, k_type_eff, v_type_eff); + mask != nullptr, use_mask_opt, logit_softcap != 0, use_sparse, gqa_tokens > 1, + k_type_eff, v_type_eff); vk_pipeline pipeline = nullptr; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp index 2c5e85908576..58e7a4fcc9d3 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp @@ -112,16 +112,16 @@ void main() { [[unroll]] for (uint32_t idx = 0; idx < Br * HSK / 4; idx += gl_WorkGroupSize.x) { uint32_t d = (idx + tid) % (HSK / 4); uint32_t r = (idx + tid) / (HSK / 4); - const bool is_in_bounds = r < Br && d < HSK / 4 && i * Br + r < N; + const bool is_in_bounds = r < Br && d < HSK / 4 && i * Br + r < N && gqa_row_valid(i * Br + r); #ifndef MMQ if (is_in_bounds) { - Qf[r * qf_stride + d] = FLOAT_TYPEV4(data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale); + Qf[r * qf_stride + d] = FLOAT_TYPEV4(data_qv4[q_offset / 4 + q_row_offset(i * Br + r) / 4 + d] * p.scale); } #else const uint buf_ib = r * qf_stride + d / 8; const uint buf_iqs = d % 8; - vec4 vals = is_in_bounds ? data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale : vec4(0.0f); + vec4 vals = is_in_bounds ? data_qv4[q_offset / 4 + q_row_offset(i * Br + r) / 4 + d] * p.scale : vec4(0.0f); const vec4 abs_vals = abs(vals); const float thread_max = max(max(abs_vals.x, abs_vals.y), max(abs_vals.z, abs_vals.w)); @@ -187,6 +187,10 @@ void main() { mo_offset += ((iq3 % p.nem3) * p.nem2 + (iq2 % p.nem2)) * CEIL_DIV(p.nem1, Br) * mo_stride; } + if (USE_MASK_OPT && !USE_SPARSE && p.k_num > 1) { + mask_opt_balance_split_k(mo_offset, mo_stride); + } + uint32_t mask_opt = 0; uint32_t mask_opt_idx = ~0; uint32_t mask_opt_bits = 0; @@ -216,7 +220,7 @@ void main() { uint32_t kcol; bool kv_active = fa_kv_index(j * Bc + c, kcol); if (kv_active && (!nem1_bounds_check || i * Br + r < p.nem1)) { - FLOAT_TYPE m = FLOAT_TYPE(data_m[m_offset + (i * Br + r) * m_stride + kcol]); + FLOAT_TYPE m = FLOAT_TYPE(data_m[m_offset + mask_row_offset(i * Br + r) + kcol]); masksh[c * masksh_stride + r] = m; max_mask = max(max_mask, float(m)); } else { @@ -672,7 +676,7 @@ void main() { const uint row = tile_row(r); if (row < N) { [[unroll]] for (uint32_t d = 0; d < HSV_per_thread / 4; ++d) { - gqaStore(row, d * D_split + d_tid, Of[r][d], o_offset, iq2, N); + gqa_store_row(row, d * D_split + d_tid, Of[r][d], o_offset, HSV * p.ne1 * p.k_num / 4); } } } @@ -681,8 +685,8 @@ void main() { [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) { const uint row = tile_row(r); if (row < N) { - perElemOpStoreCol0(row, 0u, ACC_TYPE(Lf[r]), o_offset, iq2, N); - perElemOpStoreCol0(row, 0u, ACC_TYPE(Mf[r]), o_offset + p.ne1, iq2, N); + gqa_store_col0_row(row, ACC_TYPE(Lf[r]), o_offset, p.ne1 * 2 * p.k_num); + gqa_store_col0_row(row, ACC_TYPE(Mf[r]), o_offset + p.ne1, p.ne1 * 2 * p.k_num); } } } else { @@ -750,7 +754,7 @@ void main() { const uint row = tile_row(r); if (row < N) { [[unroll]] for (uint32_t d = 0; d < HSV_per_thread / 4; ++d) { - gqaStore(row, d * D_split + d_tid, Of[r][d], o_offset, iq2, N); + gqa_store_row(row, d * D_split + d_tid, Of[r][d], o_offset, p.ne1 * HSV / 4); } } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl index 2e0e23bc11c1..c6e486abd918 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl @@ -26,6 +26,8 @@ const bool LOGIT_SOFTCAP = (Flags & 4) != 0; const bool OLD_AMD_WINDOWS = (Flags & 8) != 0; // Sparse: gather binding-7 indices instead of scanning [0,KV); p.split_kv = n_kv_max. const bool USE_SPARSE = (Flags & 16) != 0; +// Rows are token * gqa_ratio + head; N / gqa_ratio query tokens share one K/V head per workgroup. +const bool GQA_MULTI_TOKEN = (Flags & 32) != 0; // Round up head sizes to a multiple of 16, for coopmat1/coopmat2 paths const uint32_t HSK_pad = (HSK + 15) & ~15; @@ -148,7 +150,7 @@ ACC_TYPE perElemOpGetSink(const in uint32_t r, const in uint32_t c, const in ACC uint32_t i, N, KV, split_k_index, Tr, start_j, end_j, gqa_iq1, iq2, iq3, rk2, rk3, rv2, rv3, ik2, ik3, iv2, iv3, - q_stride, k_stride, v_stride, m_stride, sparse_base; + q_stride, k_stride, v_stride, m_stride, sparse_base, gqa_tokens, gqa_valid_tokens; void init_indices() { @@ -176,6 +178,16 @@ void init_indices() split_k_index = 0; } + // gqa_iq1 is the first query token of the workgroup. + gqa_tokens = 1; + gqa_valid_tokens = 1; + if (GQA_MULTI_TOKEN) { + gqa_tokens = p.N / p.gqa_ratio; + gqa_iq1 *= gqa_tokens; + // the last token group may be partial (p.ne2 is the number of query tokens) + gqa_valid_tokens = min(gqa_tokens, p.ne2 - gqa_iq1); + } + Tr = CEIL_DIV(N, Br); start_j = split_k_index * p.split_kv / Bc; @@ -226,6 +238,57 @@ void init_indices() } } +// Select KV blocks that are not fully masked. Only code 1 (ALL_NEG_INF) has its low bit set. +uint32_t mask_opt_active_bits(const in uint32_t word, const in uint32_t w, const in uint32_t n_blocks) +{ + const uint32_t n_valid = min(16u, n_blocks - w * 16); + const uint32_t valid = n_valid == 16 ? 0x55555555u : (((1u << (2 * n_valid)) - 1u) & 0x55555555u); + return valid & ~(word & 0x55555555u); +} + +// Balance splits over visible KV blocks; splitting the full cache leaves most sliding-window splits idle. +void mask_opt_balance_split_k(const in uint32_t mo_offset, const in uint32_t mo_stride) +{ + const uint32_t n_blocks = CEIL_DIV(KV, Bc); + uint32_t n_active = 0; + for (uint32_t w = 0; w < mo_stride; ++w) { + n_active += bitCount(mask_opt_active_bits(data_mask_opt[mo_offset + w], w, n_blocks)); + } + const uint32_t per_split = CEIL_DIV(n_active, p.k_num); + const uint32_t first = split_k_index * per_split; + const uint32_t last = min(first + per_split, n_active); + + start_j = 0; + end_j = 0; + if (first >= last) { + return; + } + uint32_t seen = 0; + for (uint32_t w = 0; w < mo_stride && seen < last; ++w) { + const uint32_t bits = mask_opt_active_bits(data_mask_opt[mo_offset + w], w, n_blocks); + const uint32_t count = bitCount(bits); + const bool has_first = seen <= first && first < seen + count; + const bool has_last = seen < last && last <= seen + count; + if (!has_first && !has_last) { + seen += count; + continue; + } + for (uint32_t b = 0; b < 16; ++b) { + if ((bits & (1u << (2 * b))) == 0) { + continue; + } + if (seen == first) { + start_j = w * 16 + b; + } + seen++; + if (seen == last) { + end_j = w * 16 + b + 1; + break; + } + } + } +} + // Resolve a linear KV slot to a real column; false for inactive (sparse padding/-1, or dense OOB). bool fa_kv_index(uint lin, out uint kv_col) { if (USE_SPARSE) { @@ -245,10 +308,52 @@ bool fa_kv_index(uint lin, out uint kv_col) { // Based on ggml-cuda issue https://github.com/ggml-org/llama.cpp/issues/18606 const float FATTN_KQ_MAX_OFFSET = 3.0f*0.6931f; -// Store the output when doing grouped query attention. -// Rows index by Q's dimension 2, and the first N rows are valid. -void gqaStore(const in uint32_t r, const in uint32_t c, const in O_TYPEV4 elems, const in uint32_t o_offset, const in uint32_t iq2, const in uint32_t N) +// Query token (relative to gqa_iq1) and head (relative to iq2) of a GQA row. +uint32_t gqa_row_token(const in uint32_t r) +{ + return GQA_MULTI_TOKEN ? r / p.gqa_ratio : 0; +} + +uint32_t gqa_row_head(const in uint32_t r) +{ + return GQA_MULTI_TOKEN ? r % p.gqa_ratio : r; +} + +// False for rows of a partial last token group that have no query token. +bool gqa_row_valid(const in uint32_t r) +{ + return !GQA_MULTI_TOKEN || gqa_row_token(r) < gqa_valid_tokens; +} + +// Offset of a Q row from q_offset, in floats. +uint32_t q_row_offset(const in uint32_t r) { - uint32_t offset = (iq2 + r) * HSV / 4 + c; - data_ov4[o_offset + offset] = D_TYPEV4(elems); + return gqa_row_token(r) * p.nb01 + gqa_row_head(r) * q_stride; +} + +// Offset of the mask row used by Q row r, from m_offset. Padding rows reuse the last token. +uint32_t mask_row_offset(const in uint32_t r) +{ + if (GQA_MULTI_TOKEN) { + return min(gqa_row_token(r), gqa_valid_tokens - 1) * KV; + } + return r * m_stride; +} + +// token_stride is the query-token stride in vec4 units for O, or elements for split_k L/M. +void gqa_store_row(const in uint32_t r, const in uint32_t c, const in O_TYPEV4 elems, const in uint32_t o_offset, const in uint32_t token_stride) +{ + if (!gqa_row_valid(r)) { + return; + } + const uint32_t offset = (iq2 + gqa_row_head(r)) * HSV / 4 + c; + data_ov4[o_offset + gqa_row_token(r) * token_stride + offset] = D_TYPEV4(elems); +} + +void gqa_store_col0_row(const in uint32_t r, const in ACC_TYPE elem, const in uint32_t o_offset, const in uint32_t token_stride) +{ + if (!gqa_row_valid(r)) { + return; + } + data_o[o_offset + gqa_row_token(r) * token_stride + iq2 + gqa_row_head(r)] = D_TYPE(elem); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp index aa9dd624bef3..0a5be1cccf74 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp @@ -25,6 +25,8 @@ // These need to be supported N,M values for a MatBc x MatBr x 16 coopmatmuladd const uint32_t MatBr = 16; const uint32_t MatBc = 16; +const uint32_t query_tiles = Br < MatBr ? 1 : Br / MatBr; +const uint32_t column_subgroups = row_split / query_tiles; const uint32_t rows_per_thread = Br / row_split; const uint32_t cols_per_iter = gl_WorkGroupSize.x / row_split; @@ -53,12 +55,12 @@ shared ACC_TYPEV4 sfsh[Bc * sfshstride]; const uint32_t D_pad = HSK_pad > HSV_pad ? HSK_pad : HSV_pad; const uint32_t kvsh_stride = (SHMEM_STAGING != 0 ? D_pad : MatBr) / 4 + 2; -const uint v_cols = MatBc / 4 * row_split; // total cols, 4 vec4s per MatBc * number of subgroups +const uint v_cols = MatBc / 4 * column_subgroups; const uint vsh_stride = v_cols; shared FLOAT_TYPEV4 kvsh[(kvsh_stride >= vsh_stride) ? (Bc * kvsh_stride) : (Bc * vsh_stride)]; -const uint32_t osh_stride = row_split * MatBr / 4; -shared O_TYPEV4 pvsh[MatBc * osh_stride]; +const uint32_t osh_stride = column_subgroups * MatBr / 4; +shared O_TYPEV4 pvsh[Br * osh_stride]; shared ACC_TYPE slope[Br]; @@ -72,8 +74,10 @@ void main() { init_indices(); const uint32_t tid = gl_LocalInvocationIndex; + const uint32_t query_tile = gl_SubgroupID / column_subgroups; + const uint32_t column_subgroup = gl_SubgroupID % column_subgroups; - const uint32_t threads_per_rowgroup = gl_WorkGroupSize.x / row_split; + const uint32_t threads_per_rowgroup = cols_per_iter; const uint32_t d_per_thread = (HSV/4 + threads_per_rowgroup - 1) / threads_per_rowgroup; const uint32_t row_tid = gl_LocalInvocationIndex / threads_per_rowgroup; const uint32_t col_tid = gl_LocalInvocationIndex % threads_per_rowgroup; @@ -95,9 +99,9 @@ void main() { [[unroll]] for (uint32_t idx = 0; idx < Br * HSK / 4; idx += gl_WorkGroupSize.x) { uint32_t d = (idx + tid) % (HSK / 4); uint32_t r = (idx + tid) / (HSK / 4); - if (r < Br && d < HSK / 4 && - i * Br + r < N) { - Qf[r * qstride + d] = FLOAT_TYPEV4(data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale); + if (r < Br && d < HSK / 4) { + Qf[r * qstride + d] = i * Br + r < N && gqa_row_valid(i * Br + r) ? + FLOAT_TYPEV4(data_qv4[q_offset / 4 + q_row_offset(i * Br + r) / 4 + d] * p.scale) : FLOAT_TYPEV4(0); } } barrier(); @@ -145,6 +149,10 @@ void main() { mo_offset += ((iq3 % p.nem3) * p.nem2 + (iq2 % p.nem2)) * CEIL_DIV(p.nem1, Br) * mo_stride; } + if (USE_MASK_OPT && !USE_SPARSE && p.k_num > 1) { + mask_opt_balance_split_k(mo_offset, mo_stride); + } + uint32_t mask_opt = 0; uint32_t mask_opt_idx = ~0; uint32_t mask_opt_bits = 0; @@ -186,25 +194,25 @@ void main() { m = f16vec4(mv); max_mask = max(max_mask, float(mv)); } else if (!nem1_bounds_check || i * Br + r * 4 + 3 < p.nem1) { - m = f16vec4(data_m[m_offset + (i * Br + r * 4 ) * m_stride + (j * Bc + c)], - data_m[m_offset + (i * Br + r * 4 + 1) * m_stride + (j * Bc + c)], - data_m[m_offset + (i * Br + r * 4 + 2) * m_stride + (j * Bc + c)], - data_m[m_offset + (i * Br + r * 4 + 3) * m_stride + (j * Bc + c)]); + m = f16vec4(data_m[m_offset + mask_row_offset(i * Br + r * 4) + (j * Bc + c)], + data_m[m_offset + mask_row_offset(i * Br + r * 4 + 1) + (j * Bc + c)], + data_m[m_offset + mask_row_offset(i * Br + r * 4 + 2) + (j * Bc + c)], + data_m[m_offset + mask_row_offset(i * Br + r * 4 + 3) + (j * Bc + c)]); max_mask = max(max(max(max(max_mask, float(m[0])), float(m[1])), float(m[2])), float(m[3])); } else if (i * Br + r * 4 + 2 < p.nem1) { - m = f16vec4(data_m[m_offset + (i * Br + r * 4 ) * m_stride + (j * Bc + c)], - data_m[m_offset + (i * Br + r * 4 + 1) * m_stride + (j * Bc + c)], - data_m[m_offset + (i * Br + r * 4 + 2) * m_stride + (j * Bc + c)], + m = f16vec4(data_m[m_offset + mask_row_offset(i * Br + r * 4) + (j * Bc + c)], + data_m[m_offset + mask_row_offset(i * Br + r * 4 + 1) + (j * Bc + c)], + data_m[m_offset + mask_row_offset(i * Br + r * 4 + 2) + (j * Bc + c)], 0.0); max_mask = max(max(max(max_mask, float(m[0])), float(m[1])), float(m[2])); } else if (i * Br + r * 4 + 1 < p.nem1) { - m = f16vec4(data_m[m_offset + (i * Br + r * 4 ) * m_stride + (j * Bc + c)], - data_m[m_offset + (i * Br + r * 4 + 1) * m_stride + (j * Bc + c)], + m = f16vec4(data_m[m_offset + mask_row_offset(i * Br + r * 4) + (j * Bc + c)], + data_m[m_offset + mask_row_offset(i * Br + r * 4 + 1) + (j * Bc + c)], 0.0, 0.0); max_mask = max(max(max_mask, float(m[0])), float(m[1])); } else if (i * Br + r * 4 < p.nem1) { - m = f16vec4(data_m[m_offset + (i * Br + r * 4 ) * m_stride + (j * Bc + c)], + m = f16vec4(data_m[m_offset + mask_row_offset(i * Br + r * 4) + (j * Bc + c)], 0.0, 0.0, 0.0); @@ -263,7 +271,7 @@ void main() { } // K * Q^T -> S^T: Bc x HSK_pad * HSK_pad x Br -> Bc x Br - // Bc split across workgroup (four subgroups), loop over HSK in chunks of 16: 16 x 16 * 16 x 16 -> 16 x 16 + // Each subgroup owns one query tile and one 16-column KV tile. // This is written transposed in order to allow for N being 8 if implementations need it coopmat SfMat = coopmat(0); coopmat KMat; @@ -308,23 +316,23 @@ void main() { } if (stage_k) { - uint coord = (gl_SubgroupID * MatBc) * kvsh_stride; + uint coord = (column_subgroup * MatBc) * kvsh_stride; coopMatLoad(KMat, kvsh, coord, kvsh_stride, gl_CooperativeMatrixLayoutRowMajor); } else { - const uint coord = k_offset / 4 + (j * Bc + gl_SubgroupID * MatBc) * k_stride / 4 + d * 16 / 4; + const uint coord = k_offset / 4 + (j * Bc + column_subgroup * MatBc) * k_stride / 4 + d * 16 / 4; coopMatLoad(KMat, data_kv4, coord, k_stride / 4, gl_CooperativeMatrixLayoutRowMajor); } } else { - uint coord = (gl_SubgroupID * MatBc) * kvsh_stride + d * 16 / 4; + uint coord = (column_subgroup * MatBc) * kvsh_stride + d * 16 / 4; coopMatLoad(KMat, kvsh, coord, kvsh_stride, gl_CooperativeMatrixLayoutRowMajor); } - coopMatLoad(QMat, Qf, d * 16 / 4, qstride, gl_CooperativeMatrixLayoutColumnMajor); + coopMatLoad(QMat, Qf, query_tile * MatBr * qstride + d * 16 / 4, qstride, gl_CooperativeMatrixLayoutColumnMajor); SfMat = coopMatMulAdd(KMat, QMat, SfMat); } - uint coord = gl_SubgroupID * MatBc * sfshstride; + uint coord = column_subgroup * MatBc * sfshstride + query_tile * MatBr / 4; coopMatStore(SfMat, sfsh, coord, sfshstride, gl_CooperativeMatrixLayoutRowMajor); barrier(); @@ -436,18 +444,18 @@ void main() { } barrier(); - const uint num_hsv_tiles = (HSV + MatBc * row_split - 1) / (MatBc * row_split); // round up + const uint num_hsv_tiles = (HSV + MatBc * column_subgroups - 1) / (MatBc * column_subgroups); - // Each subgroup handles HSV/4 columns + // Each subgroup computes one MatBc-column output tile per iteration. [[unroll]] for (uint32_t hsv_tile = 0; hsv_tile < num_hsv_tiles; ++hsv_tile) { - const uint hsv_offset = (hsv_tile * row_split + gl_SubgroupID) * 16; + const uint hsv_offset = (hsv_tile * column_subgroups + column_subgroup) * 16; coopmat PVMat = coopmat(0); - // Preload V tiles for [Bc, 16 * num subgroups] + // V staging tile: Bc rows by MatBc * column_subgroups columns. const uint v_rows = Bc; const uint v_total = v_rows * v_cols; - const uint v_loads_per_thread = v_total / gl_WorkGroupSize.x; + const uint v_loads_per_thread = CEIL_DIV(v_total, gl_WorkGroupSize.x); // If SHMEM_STAGING is set, a Bc * HSV_pad size tile of V is loaded to shmem. // If not, V is loaded directly from global memory if aligned, otherwise @@ -460,12 +468,15 @@ void main() { if (stage_v) { [[unroll]] for (uint32_t i = 0; i < v_loads_per_thread; ++i) { const uint idx = i * gl_WorkGroupSize.x + tid; + if (idx >= v_total) { + continue; + } const uint row = idx / v_cols; const uint col = idx % v_cols; uint32_t v_row; bool kv_active = fa_kv_index(j * Bc + row, v_row); - const uint v_col = hsv_tile * MatBc * row_split + col * 4; + const uint v_col = hsv_tile * MatBc * column_subgroups + col * 4; const uint coord = v_row * v_stride * BLOCK_SIZE_V + v_col; const uint ib = coord / BLOCK_SIZE_V; @@ -488,11 +499,11 @@ void main() { } barrier(); - const uint o_offset = gl_SubgroupID * MatBr / 4; + const uint o_offset = query_tile * MatBr * osh_stride + column_subgroup * MatBr / 4; if (hsv_offset < HSV_pad) { [[unroll]] for (uint32_t bc_chunk = 0; bc_chunk < Bc / MatBc; ++bc_chunk) { - coopMatLoad(KMat, Psh, bc_chunk * MatBc * psh_stride, psh_stride, gl_CooperativeMatrixLayoutColumnMajor); + coopMatLoad(KMat, Psh, bc_chunk * MatBc * psh_stride + query_tile * MatBr / 4, psh_stride, gl_CooperativeMatrixLayoutColumnMajor); if (SHMEM_STAGING == 0) { if (!USE_DECODE_V && !KV_bounds_check && !USE_SPARSE) { @@ -501,11 +512,11 @@ void main() { const uint v_tile_offset = v_offset / 4 + v_tile_row * v_stride / 4 + hsv_offset / 4; coopMatLoad(QMat, data_vv4, v_tile_offset, v_stride / 4, gl_CooperativeMatrixLayoutRowMajor); } else { - const uint v_tile_offset = bc_chunk * MatBr * v_cols + gl_SubgroupID * (MatBc / 4); + const uint v_tile_offset = bc_chunk * MatBr * v_cols + column_subgroup * (MatBc / 4); coopMatLoad(QMat, kvsh, v_tile_offset, vsh_stride, gl_CooperativeMatrixLayoutRowMajor); } } else { - const uint v_tile_offset = bc_chunk * MatBc * kvsh_stride + (hsv_tile * row_split + gl_SubgroupID) * (MatBc / 4); + const uint v_tile_offset = bc_chunk * MatBc * kvsh_stride + (hsv_tile * column_subgroups + column_subgroup) * (MatBc / 4); coopMatLoad(QMat, kvsh, v_tile_offset, kvsh_stride, gl_CooperativeMatrixLayoutRowMajor); } @@ -518,7 +529,7 @@ void main() { barrier(); - const uint hsv_per_tile = row_split * MatBc; + const uint hsv_per_tile = column_subgroups * MatBc; const uint hsv_base = hsv_tile * hsv_per_tile; const uint d_values_per_tile = hsv_per_tile / 4; @@ -560,16 +571,16 @@ void main() { const uint d = d0 + col_tid; if (d >= HSV/4) break; const uint d_local = d0 / threads_per_rowgroup; - gqaStore(tile_row(r), d, Of[r][d_local], o_offset, iq2, N); + gqa_store_row(tile_row(r), d, Of[r][d_local], o_offset, HSV * p.ne1 * p.k_num / 4); } } } o_offset = HSV * p.ne1 * p.k_num * p.ne2 * p.ne3 + p.ne1 * 2 * (split_k_index + p.k_num * (gqa_iq1 + p.ne2 * iq3)); [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) { - if (tile_row(r) < N) { - perElemOpStoreCol0(tile_row(r), 0u, ACC_TYPE(Lf[r]), o_offset, iq2, N); - perElemOpStoreCol0(tile_row(r), 0u, ACC_TYPE(Mf[r]), o_offset + p.ne1, iq2, N); + if (tile_row(r) < N && col_tid == 0) { + gqa_store_col0_row(tile_row(r), ACC_TYPE(Lf[r]), o_offset, p.ne1 * 2 * p.k_num); + gqa_store_col0_row(tile_row(r), ACC_TYPE(Mf[r]), o_offset + p.ne1, p.ne1 * 2 * p.k_num); } } } else { @@ -644,7 +655,7 @@ void main() { const uint d = d0 + col_tid; if (d >= HSV / 4) break; const uint d_local = d0 / threads_per_rowgroup; - gqaStore(tile_row(r), d, Of[r][d_local], o_offset, iq2, N); + gqa_store_row(tile_row(r), d, Of[r][d_local], o_offset, p.ne1 * HSV / 4); } } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl index 03c348bf9e3c..4f45b539aace 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl @@ -221,19 +221,22 @@ FLOAT_TYPE mmvq_dot_product(const uint ib_a, const uint iqs) { const i32vec4 qs_a = repack4(ib_a, iqs * 4); const uint8_t scale = get_scale(ib_a, iqs * 4); const vec2 dm = vec2(get_dm(ib_a)); - const int32_t scale_m = int32_t(scale >> 4) * 0x01010101; // Duplicate 8-bit value across 32-bits. - sum_d += dotPacked4x8EXT(qs_a.x, cache_b_qs[0]) * (scale & 0xF); - sum_m += dotPacked4x8EXT(scale_m, cache_b_qs[0]); + sum_d += dotPacked4x8EXT(qs_a.x, cache_b_qs[0]); + sum_m += dotPacked4x8EXT(0x01010101, cache_b_qs[0]); - sum_d += dotPacked4x8EXT(qs_a.y, cache_b_qs[1]) * (scale & 0xF); - sum_m += dotPacked4x8EXT(scale_m, cache_b_qs[1]); + sum_d += dotPacked4x8EXT(qs_a.y, cache_b_qs[1]); + sum_m += dotPacked4x8EXT(0x01010101, cache_b_qs[1]); - sum_d += dotPacked4x8EXT(qs_a.z, cache_b_qs[2]) * (scale & 0xF); - sum_m += dotPacked4x8EXT(scale_m, cache_b_qs[2]); + sum_d += dotPacked4x8EXT(qs_a.z, cache_b_qs[2]); + sum_m += dotPacked4x8EXT(0x01010101, cache_b_qs[2]); - sum_d += dotPacked4x8EXT(qs_a.w, cache_b_qs[3]) * (scale & 0xF); - sum_m += dotPacked4x8EXT(scale_m, cache_b_qs[3]); + sum_d += dotPacked4x8EXT(qs_a.w, cache_b_qs[3]); + sum_m += dotPacked4x8EXT(0x01010101, cache_b_qs[3]); + + // Integer factoring preserves the dot product while applying each scale once. + sum_d *= int32_t(scale & 0xF); + sum_m *= int32_t(scale >> 4); return FLOAT_TYPE(float(cache_b_ds.x) * (float(dm.x) * float(sum_d) - float(dm.y) * float(sum_m))); } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 3abdff5fd0da..9d6e847cafad 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9126,6 +9126,36 @@ struct test_flash_attn_ext_large_logits : public test_flash_attn_ext { } }; +// A causal window of n_swa positions at the end of a full KV cache, without an n_kv_max hint. +struct test_flash_attn_ext_swa_window : public test_flash_attn_ext { + int64_t n_swa; + + test_flash_attn_ext_swa_window(int64_t hs, int64_t nh, int64_t gqa, int64_t kv, int64_t nb, int64_t n_swa) + : test_flash_attn_ext(hs, hs, nh, {gqa, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, + GGML_TYPE_F16, {0, 2, 1, 3}), + n_swa(n_swa) {} + + std::string vars() override { + return test_flash_attn_ext::vars() + ",n_swa=" + std::to_string(n_swa); + } + + void initialize_tensors(ggml_context * ctx) override { + test_flash_attn_ext::initialize_tensors(ctx); + ggml_tensor * m = ggml_get_tensor(ctx, "m"); + const int64_t n_kv = m->ne[0]; + std::vector data(ggml_nelements(m)); + for (int64_t row = 0; row < ggml_nrows(m); ++row) { + const int64_t token = row % m->ne[1]; + const int64_t pos = n_kv - m->ne[1] + token; // position of this query token + for (int64_t col = 0; col < n_kv; ++col) { + const bool visible = col <= pos && pos - col < n_swa; + data[row * n_kv + col] = ggml_fp32_to_fp16(visible ? 0.0f : -INFINITY); + } + } + ggml_backend_tensor_set(m, data.data(), 0, data.size() * sizeof(ggml_fp16_t)); + } +}; + // GGML_OP_FLASH_ATTN_EXT with kv_rows: each q slice attends to its own list of K/V cache rows struct test_flash_attn_ext_kv_rows : public test_case { const int64_t hs; @@ -12063,6 +12093,15 @@ static std::vector> make_test_cases_eval() { } } + for (int n : {1, 4, 8}) { + for (int k : {256, 1536}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q2_K, GGML_TYPE_F32, 17, n, k, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q3_K, GGML_TYPE_F32, 17, n, k, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 17, n, k, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q2_K, GGML_TYPE_F32, 8, 4, false, 17, n, k, 0, true, true)); + } + } + // Test IQP panel path for all grid IQ types for (ggml_type type_a : {GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, GGML_TYPE_IQ4_XS}) { @@ -12965,6 +13004,47 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(65, 67, 4, {1, 1}, 17, 75, false, false, 0, 1.0f, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(65, 67, 4, {1, 1}, 113, 75, false, false, 0, 1.0f, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + // GQA with a few query tokens (speculative decoding): exercises folding tokens into GQA rows. + for (int64_t gqa : {2, 4, 6}) { + for (int64_t nb : {2, 3, 4, 8}) { + for (int64_t kv : {512, 8192}) { + for (bool sinks : {false, true}) { + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 2, {gqa, 1}, kv, nb, true, sinks, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3})); + } + } + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 2, {gqa, 2}, 1024, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 2, {gqa, 1}, 1024, nb, true, false, 8.0f, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 2, {gqa, 1}, 1024, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + } + } + + for (int64_t nb : {5, 7, 8}) { + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 2, {6, 1}, 8193, nb, true, true, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3})); + test_cases.emplace_back(new test_flash_attn_ext_swa_window(128, 2, 6, 16385, nb, 512)); + } + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 40, {6, 1}, 8193, 8, true, true, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3})); + for (int64_t gqa : {9, 13}) { + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 2, {gqa, 1}, 8193, 8, true, true, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3})); + } + for (float max_bias : {0.0f, 8.0f}) { + for (float logit_softcap : {0.0f, 10.0f}) { + if (max_bias == 0.0f && logit_softcap == 0.0f) { + continue; + } + for (int64_t nh : {2, 40}) { + test_cases.emplace_back(new test_flash_attn_ext(128, 128, nh, {6, 1}, 8193, 5, true, true, max_bias, logit_softcap, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3})); + } + } + } + + // Sliding-window and causal masks over a long full-size cache with a few query rows (mask_opt path). + for (int64_t gqa : {1, 6}) { + for (int64_t nb : {1, 2, 4, 8}) { + for (int64_t n_swa : {512, 4096, 16384}) { + test_cases.emplace_back(new test_flash_attn_ext_swa_window(128, 2, gqa, 16384, nb, n_swa)); + } + } + } // mixed quant and Q1_0 test cases test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_F16)); @@ -13764,6 +13844,15 @@ static std::vector> make_test_cases_perf() { } test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + + test_cases.emplace_back(new test_flash_attn_ext_swa_window(128, 8, 6, 131072, 8, 131072)); + + // Long-context GQA decode and verification with dense, sparse-list and sliding-window masks. + for (int64_t nb : {1, 8}) { + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 8, {6, 1}, 131072, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3})); + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 8, {6, 1}, 131072, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3}, true, false, 8192)); + test_cases.emplace_back(new test_flash_attn_ext_swa_window(128, 8, 6, 131072, nb, 512)); + } test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 4, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0));