diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp index 9ef56a6d0d15..4f3bae392d6c 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp @@ -103,12 +103,21 @@ struct GetRowsMatch { int64_t token_count = 0; int64_t row_count = 0; int64_t hidden_size = 0; + std::string weight_layout = kNativeWeightLayout; bool matched() const { return ids != nullptr && weight != nullptr && output != nullptr && weight_format_value >= 0; } }; +static DispatchBinding get_rows_weight_binding(const GetRowsMatch & match) { + if (match.weight_layout == kNativeWeightLayout) { + return { match.weight->id, 0, match.weight->byte_count }; + } + return { match.weight->id, 0, match.weight->byte_count, match.weight_layout, match.weight->type, + match.hidden_size, match.row_count, match.weight->byte_count }; +} + struct GetRowsRmsNormMatch { GetRowsMatch rows; const GraphNode * rms_node = nullptr; @@ -153,7 +162,7 @@ static void add_common_integer_parameters(Dispatch & dispatch, const GetRowsMatc static void add_primary_bindings(Dispatch & dispatch, const GetRowsMatch & match) { dispatch.bindings.push_back({ match.ids->id, 0, match.ids->byte_count }); - dispatch.bindings.push_back({ match.weight->id, 0, match.weight->byte_count }); + dispatch.bindings.push_back(get_rows_weight_binding(match)); dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count }); } @@ -256,8 +265,10 @@ static bool match_get_rows_scalar_3d_f32_dispatch(const DispatchMatchContext & c return true; } -static GetRowsMatch match_get_rows_f32(const Graph & graph, const GraphNode * node) { +static GetRowsMatch match_get_rows_f32(const DispatchMatchContext & context) { GetRowsMatch match; + const Graph & graph = context.graph; + const GraphNode * node = context.root_node; if (node == nullptr || node->op != GGML_OP_GET_ROWS || node->inputs.size() != 2) { return match; } @@ -293,12 +304,23 @@ static GetRowsMatch match_get_rows_f32(const Graph & graph, const GraphNode * no match.token_count = token_count; match.row_count = row_count; match.hidden_size = hidden_size; + if (context.row64_weight_policy && weight->alias_source.value < 0 && row_count % 64 == 0 && + hidden_size % 256 == 0 && + dispatch_weight_layout_eligible(*weight)) { + if (weight->type == GGML_TYPE_Q4_K) { + match.weight_format_value = common_mul_mat_format_config_value(CommonMulMatWeightFormat::Q4KRow64); + match.weight_layout = kQ4KPackedK256Row64Layout; + } else if (weight->type == GGML_TYPE_Q6_K) { + match.weight_format_value = common_mul_mat_format_config_value(CommonMulMatWeightFormat::Q6KRow64); + match.weight_layout = kQ6KPackedK256Row64ScaleRowLayout; + } + } return match; } static GetRowsScaleMatch match_get_rows_scale_f32(const DispatchMatchContext & context) { GetRowsScaleMatch match; - match.rows = match_get_rows_f32(context.graph, context.root_node); + match.rows = match_get_rows_f32(context); if (!match.rows.matched() || !context.graph.has_index()) { return {}; } @@ -345,7 +367,7 @@ static GetRowsScaleMatch match_get_rows_scale_f32(const DispatchMatchContext & c static GetRowsRmsNormMatch match_get_rows_rmsnorm_binary(const DispatchMatchContext & context) { GetRowsRmsNormMatch match; - match.rows = match_get_rows_f32(context.graph, context.root_node); + match.rows = match_get_rows_f32(context); if (!match.rows.matched() || !context.graph.has_index() || match.rows.hidden_size < 128 || match.rows.hidden_size > 32768 || match.rows.hidden_size % 128 != 0) { return {}; @@ -433,7 +455,7 @@ static Dispatch make_get_rows_next_dispatch(const GetRowsMatch & match, } static bool match_get_rows_f32_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) { - const GetRowsMatch match = match_get_rows_f32(context.graph, context.root_node); + const GetRowsMatch match = match_get_rows_f32(context); if (!match.matched()) { return false; } @@ -458,7 +480,7 @@ static bool match_get_rows_scale_f32_dispatch(const DispatchMatchContext & conte dispatch.kernel.compile_parameters.emplace("ggml.get_rows_scale_f32.scale", to_config_value(match.scale)); dispatch.kernel.compile_parameters.emplace("ggml.get_rows_scale_f32.bias", to_config_value(match.bias)); dispatch.bindings.push_back({ match.rows.ids->id, 0, match.rows.ids->byte_count }); - dispatch.bindings.push_back({ match.rows.weight->id, 0, match.rows.weight->byte_count }); + dispatch.bindings.push_back(get_rows_weight_binding(match.rows)); dispatch.bindings.push_back({ match.rows.output->id, 0, match.rows.output->byte_count }); dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count }); @@ -469,7 +491,7 @@ static bool match_get_rows_scale_f32_dispatch(const DispatchMatchContext & conte } static bool match_get_rows_f32_next_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) { - const GetRowsMatch match = match_get_rows_f32(context.graph, context.root_node); + const GetRowsMatch match = match_get_rows_f32(context); if (!match.matched()) { return false; } @@ -558,7 +580,7 @@ static bool match_get_rows_rmsnorm_binary_dispatch(const DispatchMatchContext & add_common_compile_parameters(dispatch, match.rows); dispatch.kernel.compile_parameters.emplace("ggml.get_rows_rmsnorm.rms_epsilon", to_config_value(match.epsilon)); dispatch.bindings.push_back({ match.rows.ids->id, 0, match.rows.ids->byte_count }); - dispatch.bindings.push_back({ match.rows.weight->id, 0, match.rows.weight->byte_count }); + dispatch.bindings.push_back(get_rows_weight_binding(match.rows)); dispatch.bindings.push_back({ match.norm_weight->id, 0, match.norm_weight->byte_count }); dispatch.bindings.push_back({ match.rows.output->id, 0, match.rows.output->byte_count }); dispatch.bindings.push_back({ match.normalized_output->id, 0, match.normalized_output->byte_count }); diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp index 848c0b98ddfa..ab09d0cde83c 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp @@ -103,6 +103,8 @@ static constexpr int64_t kMulMatVectorPackedMaxOutputSize = 26 static constexpr int64_t kMulMatVectorSelectedOutputsMaxOutputSize = 262144; static constexpr KernelCatalogRef kMulMatQ6KF32WmmaPrefillWave32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_q6_k_f32_wmma_prefill_wave32"); +static constexpr KernelCatalogRef kMulMatQ6KF32WmmaPrefillM64Wave32Kernel = + GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32"); static constexpr KernelCatalogRef kMulMatQ6KF16WmmaPrefillWave32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_q6_k_f16_wmma_prefill_wave32"); static constexpr KernelCatalogRef kSelectSymmetricI4K32GroupsKernel = @@ -903,10 +905,13 @@ static bool match_q6_k_prefill_wave32_dispatch(const DispatchMatchContext & cont return false; } - // Short prompts use masked M128 tiles without a separate activation copy. + // Short prompts use masked WMMA tiles without a separate activation copy. if (short_row64) { + // M64 halves reduce register pressure at these measured short shapes. + const bool use_m64 = match.output_size == 1024 && (match.input_size == 1024 || match.input_size == 3072); return build_q6_k_prefill_wave32_dispatch(match, match.input->id, match.input->byte_count, - kMulMatQ6KF32WmmaPrefillWave32Kernel, row64_weight, + use_m64 ? kMulMatQ6KF32WmmaPrefillM64Wave32Kernel : + kMulMatQ6KF32WmmaPrefillWave32Kernel, row64_weight, context.root_index, dispatch_match); } diff --git a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp index fcd7597bb721..bba2f27a5648 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp @@ -18,6 +18,8 @@ static constexpr KernelCatalogRef kQwenDenseLinearQ4KQ8NextQ8Kernel = GGML_HRX_KERNEL_REF("qwen3_moe", "qwen3_moe_dense_linear_q4k_q8_1_x4_next_q8"); static constexpr KernelCatalogRef kGgmlLinearQ6KQ8_1X4Kernel = GGML_HRX_KERNEL_REF("qwen3_moe", "ggml_linear_q6k_q8_1_x4"); +static constexpr KernelCatalogRef kMulMatVectorF32F32Kernel = + GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_vector_f32_f32"); static constexpr int64_t kQwenHiddenSize = kQwen30BMoeDispatchProfile.hidden_size; static constexpr int64_t kQwenVocabularyCount = 151936; @@ -336,24 +338,39 @@ bool qwen_accepts_q6k_q8_input(const DispatchMatchContext & context, output->ne[2] == 1 && output->ne[3] == 1; } -static void build_qwen_matmul_dispatch(const QwenMatmulMatch & match, - DispatchMatch & dispatch_match, - size_t root_index) { +Dispatch make_qwen_q6k_q8_projection_dispatch(const Value & weight, const Value & output, + const DispatchBinding & input, bool row64_weight_policy) { + const int64_t input_size = weight.ne[0]; + const int64_t output_size = weight.ne[1]; + const bool row64_weight = row64_weight_policy && weight.alias_source.value < 0 && + dispatch_weight_layout_eligible(weight); Dispatch dispatch; - dispatch.kernel = make_kernel_specialization(match.kernel); - dispatch.kernel.integer_parameters.emplace("token_count", match.token_count); - dispatch.kernel.integer_parameters.emplace("input_size", match.input_size); - dispatch.kernel.integer_parameters.emplace("output_size", match.output_size); - dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.token_capacity", - to_config_value(match.token_count)); - dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.output_capacity", - to_config_value(match.output_size)); - dispatch.bindings.push_back({ match.input_value, 0, match.input_bytes }); - dispatch.bindings.push_back({ match.weight->id, 0, match.weight->byte_count }); - dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count }); - - dispatch_match.covered_nodes.push_back(root_index); - dispatch_match.dispatches.push_back(std::move(dispatch)); + dispatch.kernel = + make_kernel_specialization(row64_weight ? kMulMatVectorF32F32Kernel : kGgmlLinearQ6KQ8_1X4Kernel); + dispatch.kernel.integer_parameters.emplace("token_count", 1); + dispatch.kernel.compile_parameters.emplace("ggml.workload.token_capacity", "1"); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.input_size", to_config_value(input_size)); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.output_size", to_config_value(output_size)); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.output_accumulation", "0"); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.output_unary_op", + std::to_string(unary_kind_config_value(UnaryKind::Identity))); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.publish_format", "0"); + dispatch.kernel.compile_parameters.emplace( + "ggml.matmul.vector.weight_format", + to_config_value(common_mul_mat_format_config_value( + row64_weight ? CommonMulMatWeightFormat::Q6KRow64 : CommonMulMatWeightFormat::Q6K))); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.activation_format", + std::to_string(GGML_TYPE_Q8_1)); + dispatch.bindings.push_back(input); + if (row64_weight) { + dispatch.bindings.push_back({ weight.id, 0, weight.byte_count, kQ6KPackedK256Row64ScaleRowLayout, + weight.type, input_size, output_size, weight.byte_count }); + } else { + dispatch.bindings.push_back({ weight.id, 0, weight.byte_count }); + } + dispatch.bindings.push_back({ output.id, 0, output.byte_count }); + dispatch.bindings.push_back({ output.id, 0, output.byte_count }); + return dispatch; } static bool match_qwen_q6k_q8_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) { @@ -361,7 +378,9 @@ static bool match_qwen_q6k_q8_dispatch(const DispatchMatchContext & context, Dis if (!match.matched()) { return false; } - build_qwen_matmul_dispatch(match, dispatch_match, context.root_index); + dispatch_match.covered_nodes.push_back(context.root_index); + dispatch_match.dispatches.push_back(make_qwen_q6k_q8_projection_dispatch( + *match.weight, *match.output, { match.input_value, 0, match.input_bytes }, context.row64_weight_policy)); return true; } diff --git a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.h b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.h index 28f348e9cacb..333279285a24 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.h +++ b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.h @@ -8,6 +8,10 @@ bool qwen_accepts_q6k_q8_input(const DispatchMatchContext & context, const GraphNode & consumer, const Value & input); +// The endpoint Q8 projection is a single-token path; prefill uses common matmul dispatch. +Dispatch make_qwen_q6k_q8_projection_dispatch(const Value & weight, const Value & output, + const DispatchBinding & input, bool row64_weight_policy); + void register_qwen_matmul_dispatches(DispatchRegistryBuilder & registry); } // namespace ggml::hrx diff --git a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-rmsnorm.cpp b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-rmsnorm.cpp index dce10fa60c64..dd3cacb97cff 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-rmsnorm.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-rmsnorm.cpp @@ -1,4 +1,5 @@ #include "dispatch-qwen-rmsnorm.h" +#include "dispatch-qwen-matmul.h" #include "dispatch-llm-profiles.h" #include "../common/dispatch-activation-publication.h" @@ -17,8 +18,6 @@ namespace { static constexpr KernelCatalogRef kQwenRmsNormF32QuantizeQ8_1X4Kernel = GGML_HRX_KERNEL_REF("qwen3_moe", "qwen3_moe_rmsnorm_f32_quantize_q8_1_x4"); -static constexpr KernelCatalogRef kGgmlLinearQ6KQ8_1X4Kernel = - GGML_HRX_KERNEL_REF("qwen3_moe", "ggml_linear_q6k_q8_1_x4"); static constexpr float kQwenRmsNormEpsilon = kQwen30BMoeDispatchProfile.rms_norm_epsilon; static constexpr int64_t kQwenHiddenSize = kQwen30BMoeDispatchProfile.hidden_size; static constexpr int64_t kQwenVocabularyCount = 151936; @@ -187,7 +186,7 @@ static QwenEndpointProjectionMatch match_qwen_endpoint_projection(const Graph & } if (input->id != match.output->id || weight->type != GGML_TYPE_Q6_K || output->type != GGML_TYPE_F32 || !weight->contiguous || !input->contiguous || !output->contiguous || match.hidden_size != kQwenHiddenSize || - !is_supported_token_count(match.token_count) || weight->ne[0] != match.hidden_size || + match.token_count != 1 || weight->ne[0] != match.hidden_size || weight->ne[1] != kQwenVocabularyCount || output->ne[0] != kQwenVocabularyCount || output->ne[1] != match.token_count || output->ne[2] != 1 || output->ne[3] != 1) { return {}; @@ -284,18 +283,9 @@ static bool match_qwen_rmsnorm_f32_quantize_q8_1_x4_dispatch(const DispatchMatch rms_dispatch.bindings.push_back({ rms_match.output->id, 0, rms_match.output->byte_count }); rms_dispatch.bindings.push_back({ q8_value, 0, q8_byte_count }); - Dispatch projection_dispatch; - projection_dispatch.kernel = make_kernel_specialization(kGgmlLinearQ6KQ8_1X4Kernel); - projection_dispatch.kernel.integer_parameters.emplace("token_count", rms_match.token_count); - projection_dispatch.kernel.integer_parameters.emplace("input_size", rms_match.hidden_size); - projection_dispatch.kernel.integer_parameters.emplace("output_size", kQwenVocabularyCount); - projection_dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.token_capacity", - to_config_value(rms_match.token_count)); - projection_dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.output_capacity", - to_config_value(kQwenVocabularyCount)); - projection_dispatch.bindings.push_back({ q8_value, 0, q8_byte_count }); - projection_dispatch.bindings.push_back({ projection_match.weight->id, 0, projection_match.weight->byte_count }); - projection_dispatch.bindings.push_back({ projection_match.output->id, 0, projection_match.output->byte_count }); + Dispatch projection_dispatch = make_qwen_q6k_q8_projection_dispatch( + *projection_match.weight, *projection_match.output, { q8_value, 0, q8_byte_count }, + context.row64_weight_policy); match.covered_nodes.push_back(rms_match.rms_node_index); match.covered_nodes.push_back(rms_match.mul_node_index); diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json index e88e8e69c243..4ff469eeeebb 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json @@ -667,6 +667,12 @@ }, { "path": "ops/reduction/amax_scale_f32.loom" + }, + { + "path": "motifs/matmul/q6_row64_q8_1_x4_decode.loom" + }, + { + "path": "ops/matmul/vector_q6_q8_native_test.loom" } ], "exports": [ @@ -2737,6 +2743,9 @@ ], "library_sources": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/dequant.loom", @@ -2752,6 +2761,9 @@ }, "compile_dependencies": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/dequant.loom", @@ -2806,6 +2818,9 @@ ], "library_sources": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/matmul/vector_postops.loom", @@ -2821,6 +2836,9 @@ }, "compile_dependencies": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/matmul/vector_postops.loom", @@ -3072,6 +3090,9 @@ ], "library_sources": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/llm_attention_qkv_matmul_postops.loom", @@ -3088,6 +3109,9 @@ }, "compile_dependencies": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/llm_attention_qkv_matmul_postops.loom", @@ -3145,6 +3169,9 @@ ], "library_sources": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/llm_attention_qkv_matmul_postops.loom", @@ -3161,6 +3188,9 @@ }, "compile_dependencies": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/llm_attention_qkv_matmul_postops.loom", @@ -3212,6 +3242,9 @@ ], "library_sources": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/llm_attention_qkv_matmul_postops.loom", @@ -3227,6 +3260,9 @@ }, "compile_dependencies": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/llm_attention_qkv_matmul_postops.loom", @@ -6131,6 +6167,53 @@ "motifs/elementwise/binary_apply.loom" ] }, + { + "name": "ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32", + "family": "loom_libs", + "symbol": "ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32", + "source": "ops/mul_mat_q6_k_packed_f16_wmma.loom", + "workload_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "launch_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "bindings": [ + "input", + "weight", + "output" + ], + "binding_access": [ + "read", + "read", + "write" + ], + "target_selector": "", + "compile_recipe": { + "mode": "direct", + "primary_sources": [ + "ops/mul_mat_q6_k_packed_f16_wmma.loom" + ], + "library_sources": [ + "motifs/q6_k_f16.loom", + "motifs/dequant.loom", + "motifs/mul_mat_quantized_f16_prefill.loom", + "motifs/elementwise/binary_apply.loom" + ] + }, + "compile_dependencies": [ + "motifs/q6_k_f16.loom", + "motifs/dequant.loom", + "motifs/mul_mat_quantized_f16_prefill.loom", + "motifs/elementwise/binary_apply.loom" + ] + }, { "name": "ggml_mul_mat_q6_k_f16_wmma_prefill_wave32", "family": "loom_libs", @@ -9921,6 +10004,148 @@ "compile_dependencies": [ "motifs/elementwise/unary_apply.loom" ] + }, + { + "name": "ggml_linear_q6k_q8_1_x4", + "family": "qwen3_moe", + "symbol": "ggml_linear_q6k_q8_1_x4", + "source": "ops/matmul/vector_f32_f32.loom", + "workload_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "launch_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "bindings": [ + "input", + "weight", + "output", + "next_output" + ], + "binding_access": [ + "read", + "read", + "read_write", + "write" + ], + "target_selector": "", + "compile_recipe": { + "mode": "direct", + "primary_sources": [ + "ops/matmul/vector_f32_f32.loom" + ], + "library_sources": [ + "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", + "motifs/matmul/vector_decode_pair.loom", + "motifs/matmul/vector_publish.loom", + "motifs/dequant.loom", + "motifs/elementwise/unary_apply.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ] + }, + "compile_dependencies": [ + "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", + "motifs/matmul/vector_decode_pair.loom", + "motifs/matmul/vector_publish.loom", + "motifs/dequant.loom", + "motifs/elementwise/unary_apply.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ] + }, + { + "name": "ggml_linear_q6k_q8_1_x4", + "family": "qwen3_moe", + "symbol": "ggml_linear_q6k_q8_1_x4_wave64_block4", + "source": "ops/matmul/vector_f32_f32.loom", + "workload_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "launch_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "bindings": [ + "input", + "weight", + "output", + "next_output" + ], + "binding_access": [ + "read", + "read", + "read_write", + "write" + ], + "target_selector": "gfx1151", + "compile_recipe": { + "mode": "direct", + "primary_sources": [ + "ops/matmul/vector_f32_f32.loom" + ], + "library_sources": [ + "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", + "motifs/matmul/vector_decode_pair.loom", + "motifs/matmul/vector_publish.loom", + "motifs/dequant.loom", + "motifs/elementwise/unary_apply.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ] + }, + "compile_dependencies": [ + "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", + "motifs/matmul/vector_decode_pair.loom", + "motifs/matmul/vector_publish.loom", + "motifs/dequant.loom", + "motifs/elementwise/unary_apply.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ] } ], "link_modules": [ @@ -9938,6 +10163,30 @@ "motifs/flash_attention/completion_counter_reduce.loom", "motifs/quantize_q8_1_x4.loom" ] + }, + { + "name": "vector_q6_q8_native_test_linked", + "srcs": [ + "ops/matmul/vector_q6_q8_native_test.loom" + ], + "libraries": [ + "ops/matmul/vector_f32_f32.loom", + "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", + "motifs/matmul/vector_decode_pair.loom", + "motifs/matmul/vector_publish.loom", + "motifs/dequant.loom", + "motifs/elementwise/unary_apply.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ] } ], "plan_cases": [ @@ -9981,6 +10230,78 @@ "--output-format=jsonl" ], "source": "ops/rows/get_rows_scale_f32.loom" + }, + { + "name": "vector_q6_q8_wave32_k768_tail_plan_test", + "args": [ + "$(location :vector_q6_q8_native_test_linked)", + "--benchmark=@ggml_mul_mat_vector_q6_q8_wave32_k768_tail_small", + "--config=ggml.workload.token_capacity=1", + "--config=ggml.matmul.vector.input_size=768", + "--config=ggml.matmul.vector.output_size=9", + "--config=ggml.matmul.vector.weight_format=6", + "--config=ggml.matmul.vector.activation_format=9", + "--config=ggml.matmul.vector.output_accumulation=0", + "--config=ggml.matmul.vector.output_unary_op=23", + "--config=ggml.matmul.vector.publish_format=0", + "--dry-run", + "--output-format=jsonl" + ], + "link_module": "vector_q6_q8_native_test_linked" + }, + { + "name": "vector_q6_q8_wave32_k2048_tail_plan_test", + "args": [ + "$(location :vector_q6_q8_native_test_linked)", + "--benchmark=@ggml_mul_mat_vector_q6_q8_wave32_k2048_tail_small", + "--config=ggml.workload.token_capacity=1", + "--config=ggml.matmul.vector.input_size=2048", + "--config=ggml.matmul.vector.output_size=9", + "--config=ggml.matmul.vector.weight_format=6", + "--config=ggml.matmul.vector.activation_format=9", + "--config=ggml.matmul.vector.output_accumulation=0", + "--config=ggml.matmul.vector.output_unary_op=23", + "--config=ggml.matmul.vector.publish_format=0", + "--dry-run", + "--output-format=jsonl" + ], + "link_module": "vector_q6_q8_native_test_linked" + }, + { + "name": "vector_q6_q8_wave64_k768_tail_plan_test", + "args": [ + "$(location :vector_q6_q8_native_test_linked)", + "--benchmark=@ggml_mul_mat_vector_q6_q8_wave64_k768_tail_small", + "--config=ggml.workload.token_capacity=1", + "--config=ggml.matmul.vector.input_size=768", + "--config=ggml.matmul.vector.output_size=9", + "--config=ggml.matmul.vector.weight_format=6", + "--config=ggml.matmul.vector.activation_format=9", + "--config=ggml.matmul.vector.output_accumulation=0", + "--config=ggml.matmul.vector.output_unary_op=23", + "--config=ggml.matmul.vector.publish_format=0", + "--dry-run", + "--output-format=jsonl" + ], + "link_module": "vector_q6_q8_native_test_linked" + }, + { + "name": "vector_q6_q8_wave64_k2048_tail_plan_test", + "args": [ + "$(location :vector_q6_q8_native_test_linked)", + "--benchmark=@ggml_mul_mat_vector_q6_q8_wave64_k2048_tail_small", + "--config=ggml.workload.token_capacity=1", + "--config=ggml.matmul.vector.input_size=2048", + "--config=ggml.matmul.vector.output_size=9", + "--config=ggml.matmul.vector.weight_format=6", + "--config=ggml.matmul.vector.activation_format=9", + "--config=ggml.matmul.vector.output_accumulation=0", + "--config=ggml.matmul.vector.output_unary_op=23", + "--config=ggml.matmul.vector.publish_format=0", + "--dry-run", + "--output-format=jsonl" + ], + "link_module": "vector_q6_q8_native_test_linked" } ], "sources": [], diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom index c906e25b77bc..a6e0f15d0108 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom @@ -1,6 +1,7 @@ // Copyright 2026 The HRX Authors // SPDX-License-Identifier: Apache-2.0 +// Native Q6_K/Q8_1 x4 contractions and wave32/wave64 schedules. template.decl @ggml.linear_q6k_q8_1_x4.body(%publish_output: i1, %token_count: index, %token0: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer) -> (f32, index, index, i1, index, index) func.decl @ggml_q8_1_x4_word(%q8_input: buffer, %row_byte_base: offset, %q8_block: index, %word_in_block: index) -> (vector<4xi8>, f32) @@ -202,3 +203,71 @@ template.def<@ggml.linear_q6k_q8_1_x4.body> device @ggml_linear_q6k_q8_1_x4_body %dot = kernel.subgroup.reduce %lane_acc : f32 template.return %dot, %token, %channel, %valid_channel, %launch_token_count, %bounded_output_size : f32, index, index, i1, index, index } + +func.def inline @ggml_q6_q8_native_wave32(%token_capacity: index, %output_capacity: index, %token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) { + %q8_noalias, %weight_noalias, %output_noalias = buffer.assume.noalias %q8_input, %weight, %output : buffer, buffer, buffer + %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048), le(%token_count, %token_capacity)] : index + %bounded_output_size = index.assume %output_size [range(%output_size, 1, 262144), le(%output_size, %output_capacity)] : index + %c0 = index.constant 0 : index + %token0 = kernel.workgroup.id : index + %valid_token = index.cmp ult, %token0, %bounded_token_count : index + %safe_token = scf.select %valid_token, %token0, %c0 : index + %dot, %token, %channel, %valid_channel, %launch_token_count, %output_bound = template.apply<@ggml.linear_q6k_q8_1_x4.body>(%valid_token, %bounded_token_count, %safe_token, %input_size, %bounded_output_size, %q8_noalias, %weight_noalias) : (i1, index, index, index, index, buffer, buffer) -> (f32, index, index, i1, index, index) + %c0_i32 = scalar.constant 0 : i32 + %c0_offset = index.constant 0 : offset + %lane = kernel.subgroup.lane.id : index + %lane_i32 = index.cast %lane : index to i32 + %is_lane_zero = scalar.cmpi eq, %lane_i32, %c0_i32 : i32 + scf.if %valid_token { + scf.if %valid_channel { + scf.if %is_lane_zero { + %output_view = buffer.view %output_noalias[%c0_offset] : buffer -> view<[%launch_token_count]x[%output_bound]xf32> + view.store %dot, %output_view[%token, %channel] : f32, view<[%launch_token_count]x[%output_bound]xf32> + } + } + } + func.return +} + +func.def inline @ggml_q6_q8_native_wave64_block4(%token_capacity: index, %output_capacity: index, %token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) { + %q8_noalias, %weight_noalias, %output_noalias = buffer.assume.noalias %q8_input, %weight, %output : buffer, buffer, buffer + %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048), le(%token_count, %token_capacity)] : index + %bounded_input_size = index.assume %input_size [range(%input_size, 256, 32768), multiple_of(%input_size, 256)] : index + %bounded_output_size = index.assume %output_size [range(%output_size, 1, 262144), le(%output_size, %output_capacity)] : index + %token = kernel.workgroup.id : index + %channel = kernel.workgroup.id : index + %lane = kernel.subgroup.lane.id : index + %valid_token = index.cmp ult, %token, %bounded_token_count : index + %valid_channel = index.cmp ult, %channel, %bounded_output_size : index + %publish_output = scalar.andi %valid_token, %valid_channel : i1 + %c0 = index.constant 0 : index + %c128 = index.constant 128 : index + %c144_bytes = index.constant 144 : offset + %c210_bytes = index.constant 210 : offset + %c256 = index.constant 256 : index + %safe_token = scf.select %valid_token, %token, %c0 : index + %safe_channel = scf.select %valid_channel, %channel, %c0 : index + %q6_block_count = index.div %bounded_input_size, %c256 : index + %weight_row_bytes = index.scale %q6_block_count, %c210_bytes : index, offset -> offset + %weight_row_byte_base = index.scale %safe_channel, %weight_row_bytes : index, offset -> offset + %q8_group_count = index.div %bounded_input_size, %c128 : index + %q8_row_bytes = index.scale %q8_group_count, %c144_bytes : index, offset -> offset + %q8_row_byte_base = index.scale %safe_token, %q8_row_bytes : index, offset -> offset + %lane_acc = scf.if %publish_output -> (f32) { + %value = func.call @ggml_q6k_q8_1_x4_row_lane_wave64_block4(%bounded_input_size, %weight_noalias, %weight_row_byte_base, %q8_noalias, %q8_row_byte_base, %lane) : (index, buffer, offset, buffer, offset, index) -> (f32) + scf.yield %value : f32 + } else { + %c0_f32 = scalar.constant 0.0 : f32 + scf.yield %c0_f32 : f32 + } + %dot = kernel.subgroup.reduce %lane_acc : f32 + %is_lane_zero = index.cmp eq, %lane, %c0 : index + scf.if %publish_output { + scf.if %is_lane_zero { + %c0_offset = index.constant 0 : offset + %output_view = buffer.view %output_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]x[%bounded_output_size]xf32> + view.store %dot, %output_view[%token, %channel] : f32, view<[%bounded_token_count]x[%bounded_output_size]xf32> + } + } + func.return +} diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_row64_q8_1_x4_decode.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_row64_q8_1_x4_decode.loom new file mode 100644 index 000000000000..024b7ec9f558 --- /dev/null +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_row64_q8_1_x4_decode.loom @@ -0,0 +1,365 @@ +// Copyright 2026 The HRX Authors +// SPDX-License-Identifier: Apache-2.0 + +// Row64 Q6_K weights contracted with packed Q8_1 x4 activations. +config.decl @ggml.matmul.vector.publish_format : %value: index where [range(%value, 0, 1)] +func.decl @ggml_matmul_vector_q8_1_x4_stage_slab(%input: buffer, %stage: buffer, %capacity: index, %slab_begin: index, %blocks: index, %global_row_bytes: offset) +func.decl pure @ggml_matmul_vector_q8_geometry(%token_capacity: index, %input_size: index, %output_size: index, %weight_format: index) -> (index, index) +func.decl @ggml_matmul_vector_q8_publish(%tokens: index, %n: index, %rows_per_group: index, %row_base: index, %local_row: index, %block_lane: index, %channel_group: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %publish_format: index, %total_g0: f32, %total_g1: f32, %total_g2: f32, %total_g3: f32, %total_g4: f32, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) + +func.def inline @ggml_matmul_vector_q6_row64_prefetch_words(%weight_na: buffer, %row_group_block: index, %block: index, %row_lane: index, %half16: index, %group: index, %half128: index, %ql_side: index, %packet: index) -> (f16, i8, vector<4xi32>, vector<4xi32>) { + %c2 = index.constant 2 : index + %record_bytes = index.constant 13440 : offset + %scale_add = index.constant 128 : offset + %raw_add = index.constant 1152 : offset + %group_block = index.add %row_group_block, %block : index + %block_base = index.scale %group_block, %record_bytes : index, offset -> offset + %scale_base = index.add %block_base, %scale_add : offset + %raw_base = index.add %block_base, %raw_add : offset + %d_view = buffer.view %weight_na[%block_base] : buffer -> view<64xf16> + %s_view = buffer.view %weight_na[%scale_base] : buffer -> view<64x2x8xi8> + %raw_view = buffer.view %weight_na[%raw_base] : buffer -> view<2x3x64x8xi32> + %dh = view.load %d_view[%row_lane] : view<64xf16> -> f16 + %si = view.load %s_view[%row_lane, %half16, %group] : view<64x2x8xi8> -> i8 + %ql = vector.load %raw_view[%half128, %ql_side, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<4xi32> + %qh = vector.load %raw_view[%half128, %c2, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<4xi32> + func.return %dh, %si, %ql, %qh : f16, i8, vector<4xi32>, vector<4xi32> +} + +func.def inline @ggml_matmul_vector_q6_q8_activation_values(%pipelined: i1, %token: index, %q8_row_bytes: offset, %q8_block_add: offset, %capacity: index, %current_slot: index, %q8_block0: index, %q8_input: buffer, %q8_word0: index, %q8_meta: index) -> (vector<16xi8>, f32) { + %code_offset = index.constant 16 : offset + %a, %ad = scf.if %pipelined -> (vector<16xi8>, f32) { + %base = index.constant 0 : offset + %c4 = index.constant 4 : index + %q8_block = index.assume %q8_block0 [range(%q8_block0, 0, 7)] : index + %word = index.add %q8_word0, %c4 : index + %q_view = buffer.view %q8_input[%base] : buffer -> view<2x[%capacity]x8x36xi32> + %ds_view = buffer.view %q8_input[%base] : buffer -> view<2x[%capacity]x8x72xf16> + %words = vector.load %q_view[%current_slot, %token, %q8_block, %word] : view<2x[%capacity]x8x36xi32> -> vector<4xi32> + %a = vector.bitcast %words : vector<4xi32> to vector<16xi8> + %scale = view.load %ds_view[%current_slot, %token, %q8_block, %q8_meta] : view<2x[%capacity]x8x72xf16> -> f16 + %ad = scalar.extf %scale : f16 to f32 + scf.yield %a, %ad : vector<16xi8>, f32 + } else { + %tb = index.scale %token, %q8_row_bytes : index, offset -> offset + %qb = index.add %tb, %q8_block_add : offset + %qc = index.add %qb, %code_offset : offset + %q_view = buffer.view %q8_input[%qc] : buffer -> view<32xi32> + %ds_view = buffer.view %q8_input[%qb] : buffer -> view<8xf16> + %words = vector.load %q_view[%q8_word0] : view<32xi32> -> vector<4xi32> + %a = vector.bitcast %words : vector<4xi32> to vector<16xi8> + %scale = view.load %ds_view[%q8_meta] : view<8xf16> -> f16 + %ad = scalar.extf %scale : f16 to f32 + scf.yield %a, %ad : vector<16xi8>, f32 + } + func.return %a, %ad : vector<16xi8>, f32 +} + +func.def inline @ggml_matmul_vector_q6_q8_dot16(%pipelined: i1, %w: vector<16xi8>, %a: vector<16xi8>) -> (i32) { + %sum = scf.if %pipelined -> (i32) { + %di0 = vector.splati 0 : vector<1xi32> + %dw0 = vector.slice %w[0] : vector<16xi8> -> vector<4xi8> + %da0 = vector.slice %a[0] : vector<16xi8> -> vector<4xi8> + %di1 = vector.dot4i %dw0, %da0, %di0 : vector<4xi8>, vector<4xi8>, vector<1xi32> + %dw1 = vector.slice %w[4] : vector<16xi8> -> vector<4xi8> + %da1 = vector.slice %a[4] : vector<16xi8> -> vector<4xi8> + %di2 = vector.dot4i %dw1, %da1, %di1 : vector<4xi8>, vector<4xi8>, vector<1xi32> + %dw2 = vector.slice %w[8] : vector<16xi8> -> vector<4xi8> + %da2 = vector.slice %a[8] : vector<16xi8> -> vector<4xi8> + %di3 = vector.dot4i %dw2, %da2, %di2 : vector<4xi8>, vector<4xi8>, vector<1xi32> + %dw3 = vector.slice %w[12] : vector<16xi8> -> vector<4xi8> + %da3 = vector.slice %a[12] : vector<16xi8> -> vector<4xi8> + %di4 = vector.dot4i %dw3, %da3, %di3 : vector<4xi8>, vector<4xi8>, vector<1xi32> + %chain_sum = vector.extract %di4[0] : vector<1xi32> -> i32 + scf.yield %chain_sum : i32 + } else { + %init = vector.splati 0 : vector<4xi32> + %dot = vector.dot4i %w, %a, %init : vector<16xi8>, vector<16xi8>, vector<4xi32> + %seed = scalar.constant 0 : i32 + %direct_sum = vector.reduce %dot, %seed : vector<4xi32>, i32 + scf.yield %direct_sum : i32 + } + func.return %sum : i32 +} + +func.def inline @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input: buffer, %capacity0: index, %slot_begin: index, %blocks: index, %global_row_bytes: offset) -> (vector<4xi32>) { + %capacity = index.assume %capacity0 [range(%capacity0, 1, 5)] : index + %c0 = index.constant 0 : index + %c4 = index.constant 4 : index + %c18 = index.constant 18 : index + %c72 = index.constant 72 : index + %packet_bytes = index.constant 16 : offset + %block_bytes = index.constant 288 : offset + %thread = kernel.workitem.id : index + %total_packets = index.mul %capacity, %c72 : index + %copy_packet = index.rem %thread, %total_packets : index + %copy_token = index.div %copy_packet, %c72 : index + %copy_inner = index.rem %copy_packet, %c72 : index + %remaining = index.sub %blocks, %slot_begin : index + %full = index.cmp uge, %remaining, %c4 : index + %active0 = scf.select %full, %c4, %remaining : index + %active = index.assume %active0 [range(%active0, 1, 4)] : index + %active_packets = index.mul %active, %c18 : index + %copy_valid = index.cmp ult, %copy_inner, %active_packets : index + %safe_inner = scf.select %copy_valid, %copy_inner, %c0 : index + %copy_row = index.scale %copy_token, %global_row_bytes : index, offset -> offset + %slot_bytes = index.scale %slot_begin, %block_bytes : index, offset -> offset + %copy_bytes = index.scale %safe_inner, %packet_bytes : index, offset -> offset + %copy_base0 = index.add %copy_row, %slot_bytes : offset + %copy_base = index.add %copy_base0, %copy_bytes : offset + %source = buffer.view %input[%copy_base] : buffer -> view<4xi32> + %values = vector.load %source[%c0] : view<4xi32> -> vector<4xi32> + func.return %values : vector<4xi32> +} +func.def inline @ggml_matmul_vector_q8_1_x4_publish_half_slab(%stage: buffer, %capacity0: index, %slot0: index, %values: vector<4xi32>) { + %capacity = index.assume %capacity0 [range(%capacity0, 1, 5)] : index + %slot = index.assume %slot0 [range(%slot0, 0, 1)] : index + %c4 = index.constant 4 : index + %c72 = index.constant 72 : index + %base = index.constant 0 : offset + %thread = kernel.workitem.id : index + %total_packets = index.mul %capacity, %c72 : index + %thread_valid = index.cmp ult, %thread, %total_packets : index + scf.if %thread_valid { + %copy_token0 = index.div %thread, %c72 : index + %copy_token = index.assume %copy_token0 [range(%copy_token0, 0, 4), lt(%copy_token0, %capacity)] : index + %copy_inner = index.rem %thread, %c72 : index + %origin = index.mul %copy_inner, %c4 : index + %destination = buffer.view %stage[%base] : buffer -> view<2x[%capacity]x288xi32> + vector.store %values, %destination[%slot, %copy_token, %origin] : vector<4xi32>, view<2x[%capacity]x288xi32> + } + func.return +} + +func.def inline @ggml_matmul_vector_q6_q8_accumulate(%pipelined: i1, %tokens: index, %q8_row_bytes: offset, %q8_block_add: offset, %capacity: index, %current_slot: index, %q8_block0: index, %q8_input: buffer, %q8_word0: index, %q8_meta: index, %w: vector<16xi8>, %ws: f32, %acc_g0: f32, %acc_g1: f32, %acc_g2: f32, %acc_g3: f32, %acc_g4: f32) -> (f32, f32, f32, f32, f32) { + %code_offset = index.constant 16 : offset + %t0 = index.constant 0 : index + %valid0 = index.cmp ult, %t0, %tokens : index + %computed_g0 = scf.if %valid0 -> (f32) { + %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t0, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) + %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) + %sumf = scalar.sitofp %sum : i32 to f32 + %scale = scalar.mulf %ws, %ad : f32 + %updated = scalar.fmaf %sumf, %scale, %acc_g0 : f32 + scf.yield %updated : f32 + } else { + scf.yield %acc_g0 : f32 + } + %t1 = index.constant 1 : index + %valid1 = index.cmp ult, %t1, %tokens : index + %computed_g1 = scf.if %valid1 -> (f32) { + %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t1, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) + %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) + %sumf = scalar.sitofp %sum : i32 to f32 + %scale = scalar.mulf %ws, %ad : f32 + %updated = scalar.fmaf %sumf, %scale, %acc_g1 : f32 + scf.yield %updated : f32 + } else { + scf.yield %acc_g1 : f32 + } + %t2 = index.constant 2 : index + %valid2 = index.cmp ult, %t2, %tokens : index + %computed_g2 = scf.if %valid2 -> (f32) { + %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t2, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) + %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) + %sumf = scalar.sitofp %sum : i32 to f32 + %scale = scalar.mulf %ws, %ad : f32 + %updated = scalar.fmaf %sumf, %scale, %acc_g2 : f32 + scf.yield %updated : f32 + } else { + scf.yield %acc_g2 : f32 + } + %t3 = index.constant 3 : index + %valid3 = index.cmp ult, %t3, %tokens : index + %computed_g3 = scf.if %valid3 -> (f32) { + %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t3, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) + %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) + %sumf = scalar.sitofp %sum : i32 to f32 + %scale = scalar.mulf %ws, %ad : f32 + %updated = scalar.fmaf %sumf, %scale, %acc_g3 : f32 + scf.yield %updated : f32 + } else { + scf.yield %acc_g3 : f32 + } + %t4 = index.constant 4 : index + %valid4 = index.cmp ult, %t4, %tokens : index + %computed_g4 = scf.if %valid4 -> (f32) { + %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t4, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) + %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) + %sumf = scalar.sitofp %sum : i32 to f32 + %scale = scalar.mulf %ws, %ad : f32 + %updated = scalar.fmaf %sumf, %scale, %acc_g4 : f32 + scf.yield %updated : f32 + } else { + scf.yield %acc_g4 : f32 + } + func.return %computed_g0, %computed_g1, %computed_g2, %computed_g3, %computed_g4 : f32, f32, f32, f32, f32 +} + +func.def inline @ggml_matmul_vector_q6_row64_q8_1_x4_activation_dot(%capacity: index, %weight_format: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) { + %staged_rows, %k_partitions = func.call pure @ggml_matmul_vector_q8_geometry(%capacity, %input_size0, %output_size0, %weight_format) : (index, index, index, index) -> (index, index) + %no_staged_rows = index.constant 0 : index + %staged = index.cmp ugt, %staged_rows, %no_staged_rows : index + %tokens = index.assume %token_count [range(%token_count, 1, 1), le(%token_count, %capacity)] : index + %k = index.assume %input_size0 [range(%input_size0, 256, 32768), multiple_of(%input_size0, 256)] : index + %n = index.assume %output_size0 [range(%output_size0, 64, 262144), multiple_of(%output_size0, 64)] : index + %c0 = index.constant 0 : index + %c1 = index.constant 1 : index + %c2 = index.constant 2 : index + %c3 = index.constant 3 : index + %c4 = index.constant 4 : index + %c7 = index.constant 7 : index + %c8 = index.constant 8 : index + %c16 = index.constant 16 : index + %c32 = index.constant 32 : index + %c64 = index.constant 64 : index + %c128 = index.constant 128 : index + %c256 = index.constant 256 : index + %base = index.constant 0 : offset + %zero = scalar.constant 0.0 : f32 + %lane0 = kernel.subgroup.lane.id : index + %lane = index.assume %lane0 [range(%lane0, 0, 63)] : index + %wave = kernel.subgroup.id : index + %rows_per_group = scf.select %staged, %c32, %c8 : index + %row_base = index.mul %channel_group, %rows_per_group : index + %wave_row = index.mul %wave, %c4 : index + %cohort = index.div %lane, %c16 : index + %local_row = index.add %wave_row, %cohort : index + %row = index.add %row_base, %local_row : index + %block_lane = index.rem %lane, %c16 : index + %group = index.div %block_lane, %c2 : index + %half16 = index.rem %block_lane, %c2 : index + %half128 = index.div %group, %c4 : index + %group_in_half = index.rem %group, %c4 : index + %ql_side = index.rem %group_in_half, %c2 : index + %nibble = index.div %group_in_half, %c2 : index + %nibble_shift0 = index.mul %nibble, %c4 : index + %nibble_shift1 = index.cast %nibble_shift0 : index to i32 + %nibble_shift = vector.splat %nibble_shift1 : vector<4xi32> + %high_shift0 = index.mul %group_in_half, %c2 : index + %high_shift1 = index.cast %high_shift0 : index to i32 + %high_shift = vector.splat %high_shift1 : vector<4xi32> + %packet = index.mul %half16, %c4 : index + %q8_group_words = index.mul %group_in_half, %c8 : index + %q8_word0 = index.add %q8_group_words, %packet : index + %q8_meta = index.mul %group_in_half, %c2 : index + %blocks = index.div %k, %c256 : index + %q8_groups = index.div %k, %c128 : index + %q8_block_bytes = index.constant 144 : offset + %global_q8_row_bytes = index.scale %q8_groups, %q8_block_bytes : index, offset -> offset + %native_block_bytes = index.constant 210 : index + %cache_bytes = index.constant 33554432 : index + %weight_blocks = index.mul %blocks, %n : index + %weight_bytes = index.mul %weight_blocks, %native_block_bytes : index + %exceeds_cache = index.cmp uge, %weight_bytes, %cache_bytes : index + %pipelined = scalar.andi %staged, %exceeds_cache : i1 + %full_slab_row_bytes = index.constant 2304 : offset + %half_slab_row_bytes = index.constant 1152 : offset + %slab_row_bytes = scf.select %pipelined, %half_slab_row_bytes, %full_slab_row_bytes : offset + %q8_row_bytes = scf.select %staged, %slab_row_bytes, %global_q8_row_bytes : offset + %code_offset = index.constant 16 : offset + %record_bytes = index.constant 13440 : offset + %scale_add = index.constant 128 : offset + %raw_add = index.constant 1152 : offset + %staged_row_group = index.div %channel_group, %c2 : index + %direct_row_group = index.div %row, %c64 : index + %row_group = scf.select %staged, %staged_row_group, %direct_row_group : index + %row_lane = index.rem %row, %c64 : index + %row_group_block = index.mul %row_group, %blocks : index + %input_na, %weight_na = buffer.assume.noalias %input, %weight : buffer, buffer + %slab_blocks = scf.select %pipelined, %c4, %c8 : index + %full_stage_bytes = index.constant 11520 : offset + %q8_stage_bytes = scf.select %staged, %full_stage_bytes, %base : offset + %q8_stage = buffer.alloca align(16) %q8_stage_bytes : buffer + %q8_input = scf.select %staged, %q8_stage, %input_na : buffer + %zero_half = scalar.constant 0.0 : f16 + %zero_code = scalar.constant 0 : i8 + %zero_words = vector.splati 0 : vector<4xi32> + %prime_d, %prime_s, %prime_ql, %prime_qh = scf.if %staged -> (f16, i8, vector<4xi32>, vector<4xi32>) { + %pd, %ps, %pql, %pqh = func.call @ggml_matmul_vector_q6_row64_prefetch_words(%weight_na, %row_group_block, %c0, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) + scf.yield %pd, %ps, %pql, %pqh : f16, i8, vector<4xi32>, vector<4xi32> + } else { + scf.yield %zero_half, %zero_code, %zero_words, %zero_words : f16, i8, vector<4xi32>, vector<4xi32> + } + scf.if %pipelined { + %prime_q8 = func.call @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input_na, %capacity, %c0, %blocks, %global_q8_row_bytes) : (buffer, index, index, index, offset) -> (vector<4xi32>) + func.call @ggml_matmul_vector_q8_1_x4_publish_half_slab(%q8_stage, %capacity, %c0, %prime_q8) : (buffer, index, index, vector<4xi32>) + kernel.barrier scope(workgroup) ordering(acq_rel) + } + %total_g0, %total_g1, %total_g2, %total_g3, %total_g4, %last_d, %last_s, %last_ql, %last_qh = scf.for %block = [%c0 to %blocks step %c1](%acc_g0 = %zero : f32, %acc_g1 = %zero : f32, %acc_g2 = %zero : f32, %acc_g3 = %zero : f32, %acc_g4 = %zero : f32, %carry_d = %prime_d : f16, %carry_s = %prime_s : i8, %carry_ql = %prime_ql : vector<4xi32>, %carry_qh = %prime_qh : vector<4xi32>) -> (f32, f32, f32, f32, f32, f16, i8, vector<4xi32>, vector<4xi32>) { + %block_in_slab = index.rem %block, %slab_blocks : index + %slab_boundary = index.cmp eq, %block_in_slab, %c0 : index + %direct_stage = scalar.andi %staged, %slab_boundary : i1 + %true = scalar.constant true : i1 + %not_pipelined = scalar.xori %pipelined, %true : i1 + %stage_current = scalar.andi %direct_stage, %not_pipelined : i1 + scf.if %stage_current { + func.call @ggml_matmul_vector_q8_1_x4_stage_slab(%input_na, %q8_stage, %capacity, %block, %blocks, %global_q8_row_bytes) : (buffer, buffer, index, index, index, offset) + } + %slab_number = index.div %block, %slab_blocks : index + %current_slot = index.rem %slab_number, %c2 : index + %following_slab = index.add %slab_number, %c1 : index + %following_begin = index.mul %following_slab, %slab_blocks : index + %following_exists = index.cmp ult, %following_begin, %blocks : index + %next_slot = index.rem %following_slab, %c2 : index + %staged_future = scalar.andi %pipelined, %following_exists : i1 + %current_d, %current_s, %current_ql, %current_qh = scf.if %staged -> (f16, i8, vector<4xi32>, vector<4xi32>) { + scf.yield %carry_d, %carry_s, %carry_ql, %carry_qh : f16, i8, vector<4xi32>, vector<4xi32> + } else { + %wd, %ws8, %wql, %wqh = func.call @ggml_matmul_vector_q6_row64_prefetch_words(%weight_na, %row_group_block, %block, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) + scf.yield %wd, %ws8, %wql, %wqh : f16, i8, vector<4xi32>, vector<4xi32> + } + %d = scalar.extf %current_d : f16 to f32 + %sf = scalar.sitofp %current_s : i8 to f32 + %ws = scalar.mulf %d, %sf : f32 + %nibble_mask = vector.splati 252645135 : vector<4xi32> + %high_mask = vector.splati 50529027 : vector<4xi32> + %four = vector.splati 4 : vector<4xi32> + %ql_shifted = vector.shrui %current_ql, %nibble_shift : vector<4xi32> + %low = vector.andi %ql_shifted, %nibble_mask : vector<4xi32> + %qh_shifted = vector.shrui %current_qh, %high_shift : vector<4xi32> + %high0 = vector.andi %qh_shifted, %high_mask : vector<4xi32> + %high = vector.shli %high0, %four : vector<4xi32> + %code = vector.ori %low, %high : vector<4xi32> + // Guard bits prevent subtraction from borrowing across packed bytes. + %guard_mask = vector.splati -2139062144 : vector<4xi32> + %center = vector.splati 538976288 : vector<4xi32> + %guarded = vector.ori %code, %guard_mask : vector<4xi32> + %centered = vector.subi %guarded, %center : vector<4xi32> + %signed_code = vector.xori %centered, %guard_mask : vector<4xi32> + %w = vector.bitcast %signed_code : vector<4xi32> to vector<16xi8> + %next_block = index.add %block, %c1 : index + %next_exists = index.cmp ult, %next_block, %blocks : index + %has_next = scalar.andi %staged, %next_exists : i1 + %next_d, %next_s, %next_ql, %next_qh = scf.if %has_next -> (f16, i8, vector<4xi32>, vector<4xi32>) { + %wd, %ws8, %wql, %wqh = func.call @ggml_matmul_vector_q6_row64_prefetch_words(%weight_na, %row_group_block, %next_block, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) + scf.yield %wd, %ws8, %wql, %wqh : f16, i8, vector<4xi32>, vector<4xi32> + } else { + scf.yield %carry_d, %carry_s, %carry_ql, %carry_qh : f16, i8, vector<4xi32>, vector<4xi32> + } + %input_block = scf.select %staged, %block_in_slab, %block : index + %q8_block0 = index.mul %input_block, %c2 : index + %q8_block = index.add %q8_block0, %half128 : index + %q8_local_block_add = index.scale %q8_block, %q8_block_bytes : index, offset -> offset + %slot_row = index.mul %current_slot, %capacity : index + %slot_byte_add0 = index.scale %slot_row, %slab_row_bytes : index, offset -> offset + %slot_byte_add = scf.select %pipelined, %slot_byte_add0, %base : offset + %q8_block_add = index.add %slot_byte_add, %q8_local_block_add : offset + %slot_complete = index.cmp eq, %block_in_slab, %c3 : index + %publish_next = scalar.andi %staged_future, %slot_complete : i1 + %next_g0, %next_g1, %next_g2, %next_g3, %next_g4 = scf.if %publish_next -> (f32, f32, f32, f32, f32) { + %next_q8 = func.call @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input_na, %capacity, %following_begin, %blocks, %global_q8_row_bytes) : (buffer, index, index, index, offset) -> (vector<4xi32>) + %v0, %v1, %v2, %v3, %v4 = func.call @ggml_matmul_vector_q6_q8_accumulate(%pipelined, %tokens, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block, %q8_input, %q8_word0, %q8_meta, %w, %ws, %acc_g0, %acc_g1, %acc_g2, %acc_g3, %acc_g4) : (i1, index, offset, offset, index, index, index, buffer, index, index, vector<16xi8>, f32, f32, f32, f32, f32, f32) -> (f32, f32, f32, f32, f32) + func.call @ggml_matmul_vector_q8_1_x4_publish_half_slab(%q8_stage, %capacity, %next_slot, %next_q8) : (buffer, index, index, vector<4xi32>) + kernel.barrier scope(workgroup) ordering(acq_rel) + scf.yield %v0, %v1, %v2, %v3, %v4 : f32, f32, f32, f32, f32 + } else { + %v0, %v1, %v2, %v3, %v4 = func.call @ggml_matmul_vector_q6_q8_accumulate(%pipelined, %tokens, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block, %q8_input, %q8_word0, %q8_meta, %w, %ws, %acc_g0, %acc_g1, %acc_g2, %acc_g3, %acc_g4) : (i1, index, offset, offset, index, index, index, buffer, index, index, vector<16xi8>, f32, f32, f32, f32, f32, f32) -> (f32, f32, f32, f32, f32) + scf.yield %v0, %v1, %v2, %v3, %v4 : f32, f32, f32, f32, f32 + } + scf.yield %next_g0, %next_g1, %next_g2, %next_g3, %next_g4, %next_d, %next_s, %next_ql, %next_qh : f32, f32, f32, f32, f32, f16, i8, vector<4xi32>, vector<4xi32> + } + %publish_format = config.get @ggml.matmul.vector.publish_format : index + func.call @ggml_matmul_vector_q8_publish(%tokens, %n, %rows_per_group, %row_base, %local_row, %block_lane, %channel_group, %output_accumulation, %output_unary_op, %epsilon, %publish_format, %total_g0, %total_g1, %total_g2, %total_g3, %total_g4, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, index, index, index, index, index, index, index, index, f32, index, f32, f32, f32, f32, f32, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) + func.return +} diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_activation.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_activation.loom index 30dacc26c240..67b8c95f67e8 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_activation.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_activation.loom @@ -1,3 +1,7 @@ +func.decl @ggml_matmul_vector_q6_row64_q8_1_x4_activation_dot(%capacity: index, %weight_format: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) +func.decl @ggml_q6_q8_native_wave32(%token_capacity: index, %output_capacity: index, %token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) +func.decl @ggml_q6_q8_native_wave64_block4(%token_capacity: index, %output_capacity: index, %token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) + func.decl @ggml_q4k_scale_min_from_header(%scale0: i32, %scale1: i32, %scale2: i32, %q4_group: index) -> (i32, i32) func.decl @ggml_dot_u8_s8_vector8_f32(%weight: vector<2xi32>, %activation: vector<2xi32>) -> (f32) @@ -1086,375 +1090,30 @@ func.def inline @ggml_matmul_vector_q4_q8_1_x4_activation_dot(%weight_format: in } // Native Q6_K carrier with signed I8 dot products and F32 scaling. -func.def inline @ggml_matmul_vector_q6_prefetch_native_words(%weight_na: buffer, %row_group_block: index, %block: index, %row_lane: index, %half16: index, %group: index, %half128: index, %ql_side: index, %packet: index) -> (f16, i8, vector<4xi32>, vector<4xi32>) { - %c2 = index.constant 2 : index - %record_bytes = index.constant 13440 : offset - %scale_add = index.constant 128 : offset - %raw_add = index.constant 1152 : offset - %group_block = index.add %row_group_block, %block : index - %block_base = index.scale %group_block, %record_bytes : index, offset -> offset - %scale_base = index.add %block_base, %scale_add : offset - %raw_base = index.add %block_base, %raw_add : offset - %d_view = buffer.view %weight_na[%block_base] : buffer -> view<64xf16> - %s_view = buffer.view %weight_na[%scale_base] : buffer -> view<64x2x8xi8> - %raw_view = buffer.view %weight_na[%raw_base] : buffer -> view<2x3x64x8xi32> - %dh = view.load %d_view[%row_lane] : view<64xf16> -> f16 - %si = view.load %s_view[%row_lane, %half16, %group] : view<64x2x8xi8> -> i8 - %ql = vector.load %raw_view[%half128, %ql_side, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<4xi32> - %qh = vector.load %raw_view[%half128, %c2, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<4xi32> - func.return %dh, %si, %ql, %qh : f16, i8, vector<4xi32>, vector<4xi32> -} - -func.def inline @ggml_matmul_vector_q6_q8_activation_values(%pipelined: i1, %token: index, %q8_row_bytes: offset, %q8_block_add: offset, %capacity: index, %current_slot: index, %q8_block0: index, %q8_input: buffer, %q8_word0: index, %q8_meta: index) -> (vector<16xi8>, f32) { - %code_offset = index.constant 16 : offset - %a, %ad = scf.if %pipelined -> (vector<16xi8>, f32) { - %base = index.constant 0 : offset - %c4 = index.constant 4 : index - %q8_block = index.assume %q8_block0 [range(%q8_block0, 0, 7)] : index - %word = index.add %q8_word0, %c4 : index - %q_view = buffer.view %q8_input[%base] : buffer -> view<2x[%capacity]x8x36xi32> - %ds_view = buffer.view %q8_input[%base] : buffer -> view<2x[%capacity]x8x72xf16> - %words = vector.load %q_view[%current_slot, %token, %q8_block, %word] : view<2x[%capacity]x8x36xi32> -> vector<4xi32> - %a = vector.bitcast %words : vector<4xi32> to vector<16xi8> - %scale = view.load %ds_view[%current_slot, %token, %q8_block, %q8_meta] : view<2x[%capacity]x8x72xf16> -> f16 - %ad = scalar.extf %scale : f16 to f32 - scf.yield %a, %ad : vector<16xi8>, f32 - } else { - %tb = index.scale %token, %q8_row_bytes : index, offset -> offset - %qb = index.add %tb, %q8_block_add : offset - %qc = index.add %qb, %code_offset : offset - %q_view = buffer.view %q8_input[%qc] : buffer -> view<32xi32> - %ds_view = buffer.view %q8_input[%qb] : buffer -> view<8xf16> - %words = vector.load %q_view[%q8_word0] : view<32xi32> -> vector<4xi32> - %a = vector.bitcast %words : vector<4xi32> to vector<16xi8> - %scale = view.load %ds_view[%q8_meta] : view<8xf16> -> f16 - %ad = scalar.extf %scale : f16 to f32 - scf.yield %a, %ad : vector<16xi8>, f32 - } - func.return %a, %ad : vector<16xi8>, f32 -} - -func.def inline @ggml_matmul_vector_q6_q8_dot16(%pipelined: i1, %w: vector<16xi8>, %a: vector<16xi8>) -> (i32) { - %sum = scf.if %pipelined -> (i32) { - %di0 = vector.splati 0 : vector<1xi32> - %dw0 = vector.slice %w[0] : vector<16xi8> -> vector<4xi8> - %da0 = vector.slice %a[0] : vector<16xi8> -> vector<4xi8> - %di1 = vector.dot4i %dw0, %da0, %di0 : vector<4xi8>, vector<4xi8>, vector<1xi32> - %dw1 = vector.slice %w[4] : vector<16xi8> -> vector<4xi8> - %da1 = vector.slice %a[4] : vector<16xi8> -> vector<4xi8> - %di2 = vector.dot4i %dw1, %da1, %di1 : vector<4xi8>, vector<4xi8>, vector<1xi32> - %dw2 = vector.slice %w[8] : vector<16xi8> -> vector<4xi8> - %da2 = vector.slice %a[8] : vector<16xi8> -> vector<4xi8> - %di3 = vector.dot4i %dw2, %da2, %di2 : vector<4xi8>, vector<4xi8>, vector<1xi32> - %dw3 = vector.slice %w[12] : vector<16xi8> -> vector<4xi8> - %da3 = vector.slice %a[12] : vector<16xi8> -> vector<4xi8> - %di4 = vector.dot4i %dw3, %da3, %di3 : vector<4xi8>, vector<4xi8>, vector<1xi32> - %chain_sum = vector.extract %di4[0] : vector<1xi32> -> i32 - scf.yield %chain_sum : i32 - } else { - %init = vector.splati 0 : vector<4xi32> - %dot = vector.dot4i %w, %a, %init : vector<16xi8>, vector<16xi8>, vector<4xi32> - %seed = scalar.constant 0 : i32 - %direct_sum = vector.reduce %dot, %seed : vector<4xi32>, i32 - scf.yield %direct_sum : i32 - } - func.return %sum : i32 -} - -func.def inline @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input: buffer, %capacity0: index, %slot_begin: index, %blocks: index, %global_row_bytes: offset) -> (vector<4xi32>) { - %capacity = index.assume %capacity0 [range(%capacity0, 1, 5)] : index - %c0 = index.constant 0 : index - %c4 = index.constant 4 : index - %c18 = index.constant 18 : index - %c72 = index.constant 72 : index - %packet_bytes = index.constant 16 : offset - %block_bytes = index.constant 288 : offset - %thread = kernel.workitem.id : index - %total_packets = index.mul %capacity, %c72 : index - %copy_packet = index.rem %thread, %total_packets : index - %copy_token = index.div %copy_packet, %c72 : index - %copy_inner = index.rem %copy_packet, %c72 : index - %remaining = index.sub %blocks, %slot_begin : index - %full = index.cmp uge, %remaining, %c4 : index - %active0 = scf.select %full, %c4, %remaining : index - %active = index.assume %active0 [range(%active0, 1, 4)] : index - %active_packets = index.mul %active, %c18 : index - %copy_valid = index.cmp ult, %copy_inner, %active_packets : index - %safe_inner = scf.select %copy_valid, %copy_inner, %c0 : index - %copy_row = index.scale %copy_token, %global_row_bytes : index, offset -> offset - %slot_bytes = index.scale %slot_begin, %block_bytes : index, offset -> offset - %copy_bytes = index.scale %safe_inner, %packet_bytes : index, offset -> offset - %copy_base0 = index.add %copy_row, %slot_bytes : offset - %copy_base = index.add %copy_base0, %copy_bytes : offset - %source = buffer.view %input[%copy_base] : buffer -> view<4xi32> - %values = vector.load %source[%c0] : view<4xi32> -> vector<4xi32> - func.return %values : vector<4xi32> -} -func.def inline @ggml_matmul_vector_q8_1_x4_publish_half_slab(%stage: buffer, %capacity0: index, %slot0: index, %values: vector<4xi32>) { - %capacity = index.assume %capacity0 [range(%capacity0, 1, 5)] : index - %slot = index.assume %slot0 [range(%slot0, 0, 1)] : index - %c4 = index.constant 4 : index - %c72 = index.constant 72 : index - %base = index.constant 0 : offset - %thread = kernel.workitem.id : index - %total_packets = index.mul %capacity, %c72 : index - %thread_valid = index.cmp ult, %thread, %total_packets : index - scf.if %thread_valid { - %copy_token0 = index.div %thread, %c72 : index - %copy_token = index.assume %copy_token0 [range(%copy_token0, 0, 4), lt(%copy_token0, %capacity)] : index - %copy_inner = index.rem %thread, %c72 : index - %origin = index.mul %copy_inner, %c4 : index - %destination = buffer.view %stage[%base] : buffer -> view<2x[%capacity]x288xi32> - vector.store %values, %destination[%slot, %copy_token, %origin] : vector<4xi32>, view<2x[%capacity]x288xi32> - } - func.return -} - - -func.def inline @ggml_matmul_vector_q6_q8_accumulate(%pipelined: i1, %tokens: index, %q8_row_bytes: offset, %q8_block_add: offset, %capacity: index, %current_slot: index, %q8_block0: index, %q8_input: buffer, %q8_word0: index, %q8_meta: index, %w: vector<16xi8>, %ws: f32, %acc_g0: f32, %acc_g1: f32, %acc_g2: f32, %acc_g3: f32, %acc_g4: f32) -> (f32, f32, f32, f32, f32) { - %code_offset = index.constant 16 : offset - %t0 = index.constant 0 : index - %valid0 = index.cmp ult, %t0, %tokens : index - %computed_g0 = scf.if %valid0 -> (f32) { - %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t0, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) - %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) - %sumf = scalar.sitofp %sum : i32 to f32 - %scale = scalar.mulf %ws, %ad : f32 - %updated = scalar.fmaf %sumf, %scale, %acc_g0 : f32 - scf.yield %updated : f32 - } else { - scf.yield %acc_g0 : f32 - } - %t1 = index.constant 1 : index - %valid1 = index.cmp ult, %t1, %tokens : index - %computed_g1 = scf.if %valid1 -> (f32) { - %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t1, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) - %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) - %sumf = scalar.sitofp %sum : i32 to f32 - %scale = scalar.mulf %ws, %ad : f32 - %updated = scalar.fmaf %sumf, %scale, %acc_g1 : f32 - scf.yield %updated : f32 - } else { - scf.yield %acc_g1 : f32 - } - %t2 = index.constant 2 : index - %valid2 = index.cmp ult, %t2, %tokens : index - %computed_g2 = scf.if %valid2 -> (f32) { - %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t2, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) - %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) - %sumf = scalar.sitofp %sum : i32 to f32 - %scale = scalar.mulf %ws, %ad : f32 - %updated = scalar.fmaf %sumf, %scale, %acc_g2 : f32 - scf.yield %updated : f32 - } else { - scf.yield %acc_g2 : f32 - } - %t3 = index.constant 3 : index - %valid3 = index.cmp ult, %t3, %tokens : index - %computed_g3 = scf.if %valid3 -> (f32) { - %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t3, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) - %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) - %sumf = scalar.sitofp %sum : i32 to f32 - %scale = scalar.mulf %ws, %ad : f32 - %updated = scalar.fmaf %sumf, %scale, %acc_g3 : f32 - scf.yield %updated : f32 - } else { - scf.yield %acc_g3 : f32 - } - %t4 = index.constant 4 : index - %valid4 = index.cmp ult, %t4, %tokens : index - %computed_g4 = scf.if %valid4 -> (f32) { - %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t4, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) - %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) - %sumf = scalar.sitofp %sum : i32 to f32 - %scale = scalar.mulf %ws, %ad : f32 - %updated = scalar.fmaf %sumf, %scale, %acc_g4 : f32 - scf.yield %updated : f32 - } else { - scf.yield %acc_g4 : f32 - } - func.return %computed_g0, %computed_g1, %computed_g2, %computed_g3, %computed_g4 : f32, f32, f32, f32, f32 -} - -func.def inline @ggml_matmul_vector_q6_q8_1_x4_activation_dot(%weight_format: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) { - %capacity = config.get @ggml.workload.token_capacity : index - %staged_rows, %k_partitions = func.call pure @ggml_matmul_vector_q8_geometry(%capacity, %input_size0, %output_size0, %weight_format) : (index, index, index, index) -> (index, index) - %no_staged_rows = index.constant 0 : index - %staged = index.cmp ugt, %staged_rows, %no_staged_rows : index - %tokens = index.assume %token_count [range(%token_count, 1, 1), le(%token_count, %capacity)] : index - %k = index.assume %input_size0 [range(%input_size0, 256, 32768), multiple_of(%input_size0, 256)] : index - %n = index.assume %output_size0 [range(%output_size0, 64, 262144), multiple_of(%output_size0, 64)] : index - %c0 = index.constant 0 : index - %c1 = index.constant 1 : index - %c2 = index.constant 2 : index - %c3 = index.constant 3 : index - %c4 = index.constant 4 : index - %c7 = index.constant 7 : index - %c8 = index.constant 8 : index - %c16 = index.constant 16 : index - %c32 = index.constant 32 : index - %c64 = index.constant 64 : index - %c128 = index.constant 128 : index - %c256 = index.constant 256 : index - %base = index.constant 0 : offset - %zero = scalar.constant 0.0 : f32 - %lane0 = kernel.subgroup.lane.id : index - %lane = index.assume %lane0 [range(%lane0, 0, 63)] : index - %wave = kernel.subgroup.id : index - %rows_per_group = scf.select %staged, %c32, %c8 : index - %row_base = index.mul %channel_group, %rows_per_group : index - %wave_row = index.mul %wave, %c4 : index - %cohort = index.div %lane, %c16 : index - %local_row = index.add %wave_row, %cohort : index - %row = index.add %row_base, %local_row : index - %block_lane = index.rem %lane, %c16 : index - %group = index.div %block_lane, %c2 : index - %half16 = index.rem %block_lane, %c2 : index - %half128 = index.div %group, %c4 : index - %group_in_half = index.rem %group, %c4 : index - %ql_side = index.rem %group_in_half, %c2 : index - %nibble = index.div %group_in_half, %c2 : index - %nibble_shift0 = index.mul %nibble, %c4 : index - %nibble_shift1 = index.cast %nibble_shift0 : index to i32 - %nibble_shift = vector.splat %nibble_shift1 : vector<4xi32> - %high_shift0 = index.mul %group_in_half, %c2 : index - %high_shift1 = index.cast %high_shift0 : index to i32 - %high_shift = vector.splat %high_shift1 : vector<4xi32> - %packet = index.mul %half16, %c4 : index - %q8_group_words = index.mul %group_in_half, %c8 : index - %q8_word0 = index.add %q8_group_words, %packet : index - %q8_meta = index.mul %group_in_half, %c2 : index - %blocks = index.div %k, %c256 : index - %q8_groups = index.div %k, %c128 : index - %q8_block_bytes = index.constant 144 : offset - %global_q8_row_bytes = index.scale %q8_groups, %q8_block_bytes : index, offset -> offset - %native_block_bytes = index.constant 210 : index - %cache_bytes = index.constant 33554432 : index - %weight_blocks = index.mul %blocks, %n : index - %weight_bytes = index.mul %weight_blocks, %native_block_bytes : index - %exceeds_cache = index.cmp uge, %weight_bytes, %cache_bytes : index - %pipelined = scalar.andi %staged, %exceeds_cache : i1 - %full_slab_row_bytes = index.constant 2304 : offset - %half_slab_row_bytes = index.constant 1152 : offset - %slab_row_bytes = scf.select %pipelined, %half_slab_row_bytes, %full_slab_row_bytes : offset - %q8_row_bytes = scf.select %staged, %slab_row_bytes, %global_q8_row_bytes : offset - %code_offset = index.constant 16 : offset - %record_bytes = index.constant 13440 : offset - %scale_add = index.constant 128 : offset - %raw_add = index.constant 1152 : offset - %staged_row_group = index.div %channel_group, %c2 : index - %direct_row_group = index.div %row, %c64 : index - %row_group = scf.select %staged, %staged_row_group, %direct_row_group : index - %row_lane = index.rem %row, %c64 : index - %row_group_block = index.mul %row_group, %blocks : index - %input_na, %weight_na = buffer.assume.noalias %input, %weight : buffer, buffer - %slab_blocks = scf.select %pipelined, %c4, %c8 : index - %full_stage_bytes = index.constant 11520 : offset - %q8_stage_bytes = scf.select %staged, %full_stage_bytes, %base : offset - %q8_stage = buffer.alloca align(16) %q8_stage_bytes : buffer - %q8_input = scf.select %staged, %q8_stage, %input_na : buffer - %zero_half = scalar.constant 0.0 : f16 - %zero_code = scalar.constant 0 : i8 - %zero_words = vector.splati 0 : vector<4xi32> - %prime_d, %prime_s, %prime_ql, %prime_qh = scf.if %staged -> (f16, i8, vector<4xi32>, vector<4xi32>) { - %pd, %ps, %pql, %pqh = func.call @ggml_matmul_vector_q6_prefetch_native_words(%weight_na, %row_group_block, %c0, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) - scf.yield %pd, %ps, %pql, %pqh : f16, i8, vector<4xi32>, vector<4xi32> - } else { - scf.yield %zero_half, %zero_code, %zero_words, %zero_words : f16, i8, vector<4xi32>, vector<4xi32> - } - scf.if %pipelined { - %prime_q8 = func.call @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input_na, %capacity, %c0, %blocks, %global_q8_row_bytes) : (buffer, index, index, index, offset) -> (vector<4xi32>) - func.call @ggml_matmul_vector_q8_1_x4_publish_half_slab(%q8_stage, %capacity, %c0, %prime_q8) : (buffer, index, index, vector<4xi32>) - kernel.barrier scope(workgroup) ordering(acq_rel) - } - %total_g0, %total_g1, %total_g2, %total_g3, %total_g4, %last_d, %last_s, %last_ql, %last_qh = scf.for %block = [%c0 to %blocks step %c1](%acc_g0 = %zero : f32, %acc_g1 = %zero : f32, %acc_g2 = %zero : f32, %acc_g3 = %zero : f32, %acc_g4 = %zero : f32, %carry_d = %prime_d : f16, %carry_s = %prime_s : i8, %carry_ql = %prime_ql : vector<4xi32>, %carry_qh = %prime_qh : vector<4xi32>) -> (f32, f32, f32, f32, f32, f16, i8, vector<4xi32>, vector<4xi32>) { - %block_in_slab = index.rem %block, %slab_blocks : index - %slab_boundary = index.cmp eq, %block_in_slab, %c0 : index - %direct_stage = scalar.andi %staged, %slab_boundary : i1 - %true = scalar.constant true : i1 - %not_pipelined = scalar.xori %pipelined, %true : i1 - %stage_current = scalar.andi %direct_stage, %not_pipelined : i1 - scf.if %stage_current { - func.call @ggml_matmul_vector_q8_1_x4_stage_slab(%input_na, %q8_stage, %capacity, %block, %blocks, %global_q8_row_bytes) : (buffer, buffer, index, index, index, offset) - } - %slab_number = index.div %block, %slab_blocks : index - %current_slot = index.rem %slab_number, %c2 : index - %following_slab = index.add %slab_number, %c1 : index - %following_begin = index.mul %following_slab, %slab_blocks : index - %following_exists = index.cmp ult, %following_begin, %blocks : index - %next_slot = index.rem %following_slab, %c2 : index - %staged_future = scalar.andi %pipelined, %following_exists : i1 - %current_d, %current_s, %current_ql, %current_qh = scf.if %staged -> (f16, i8, vector<4xi32>, vector<4xi32>) { - scf.yield %carry_d, %carry_s, %carry_ql, %carry_qh : f16, i8, vector<4xi32>, vector<4xi32> - } else { - %wd, %ws8, %wql, %wqh = func.call @ggml_matmul_vector_q6_prefetch_native_words(%weight_na, %row_group_block, %block, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) - scf.yield %wd, %ws8, %wql, %wqh : f16, i8, vector<4xi32>, vector<4xi32> - } - %d = scalar.extf %current_d : f16 to f32 - %sf = scalar.sitofp %current_s : i8 to f32 - %ws = scalar.mulf %d, %sf : f32 - %nibble_mask = vector.splati 252645135 : vector<4xi32> - %high_mask = vector.splati 50529027 : vector<4xi32> - %four = vector.splati 4 : vector<4xi32> - %ql_shifted = vector.shrui %current_ql, %nibble_shift : vector<4xi32> - %low = vector.andi %ql_shifted, %nibble_mask : vector<4xi32> - %qh_shifted = vector.shrui %current_qh, %high_shift : vector<4xi32> - %high0 = vector.andi %qh_shifted, %high_mask : vector<4xi32> - %high = vector.shli %high0, %four : vector<4xi32> - %code = vector.ori %low, %high : vector<4xi32> - // Guard bits prevent subtraction from borrowing across packed bytes. - %guard_mask = vector.splati -2139062144 : vector<4xi32> - %center = vector.splati 538976288 : vector<4xi32> - %guarded = vector.ori %code, %guard_mask : vector<4xi32> - %centered = vector.subi %guarded, %center : vector<4xi32> - %signed_code = vector.xori %centered, %guard_mask : vector<4xi32> - %w = vector.bitcast %signed_code : vector<4xi32> to vector<16xi8> - %next_block = index.add %block, %c1 : index - %next_exists = index.cmp ult, %next_block, %blocks : index - %has_next = scalar.andi %staged, %next_exists : i1 - %next_d, %next_s, %next_ql, %next_qh = scf.if %has_next -> (f16, i8, vector<4xi32>, vector<4xi32>) { - %wd, %ws8, %wql, %wqh = func.call @ggml_matmul_vector_q6_prefetch_native_words(%weight_na, %row_group_block, %next_block, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) - scf.yield %wd, %ws8, %wql, %wqh : f16, i8, vector<4xi32>, vector<4xi32> - } else { - scf.yield %carry_d, %carry_s, %carry_ql, %carry_qh : f16, i8, vector<4xi32>, vector<4xi32> - } - %input_block = scf.select %staged, %block_in_slab, %block : index - %q8_block0 = index.mul %input_block, %c2 : index - %q8_block = index.add %q8_block0, %half128 : index - %q8_local_block_add = index.scale %q8_block, %q8_block_bytes : index, offset -> offset - %slot_row = index.mul %current_slot, %capacity : index - %slot_byte_add0 = index.scale %slot_row, %slab_row_bytes : index, offset -> offset - %slot_byte_add = scf.select %pipelined, %slot_byte_add0, %base : offset - %q8_block_add = index.add %slot_byte_add, %q8_local_block_add : offset - %slot_complete = index.cmp eq, %block_in_slab, %c3 : index - %publish_next = scalar.andi %staged_future, %slot_complete : i1 - %next_g0, %next_g1, %next_g2, %next_g3, %next_g4 = scf.if %publish_next -> (f32, f32, f32, f32, f32) { - %next_q8 = func.call @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input_na, %capacity, %following_begin, %blocks, %global_q8_row_bytes) : (buffer, index, index, index, offset) -> (vector<4xi32>) - %v0, %v1, %v2, %v3, %v4 = func.call @ggml_matmul_vector_q6_q8_accumulate(%pipelined, %tokens, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block, %q8_input, %q8_word0, %q8_meta, %w, %ws, %acc_g0, %acc_g1, %acc_g2, %acc_g3, %acc_g4) : (i1, index, offset, offset, index, index, index, buffer, index, index, vector<16xi8>, f32, f32, f32, f32, f32, f32) -> (f32, f32, f32, f32, f32) - func.call @ggml_matmul_vector_q8_1_x4_publish_half_slab(%q8_stage, %capacity, %next_slot, %next_q8) : (buffer, index, index, vector<4xi32>) - kernel.barrier scope(workgroup) ordering(acq_rel) - scf.yield %v0, %v1, %v2, %v3, %v4 : f32, f32, f32, f32, f32 - } else { - %v0, %v1, %v2, %v3, %v4 = func.call @ggml_matmul_vector_q6_q8_accumulate(%pipelined, %tokens, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block, %q8_input, %q8_word0, %q8_meta, %w, %ws, %acc_g0, %acc_g1, %acc_g2, %acc_g3, %acc_g4) : (i1, index, offset, offset, index, index, index, buffer, index, index, vector<16xi8>, f32, f32, f32, f32, f32, f32) -> (f32, f32, f32, f32, f32) - scf.yield %v0, %v1, %v2, %v3, %v4 : f32, f32, f32, f32, f32 - } - scf.yield %next_g0, %next_g1, %next_g2, %next_g3, %next_g4, %next_d, %next_s, %next_ql, %next_qh : f32, f32, f32, f32, f32, f16, i8, vector<4xi32>, vector<4xi32> - } - %publish_format = config.get @ggml.matmul.vector.publish_format : index - func.call @ggml_matmul_vector_q8_publish(%tokens, %n, %rows_per_group, %row_base, %local_row, %block_lane, %channel_group, %output_accumulation, %output_unary_op, %epsilon, %publish_format, %total_g0, %total_g1, %total_g2, %total_g3, %total_g4, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, index, index, index, index, index, index, index, index, f32, index, f32, f32, f32, f32, f32, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) - func.return -} - - template.def<@ggml.matmul.vector_activation.core> device @ggml_matmul_vector_activation_core(%weight_format: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) { %activation_format = config.get @ggml.matmul.vector.activation_format : index %q8_format = index.constant 9 : index %uses_q8 = index.cmp eq, %activation_format, %q8_format : index scf.if %uses_q8 { %q6_format = index.constant 46 : index - %uses_q6 = index.cmp eq, %weight_format, %q6_format : index + %q6_native_format = index.constant 6 : index + %uses_row64_q6 = index.cmp eq, %weight_format, %q6_format : index + %uses_native_q6 = index.cmp eq, %weight_format, %q6_native_format : index + %uses_q6 = scalar.ori %uses_row64_q6, %uses_native_q6 : i1 scf.if %uses_q6 { - func.call @ggml_matmul_vector_q6_q8_1_x4_activation_dot(%weight_format, %token_count, %input_size0, %output_size0, %output_accumulation, %output_unary_op, %epsilon, %channel_group, %token_group, %input, %weight, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) + %token_capacity = config.get @ggml.workload.token_capacity : index + scf.if %uses_row64_q6 { + func.call @ggml_matmul_vector_q6_row64_q8_1_x4_activation_dot(%token_capacity, %weight_format, %token_count, %input_size0, %output_size0, %output_accumulation, %output_unary_op, %epsilon, %channel_group, %token_group, %input, %weight, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) + } else { + %wave = target.subgroup.size : index + %c64 = index.constant 64 : index + %wave64 = index.cmp eq, %wave, %c64 : index + scf.if %wave64 { + func.call @ggml_q6_q8_native_wave64_block4(%token_capacity, %output_size0, %token_count, %input_size0, %output_size0, %input, %weight, %output) : (index, index, index, index, index, buffer, buffer, buffer) + } else { + func.call @ggml_q6_q8_native_wave32(%token_capacity, %output_size0, %token_count, %input_size0, %output_size0, %input, %weight, %output) : (index, index, index, index, index, buffer, buffer, buffer) + } + } } else { func.call @ggml_matmul_vector_q4_q8_1_x4_activation_dot(%weight_format, %token_count, %input_size0, %output_size0, %output_accumulation, %output_unary_op, %epsilon, %channel_group, %token_group, %input, %weight, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) } diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_decode_pair.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_decode_pair.loom index f5e982b2adbf..d827e19929c1 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_decode_pair.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_decode_pair.loom @@ -14,6 +14,68 @@ func.decl @ggml_iq3s_grid_f32_halves() -> (vector<32xf32>, vector<32xf32>, vecto func.decl @ggml_dequant_f32_vector4(%iq4nl_table: vector<16xi8>, %iq3s_grid_low0: vector<32xf32>, %iq3s_grid_low1: vector<32xf32>, %iq3s_grid_low2: vector<32xf32>, %iq3s_grid_low3: vector<32xf32>, %iq3s_grid_low4: vector<32xf32>, %iq3s_grid_low5: vector<32xf32>, %iq3s_grid_low6: vector<32xf32>, %iq3s_grid_low7: vector<32xf32>, %iq3s_grid_low8: vector<32xf32>, %iq3s_grid_low9: vector<32xf32>, %iq3s_grid_low10: vector<32xf32>, %iq3s_grid_low11: vector<32xf32>, %iq3s_grid_low12: vector<32xf32>, %iq3s_grid_low13: vector<32xf32>, %iq3s_grid_low14: vector<32xf32>, %iq3s_grid_low15: vector<32xf32>, %iq3s_grid_high0: vector<32xf32>, %iq3s_grid_high1: vector<32xf32>, %iq3s_grid_high2: vector<32xf32>, %iq3s_grid_high3: vector<32xf32>, %iq3s_grid_high4: vector<32xf32>, %iq3s_grid_high5: vector<32xf32>, %iq3s_grid_high6: vector<32xf32>, %iq3s_grid_high7: vector<32xf32>, %iq3s_grid_high8: vector<32xf32>, %iq3s_grid_high9: vector<32xf32>, %iq3s_grid_high10: vector<32xf32>, %iq3s_grid_high11: vector<32xf32>, %iq3s_grid_high12: vector<32xf32>, %iq3s_grid_high13: vector<32xf32>, %iq3s_grid_high14: vector<32xf32>, %iq3s_grid_high15: vector<32xf32>, %weight_format: index, %weight: buffer, %row_byte_base: offset, %row: index, %input_size: index, %quant_block: index, %quant_group: index, %packet: index, %k: index) -> (vector<4xf32>) +// The four groups share a high-bit word, two low-bit words, and one block scale. +func.def inline @ggml_q6k_row64_pair_load_decode_state(%weight: buffer, %row: index, %input_size: index, %block: index, %packet: index, %half: index) -> (f32, vector<4xi8>, vector<1xi32>, vector<1xi32>, vector<1xi32>) { + %c0 = index.constant 0 : index + %c1 = index.constant 1 : index + %c2 = index.constant 2 : index + %c4 = index.constant 4 : index + %c64 = index.constant 64 : index + %c256 = index.constant 256 : index + %tile_bytes = index.constant 13440 : offset + %scale_offset = index.constant 128 : offset + %raw_offset = index.constant 1152 : offset + %blocks = index.div %input_size, %c256 : index + %row_group = index.div %row, %c64 : index + %row_lane = index.rem %row, %c64 : index + %tile = index.madd %row_group, %blocks, %block : index + %tile_base = index.scale %tile, %tile_bytes : index, offset -> offset + %scale_base = index.add %tile_base, %scale_offset : offset + %raw_base = index.add %tile_base, %raw_offset : offset + %raw = buffer.view %weight[%raw_base] : buffer -> view<2x3x64x8xi32> + %scales = buffer.view %weight[%scale_base] : buffer -> view<64x2x8xi8> + %ds = buffer.view %weight[%tile_base] : buffer -> view<64xf16> + %packet_half = index.div %packet, %c4 : index + %group_base = index.mul %half, %c4 : index + %d16 = view.load %ds[%row_lane] : view<64xf16> -> f16 + %d = scalar.extf %d16 : f16 to f32 + %s = vector.load %scales[%row_lane, %packet_half, %group_base] : view<64x2x8xi8> -> vector<4xi8> + %ql0 = vector.load %raw[%half, %c0, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<1xi32> + %ql1 = vector.load %raw[%half, %c1, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<1xi32> + %qh = vector.load %raw[%half, %c2, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<1xi32> + func.return %d, %s, %ql0, %ql1, %qh : f32, vector<4xi8>, vector<1xi32>, vector<1xi32>, vector<1xi32> +} + +func.def inline @ggml_q6k_row64_pair_preloaded_f32_vector4(%group: index, %d: f32, %scale_i8: i8, %ql_word: vector<1xi32>, %qh_word: vector<1xi32>) -> (vector<4xf32>) { + %c2 = index.constant 2 : index + %c4 = index.constant 4 : index + %c4v = vector.splati 4 : vector<1xi32> + %nibble_mask = vector.splati 252645135 : vector<1xi32> + %high_mask = vector.splati 50529027 : vector<1xi32> + %nibble = index.div %group, %c2 : index + %ls = index.mul %nibble, %c4 : index + %hs = index.mul %group, %c2 : index + %ls32 = index.cast %ls : index to i32 + %hs32 = index.cast %hs : index to i32 + %lsv = vector.splat %ls32 : vector<1xi32> + %hsv = vector.splat %hs32 : vector<1xi32> + %lshift = vector.shrui %ql_word, %lsv : vector<1xi32> + %hshift = vector.shrui %qh_word, %hsv : vector<1xi32> + %ql = vector.andi %lshift, %nibble_mask : vector<1xi32> + %qh_low = vector.andi %hshift, %high_mask : vector<1xi32> + %qh = vector.shli %qh_low, %c4v : vector<1xi32> + %codes = vector.ori %ql, %qh : vector<1xi32> + %bytes = vector.bitcast %codes : vector<1xi32> to vector<4xi8> + %c32 = vector.splatf 32.0 : vector<4xf32> + %unbiased = vector.uitofp %bytes : vector<4xi8> to vector<4xf32> + %wide = vector.subf %unbiased, %c32 : vector<4xf32> + %scale = scalar.sitofp %scale_i8 : i8 to f32 + %combined = scalar.mulf %scale, %d : f32 + %sv = vector.splat %combined : vector<4xf32> + %result = vector.mulf %wide, %sv : vector<4xf32> + func.return %result : vector<4xf32> +} + func.def inline @ggml_q5k_pair_load_decode_state(%weight: buffer, %row_base: offset, %block: index, %packet: index, %group0: index) -> (vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>, vector<1xi32>) { %c0 = index.constant 0 : index %c4 = index.constant 4 : index @@ -238,81 +300,97 @@ func.def inline @ggml_matmul_vector_decode_pair_block_row(%iq4nl_table: vector<1 } %pair_q5_format = index.constant 5 : index %pair_q5 = index.cmp eq, %weight_format, %pair_q5_format : index - %weight0, %weight1, %weight2, %weight3 = scf.if %pair_q5 -> (vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32>) { - %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload0, %payload2 = func.call @ggml_q5k_pair_load_decode_state(%weight, %row_base, %safe_block, %packet, %group0) : (buffer, offset, index, index, index) -> (vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>, vector<1xi32>) - %weight0 = scf.if %valid_k0 -> (vector<4xf32>) { - %result = func.call @ggml_q5k_pair_preloaded_f32_vector4(%group0, %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload0) : (index, vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>) -> (vector<4xf32>) - scf.yield %result : vector<4xf32> - } else { - scf.yield %zero : vector<4xf32> - } - %weight1 = scf.if %valid_k1 -> (vector<4xf32>) { - %result = func.call @ggml_q5k_pair_preloaded_f32_vector4(%group1, %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload0) : (index, vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>) -> (vector<4xf32>) - scf.yield %result : vector<4xf32> - } else { - scf.yield %zero : vector<4xf32> - } - %weight2 = scf.if %valid_k2 -> (vector<4xf32>) { - %result = func.call @ggml_q5k_pair_preloaded_f32_vector4(%group2, %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload2) : (index, vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>) -> (vector<4xf32>) - scf.yield %result : vector<4xf32> - } else { - scf.yield %zero : vector<4xf32> - } - %weight3 = scf.if %valid_k3 -> (vector<4xf32>) { - %result = func.call @ggml_q5k_pair_preloaded_f32_vector4(%group3, %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload2) : (index, vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>) -> (vector<4xf32>) - scf.yield %result : vector<4xf32> - } else { - scf.yield %zero : vector<4xf32> - } - scf.yield %weight0, %weight1, %weight2, %weight3 : vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32> + %q6_row64_format = index.constant 46 : index + %pair_q6 = index.cmp eq, %weight_format, %q6_row64_format : index + %weight0, %weight1, %weight2, %weight3 = scf.if %pair_q6 -> (vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32>) { + %d, %scales, %ql0, %ql1, %qh = func.call @ggml_q6k_row64_pair_load_decode_state(%weight, %row, %input_size, %safe_block, %packet, %vector_half) : (buffer, index, index, index, index, index) -> (f32, vector<4xi8>, vector<1xi32>, vector<1xi32>, vector<1xi32>) + %s0 = vector.extract %scales[0] : vector<4xi8> -> i8 + %w0 = func.call @ggml_q6k_row64_pair_preloaded_f32_vector4(%c0, %d, %s0, %ql0, %qh) : (index, f32, i8, vector<1xi32>, vector<1xi32>) -> (vector<4xf32>) + %s1 = vector.extract %scales[1] : vector<4xi8> -> i8 + %w1 = func.call @ggml_q6k_row64_pair_preloaded_f32_vector4(%c1, %d, %s1, %ql1, %qh) : (index, f32, i8, vector<1xi32>, vector<1xi32>) -> (vector<4xf32>) + %s2 = vector.extract %scales[2] : vector<4xi8> -> i8 + %w2 = func.call @ggml_q6k_row64_pair_preloaded_f32_vector4(%c2, %d, %s2, %ql0, %qh) : (index, f32, i8, vector<1xi32>, vector<1xi32>) -> (vector<4xf32>) + %s3 = vector.extract %scales[3] : vector<4xi8> -> i8 + %w3 = func.call @ggml_q6k_row64_pair_preloaded_f32_vector4(%c3, %d, %s3, %ql1, %qh) : (index, f32, i8, vector<1xi32>, vector<1xi32>) -> (vector<4xf32>) + scf.yield %w0, %w1, %w2, %w3 : vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32> } else { - %weight0 = scf.if %valid_k0 -> (vector<4xf32>) { - %decoded = scf.if %metadata_enabled -> (vector<4xf32>) { - %v = func.call @ggml_q1_0_preloaded_f32_vector4(%weight, %row_base, %k0, %cached0) : (buffer, offset, index, vector<1xf16>) -> (vector<4xf32>) - scf.yield %v : vector<4xf32> + %weight0, %weight1, %weight2, %weight3 = scf.if %pair_q5 -> (vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32>) { + %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload0, %payload2 = func.call @ggml_q5k_pair_load_decode_state(%weight, %row_base, %safe_block, %packet, %group0) : (buffer, offset, index, index, index) -> (vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>, vector<1xi32>) + %weight0 = scf.if %valid_k0 -> (vector<4xf32>) { + %result = func.call @ggml_q5k_pair_preloaded_f32_vector4(%group0, %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload0) : (index, vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>) -> (vector<4xf32>) + scf.yield %result : vector<4xf32> } else { - %fallback = func.call @ggml_dequant_f32_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight, %row_base, %row, %input_size, %safe_block, %group0, %packet, %k0) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index, index) -> (vector<4xf32>) - scf.yield %fallback : vector<4xf32> + scf.yield %zero : vector<4xf32> } - scf.yield %decoded : vector<4xf32> - } else { - scf.yield %zero : vector<4xf32> - } - %weight1 = scf.if %valid_k1 -> (vector<4xf32>) { - %decoded = scf.if %metadata_enabled -> (vector<4xf32>) { - %v = func.call @ggml_q1_0_preloaded_f32_vector4(%weight, %row_base, %k1, %cached0) : (buffer, offset, index, vector<1xf16>) -> (vector<4xf32>) - scf.yield %v : vector<4xf32> + %weight1 = scf.if %valid_k1 -> (vector<4xf32>) { + %result = func.call @ggml_q5k_pair_preloaded_f32_vector4(%group1, %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload0) : (index, vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>) -> (vector<4xf32>) + scf.yield %result : vector<4xf32> } else { - %fallback = func.call @ggml_dequant_f32_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight, %row_base, %row, %input_size, %safe_block, %group1, %packet, %k1) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index, index) -> (vector<4xf32>) - scf.yield %fallback : vector<4xf32> + scf.yield %zero : vector<4xf32> } - scf.yield %decoded : vector<4xf32> - } else { - scf.yield %zero : vector<4xf32> - } - %weight2 = scf.if %valid_k2 -> (vector<4xf32>) { - %decoded = scf.if %metadata_enabled -> (vector<4xf32>) { - %v = func.call @ggml_q1_0_preloaded_f32_vector4(%weight, %row_base, %k2, %cached0) : (buffer, offset, index, vector<1xf16>) -> (vector<4xf32>) - scf.yield %v : vector<4xf32> + %weight2 = scf.if %valid_k2 -> (vector<4xf32>) { + %result = func.call @ggml_q5k_pair_preloaded_f32_vector4(%group2, %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload2) : (index, vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>) -> (vector<4xf32>) + scf.yield %result : vector<4xf32> } else { - %fallback = func.call @ggml_dequant_f32_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight, %row_base, %row, %input_size, %safe_block, %group2, %packet, %k2) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index, index) -> (vector<4xf32>) - scf.yield %fallback : vector<4xf32> + scf.yield %zero : vector<4xf32> } - scf.yield %decoded : vector<4xf32> - } else { - scf.yield %zero : vector<4xf32> - } - %weight3 = scf.if %valid_k3 -> (vector<4xf32>) { - %decoded = scf.if %metadata_enabled -> (vector<4xf32>) { - %v = func.call @ggml_q1_0_preloaded_f32_vector4(%weight, %row_base, %k3, %cached0) : (buffer, offset, index, vector<1xf16>) -> (vector<4xf32>) - scf.yield %v : vector<4xf32> + %weight3 = scf.if %valid_k3 -> (vector<4xf32>) { + %result = func.call @ggml_q5k_pair_preloaded_f32_vector4(%group3, %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload2) : (index, vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>) -> (vector<4xf32>) + scf.yield %result : vector<4xf32> } else { - %fallback = func.call @ggml_dequant_f32_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight, %row_base, %row, %input_size, %safe_block, %group3, %packet, %k3) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index, index) -> (vector<4xf32>) - scf.yield %fallback : vector<4xf32> + scf.yield %zero : vector<4xf32> } - scf.yield %decoded : vector<4xf32> + scf.yield %weight0, %weight1, %weight2, %weight3 : vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32> } else { - scf.yield %zero : vector<4xf32> + %weight0 = scf.if %valid_k0 -> (vector<4xf32>) { + %decoded = scf.if %metadata_enabled -> (vector<4xf32>) { + %v = func.call @ggml_q1_0_preloaded_f32_vector4(%weight, %row_base, %k0, %cached0) : (buffer, offset, index, vector<1xf16>) -> (vector<4xf32>) + scf.yield %v : vector<4xf32> + } else { + %fallback = func.call @ggml_dequant_f32_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight, %row_base, %row, %input_size, %safe_block, %group0, %packet, %k0) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index, index) -> (vector<4xf32>) + scf.yield %fallback : vector<4xf32> + } + scf.yield %decoded : vector<4xf32> + } else { + scf.yield %zero : vector<4xf32> + } + %weight1 = scf.if %valid_k1 -> (vector<4xf32>) { + %decoded = scf.if %metadata_enabled -> (vector<4xf32>) { + %v = func.call @ggml_q1_0_preloaded_f32_vector4(%weight, %row_base, %k1, %cached0) : (buffer, offset, index, vector<1xf16>) -> (vector<4xf32>) + scf.yield %v : vector<4xf32> + } else { + %fallback = func.call @ggml_dequant_f32_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight, %row_base, %row, %input_size, %safe_block, %group1, %packet, %k1) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index, index) -> (vector<4xf32>) + scf.yield %fallback : vector<4xf32> + } + scf.yield %decoded : vector<4xf32> + } else { + scf.yield %zero : vector<4xf32> + } + %weight2 = scf.if %valid_k2 -> (vector<4xf32>) { + %decoded = scf.if %metadata_enabled -> (vector<4xf32>) { + %v = func.call @ggml_q1_0_preloaded_f32_vector4(%weight, %row_base, %k2, %cached0) : (buffer, offset, index, vector<1xf16>) -> (vector<4xf32>) + scf.yield %v : vector<4xf32> + } else { + %fallback = func.call @ggml_dequant_f32_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight, %row_base, %row, %input_size, %safe_block, %group2, %packet, %k2) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index, index) -> (vector<4xf32>) + scf.yield %fallback : vector<4xf32> + } + scf.yield %decoded : vector<4xf32> + } else { + scf.yield %zero : vector<4xf32> + } + %weight3 = scf.if %valid_k3 -> (vector<4xf32>) { + %decoded = scf.if %metadata_enabled -> (vector<4xf32>) { + %v = func.call @ggml_q1_0_preloaded_f32_vector4(%weight, %row_base, %k3, %cached0) : (buffer, offset, index, vector<1xf16>) -> (vector<4xf32>) + scf.yield %v : vector<4xf32> + } else { + %fallback = func.call @ggml_dequant_f32_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight, %row_base, %row, %input_size, %safe_block, %group3, %packet, %k3) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index, index) -> (vector<4xf32>) + scf.yield %fallback : vector<4xf32> + } + scf.yield %decoded : vector<4xf32> + } else { + scf.yield %zero : vector<4xf32> + } + scf.yield %weight0, %weight1, %weight2, %weight3 : vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32> } scf.yield %weight0, %weight1, %weight2, %weight3 : vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32> } diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_f32_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_f32_f32.loom index 0b869108f858..77f0f80fa291 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_f32_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_f32_f32.loom @@ -1,3 +1,5 @@ +template.decl @ggml.matmul.vector_f32.entry(%token_count: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) + // Vector-shaped GGML matmul reduction for batch-size-1 F32 outputs. template.decl @ggml.matmul.vector_publish.next_vector4(%arg0: i1, %arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index, %arg6: vector<4xf32>, %arg7: buffer) template.decl @ggml.matmul.vector_activation.core(%arg0: index, %arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index, %arg6: f32, %arg7: index, %arg8: index, %arg9: buffer, %arg10: buffer, %arg11: buffer, %arg12: buffer, %arg13: buffer, %arg14: buffer, %arg15: buffer, %arg16: buffer, %arg17: buffer, %arg18: buffer) @@ -102,15 +104,7 @@ template.def<@ggml.matmul.vector_activation.finish_output_group> device @ggml_ma template.return } -kernel.def target(@ggml_matmul_vector_gfx11_wave64) @ggml_mul_mat_vector_f32_f32(%token_count: index) { - %input_size = config.get @ggml.matmul.vector.input_size : index - %output_size = config.get @ggml.matmul.vector.output_size : index - %token_capacity = config.get @ggml.workload.token_capacity : index - %weight_format = config.get @ggml.matmul.vector.weight_format : index - %c1 = index.constant 1 : index - %launch_x, %launch_y, %launch_z, %workgroup_size = template.apply<@ggml.matmul.vector_activation.launch>(%token_capacity, %input_size, %output_size, %weight_format) pure : (index, index, index, index) -> (index, index, index, index) - kernel.launch.config workgroups(%launch_x, %launch_y, %launch_z) workgroup_size(%workgroup_size, %c1, %c1) : index -} launch(%token_count: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) { +template.def<@ggml.matmul.vector_f32.entry> device @ggml_matmul_vector_f32_entry(%token_count: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) { %input_size = config.get @ggml.matmul.vector.input_size : index %output_size = config.get @ggml.matmul.vector.output_size : index %token_capacity = config.get @ggml.workload.token_capacity : index @@ -122,5 +116,47 @@ kernel.def target(@ggml_matmul_vector_gfx11_wave64) @ggml_mul_mat_vector_f32_f32 %channel_group = kernel.workgroup.id : index %token_group = kernel.workgroup.id : index template.apply<@ggml.matmul.vector_activation.core>(%weight_format, %bounded_token_count, %input_size, %output_size, %output_accumulation, %output_unary_op, %c0_f32, %channel_group, %token_group, %input, %weight, %output, %output, %output, %output, %output, %output, %output, %next_output) : (index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) + template.return +} + +kernel.def target(@ggml_matmul_vector_gfx11_wave64) @ggml_mul_mat_vector_f32_f32(%token_count: index) { + %input_size = config.get @ggml.matmul.vector.input_size : index + %output_size = config.get @ggml.matmul.vector.output_size : index + %token_capacity = config.get @ggml.workload.token_capacity : index + %weight_format = config.get @ggml.matmul.vector.weight_format : index + %c1 = index.constant 1 : index + %launch_x, %launch_y, %launch_z, %workgroup_size = template.apply<@ggml.matmul.vector_activation.launch>(%token_capacity, %input_size, %output_size, %weight_format) pure : (index, index, index, index) -> (index, index, index, index) + kernel.launch.config workgroups(%launch_x, %launch_y, %launch_z) workgroup_size(%workgroup_size, %c1, %c1) : index +} launch(%token_count: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) { + template.apply<@ggml.matmul.vector_f32.entry>(%token_count, %input, %weight, %output, %next_output) : (index, buffer, buffer, buffer, buffer) + kernel.return +} + +// Native Q6/Q8 retains eight wave32 rows per workgroup on gfx1100. +amdgpu.target @ggml_matmul_vector_gfx11_wave32 {subgroup_size = 32} +kernel.def target(@ggml_matmul_vector_gfx11_wave32) export("ggml_linear_q6k_q8_1_x4") @ggml_linear_q6k_q8_1_x4(%token_count: index) { + %output_size = config.get @ggml.matmul.vector.output_size : index + %token_capacity = config.get @ggml.workload.token_capacity : index + %c1 = index.constant 1 : index + %c7 = index.constant 7 : index + %c8 = index.constant 8 : index + %threads = index.constant 256 : index + %rounded = index.add %output_size, %c7 : index + %groups = index.div %rounded, %c8 : index + kernel.launch.config workgroups(%groups, %token_capacity, %c1) workgroup_size(%threads, %c1, %c1) : index +} launch(%token_count: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) { + template.apply<@ggml.matmul.vector_f32.entry>(%token_count, %input, %weight, %output, %next_output) : (index, buffer, buffer, buffer, buffer) + kernel.return +} + +// The catalog export name is shared with wave32; both use the common vector ABI. +kernel.def target(@ggml_matmul_vector_gfx11_wave64) export("ggml_linear_q6k_q8_1_x4") @ggml_linear_q6k_q8_1_x4_wave64_block4(%token_count: index) { + %output_size = config.get @ggml.matmul.vector.output_size : index + %token_capacity = config.get @ggml.workload.token_capacity : index + %c1 = index.constant 1 : index + %wave = target.subgroup.size : index + kernel.launch.config workgroups(%output_size, %token_capacity, %c1) workgroup_size(%wave, %c1, %c1) : index +} launch(%token_count: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) { + template.apply<@ggml.matmul.vector_f32.entry>(%token_count, %input, %weight, %output, %next_output) : (index, buffer, buffer, buffer, buffer) kernel.return } diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_q6_q8_native_test.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_q6_q8_native_test.loom new file mode 100644 index 000000000000..b8ca3a7b0937 --- /dev/null +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_q6_q8_native_test.loom @@ -0,0 +1,64 @@ +// Nonzero native Q6/Q8 checks call both production entries with the shared ABI. +// K=768 covers the wave64 four-block tail; nine rows cross the wave32 row tile. + +target.decl @ggml_matmul_vector_gfx11_wave32 + +kernel.decl target(@ggml_matmul_vector_gfx11_wave32) @ggml_linear_q6k_q8_1_x4(%token_count$0: index) launch(%token_count$1: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) + +target.decl @ggml_matmul_vector_gfx11_wave64 + +kernel.decl target(@ggml_matmul_vector_gfx11_wave64) @ggml_linear_q6k_q8_1_x4_wave64_block4(%token_count$6: index) launch(%token_count$7: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) + +// Q6 bytes 0xaa decode to (42 - 32) * (-86) * f16(0xaaaa). +// Q8 bytes 0x01 decode to 1 * f16(0x0101); the oracle is K times their product. +check.case public @ggml_mul_mat_vector_q6_q8_wave32_k768_tail { + %token_count = check.literal value(1) : index + %input = check.generate.fill value(1) : tensor<1x864xi8> + %weight = check.generate.fill value(-86) : tensor<9x3x210xi8> + %output = check.generate.fill value(-1.0) : tensor<1x9xf32> + %expected = check.generate.fill value(0.52674690261483192) : tensor<1x9xf32> + kernel.launch @ggml_linear_q6k_q8_1_x4[%token_count](%token_count, %input, %weight, %output, %output) : [index](index, tensor<1x864xi8>, tensor<9x3x210xi8>, tensor<1x9xf32>, tensor<1x9xf32>) + check.expect.approximate actual(%output) expected(%expected) atol(9.9999999999999995e-07) rtol(9.9999999999999995e-07) nan(same) : tensor<1x9xf32>, tensor<1x9xf32> + check.return +} + +check.benchmark<@ggml_mul_mat_vector_q6_q8_wave32_k768_tail> @ggml_mul_mat_vector_q6_q8_wave32_k768_tail_small + +check.case public @ggml_mul_mat_vector_q6_q8_wave32_k2048_tail { + %token_count = check.literal value(1) : index + %input = check.generate.fill value(1) : tensor<1x2304xi8> + %weight = check.generate.fill value(-86) : tensor<9x8x210xi8> + %output = check.generate.fill value(-1.0) : tensor<1x9xf32> + %expected = check.generate.fill value(1.4046584069728851) : tensor<1x9xf32> + kernel.launch @ggml_linear_q6k_q8_1_x4[%token_count](%token_count, %input, %weight, %output, %output) : [index](index, tensor<1x2304xi8>, tensor<9x8x210xi8>, tensor<1x9xf32>, tensor<1x9xf32>) + check.expect.approximate actual(%output) expected(%expected) atol(9.9999999999999995e-07) rtol(9.9999999999999995e-07) nan(same) : tensor<1x9xf32>, tensor<1x9xf32> + check.return +} + +check.benchmark<@ggml_mul_mat_vector_q6_q8_wave32_k2048_tail> @ggml_mul_mat_vector_q6_q8_wave32_k2048_tail_small + +check.case public @ggml_mul_mat_vector_q6_q8_wave64_k768_tail { + %token_count = check.literal value(1) : index + %input = check.generate.fill value(1) : tensor<1x864xi8> + %weight = check.generate.fill value(-86) : tensor<9x3x210xi8> + %output = check.generate.fill value(-1.0) : tensor<1x9xf32> + %expected = check.generate.fill value(0.52674690261483192) : tensor<1x9xf32> + kernel.launch @ggml_linear_q6k_q8_1_x4_wave64_block4[%token_count](%token_count, %input, %weight, %output, %output) : [index](index, tensor<1x864xi8>, tensor<9x3x210xi8>, tensor<1x9xf32>, tensor<1x9xf32>) + check.expect.approximate actual(%output) expected(%expected) atol(9.9999999999999995e-07) rtol(9.9999999999999995e-07) nan(same) : tensor<1x9xf32>, tensor<1x9xf32> + check.return +} + +check.benchmark<@ggml_mul_mat_vector_q6_q8_wave64_k768_tail> @ggml_mul_mat_vector_q6_q8_wave64_k768_tail_small + +check.case public @ggml_mul_mat_vector_q6_q8_wave64_k2048_tail { + %token_count = check.literal value(1) : index + %input = check.generate.fill value(1) : tensor<1x2304xi8> + %weight = check.generate.fill value(-86) : tensor<9x8x210xi8> + %output = check.generate.fill value(-1.0) : tensor<1x9xf32> + %expected = check.generate.fill value(1.4046584069728851) : tensor<1x9xf32> + kernel.launch @ggml_linear_q6k_q8_1_x4_wave64_block4[%token_count](%token_count, %input, %weight, %output, %output) : [index](index, tensor<1x2304xi8>, tensor<9x8x210xi8>, tensor<1x9xf32>, tensor<1x9xf32>) + check.expect.approximate actual(%output) expected(%expected) atol(9.9999999999999995e-07) rtol(9.9999999999999995e-07) nan(same) : tensor<1x9xf32>, tensor<1x9xf32> + check.return +} + +check.benchmark<@ggml_mul_mat_vector_q6_q8_wave64_k2048_tail> @ggml_mul_mat_vector_q6_q8_wave64_k2048_tail_small diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom index 36ca4f94b66f..ede8a3d08561 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom @@ -28,7 +28,7 @@ template.decl @ggml.mul_mat_q6_k_i8_prepacked.body(%token_count: index, %input_s template.decl @ggml.mul_mat_q6_k_packed.selected_refine.body(%token_count: index, %input_size0: index, %output_size0: index, %scale_row_layout: index, %candidate_count0: index, %input: buffer, %weight: buffer, %candidates: buffer, %output: buffer) -template.decl @ggml.mul_mat_q6_k_packed.prefill_wave32.body(%input_is_f16: i1, %token_count: index, %input_size0: index, %output_size0: index, %channel_tile: index, %token_tile: index, %input: buffer, %weight: buffer, %output: buffer) +template.decl @ggml.mul_mat_q6_k_packed.prefill_wave32.body(%input_is_f16: i1, %single_half: i1, %token_count: index, %input_size0: index, %output_size0: index, %channel_tile: index, %token_tile: index, %input: buffer, %weight: buffer, %output: buffer) amdgpu.target @ggml_mul_mat_q6_k_prefill_gfx11_wave32 {subgroup_size = 32} @@ -1280,7 +1280,7 @@ template.def<@ggml.mul_mat_q6_k_packed.publish_f32_wave32> device @ggml_mul_mat_ template.return } -template.def<@ggml.mul_mat_q6_k_packed.prefill_wave32.body> device @ggml_mul_mat_q6_k_packed_prefill_wave32_body(%input_is_f16: i1, %token_count: index, %input_size0: index, %output_size0: index, %channel_tile: index, %token_tile: index, %input: buffer, %weight: buffer, %output: buffer) { +template.def<@ggml.mul_mat_q6_k_packed.prefill_wave32.body> device @ggml_mul_mat_q6_k_packed_prefill_wave32_body(%input_is_f16: i1, %single_half: i1, %token_count: index, %input_size0: index, %output_size0: index, %channel_tile: index, %token_tile: index, %input: buffer, %weight: buffer, %output: buffer) { %output_accumulation = config.get @ggml.mul_mat_q6_k_packed.output_accumulation : index %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048)] : index %bounded_input_size = index.assume %input_size0 [range(%input_size0, 256, 32768), multiple_of(%input_size0, 256)] : index @@ -1323,7 +1323,8 @@ template.def<@ggml.mul_mat_q6_k_packed.prefill_wave32.body> device @ggml_mul_mat %activation_fragment_layout = encoding.layout.strided [1, %c72] : encoding %activation_fragment_view = buffer.view %activation_stage[%c0_offset] : buffer -> view<64x64xf16, %activation_fragment_layout> %channel_tile_base = index.mul %channel_tile, %c128 : index - %token_tile_base = index.mul %token_tile, %c128 : index + %token_tile_width = scf.select %single_half, %c64, %c128 : index + %token_tile_base = index.mul %token_tile, %token_tile_width : index %second_token_tile_base = index.add %token_tile_base, %c64 : index %load_half = index.rem %workitem, %c2 : index %load_packet = index.mul %load_half, %c4 : index @@ -1415,44 +1416,49 @@ template.def<@ggml.mul_mat_q6_k_packed.prefill_wave32.body> device @ggml_mul_mat scf.yield %half_next00, %half_next01, %half_next02, %half_next03 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> } kernel.barrier scope(workgroup) ordering(acq_rel) - scf.for %stage_half1 = [%c0 to %c2 step %c1] unroll { - %stage_k_base1 = index.mul %stage_half1, %c32 : index - %token1 = index.add %second_token_tile_base, %activation_load_row : index - %valid_token1 = index.cmp ult, %token1, %bounded_token_count : index - %activation_k1 = index.add %stage_k_base1, %activation_load_k : index - %activation_values1 = scf.if %valid_token1 -> (vector<8xf16>) { - %bounded_token1, %input_token_count1 = index.assume %token1, %bounded_token_count [lt(%token1, %bounded_token_count)] : index, index - %input_k1 = index.add %k_origin, %activation_k1 : index - %loaded1 = scf.if %input_is_f16 -> (vector<8xf16>) { - %narrow1 = vector.load %input_f16_view[%bounded_token1, %input_k1] : view<[%bounded_token_count]x[%bounded_input_size]xf16> -> vector<8xf16> - scf.yield %narrow1 : vector<8xf16> + %next10, %next11, %next12, %next13 = scf.if %single_half -> (vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>) { + scf.yield %acc10, %acc11, %acc12, %acc13 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> + } else { + scf.for %stage_half1 = [%c0 to %c2 step %c1] unroll { + %stage_k_base1 = index.mul %stage_half1, %c32 : index + %token1 = index.add %second_token_tile_base, %activation_load_row : index + %valid_token1 = index.cmp ult, %token1, %bounded_token_count : index + %activation_k1 = index.add %stage_k_base1, %activation_load_k : index + %activation_values1 = scf.if %valid_token1 -> (vector<8xf16>) { + %bounded_token1, %input_token_count1 = index.assume %token1, %bounded_token_count [lt(%token1, %bounded_token_count)] : index, index + %input_k1 = index.add %k_origin, %activation_k1 : index + %loaded1 = scf.if %input_is_f16 -> (vector<8xf16>) { + %narrow1 = vector.load %input_f16_view[%bounded_token1, %input_k1] : view<[%bounded_token_count]x[%bounded_input_size]xf16> -> vector<8xf16> + scf.yield %narrow1 : vector<8xf16> + } else { + %wide1 = vector.load %input_f32_view[%bounded_token1, %input_k1] : view<[%bounded_token_count]x[%bounded_input_size]xf32> -> vector<8xf32> + %narrow1 = vector.fptrunc %wide1 : vector<8xf32> to vector<8xf16> + scf.yield %narrow1 : vector<8xf16> + } + scf.yield %loaded1 : vector<8xf16> } else { - %wide1 = vector.load %input_f32_view[%bounded_token1, %input_k1] : view<[%bounded_token_count]x[%bounded_input_size]xf32> -> vector<8xf32> - %narrow1 = vector.fptrunc %wide1 : vector<8xf32> to vector<8xf16> - scf.yield %narrow1 : vector<8xf16> + scf.yield %c0_f16x8 : vector<8xf16> } - scf.yield %loaded1 : vector<8xf16> - } else { - scf.yield %c0_f16x8 : vector<8xf16> + vector.store %activation_values1, %activation_stage_physical_view[%activation_load_row, %activation_k1] : vector<8xf16>, view<64x72xf16> } - vector.store %activation_values1, %activation_stage_physical_view[%activation_load_row, %activation_k1] : vector<8xf16>, view<64x72xf16> - } - kernel.barrier scope(workgroup) ordering(acq_rel) - %next10, %next11, %next12, %next13 = scf.for %k_half1 = [%c0 to %c64 step %c16](%half_acc10 = %acc10 : vector<8xf32>, %half_acc11 = %acc11 : vector<8xf32>, %half_acc12 = %acc12 : vector<8xf32>, %half_acc13 = %acc13 : vector<8xf32>) -> (vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>) unroll { - %lhs1 = vector.fragment.load %weight_stage_view[%subgroup_channel_add, %k_half1] shape [%m, %k] : view<128x72xf16> -> vector<16xf16> - %rhs10 = vector.fragment.load %activation_fragment_view[%k_half1, %c0] shape [%k, %n] : view<64x64xf16, %activation_fragment_layout> -> vector<16xf16> - %rhs11 = vector.fragment.load %activation_fragment_view[%k_half1, %c16] shape [%k, %n] : view<64x64xf16, %activation_fragment_layout> -> vector<16xf16> - %half_next10 = vector.mma %lhs1, %rhs10, %half_acc10 : vector<16xf16>, vector<16xf16>, vector<8xf32> - %half_next11 = vector.mma %lhs1, %rhs11, %half_acc11 : vector<16xf16>, vector<16xf16>, vector<8xf32> - scf.schedule.fence - %rhs12 = vector.fragment.load %activation_fragment_view[%k_half1, %c32] shape [%k, %n] : view<64x64xf16, %activation_fragment_layout> -> vector<16xf16> - %rhs13 = vector.fragment.load %activation_fragment_view[%k_half1, %c48] shape [%k, %n] : view<64x64xf16, %activation_fragment_layout> -> vector<16xf16> - %half_next12 = vector.mma %lhs1, %rhs12, %half_acc12 : vector<16xf16>, vector<16xf16>, vector<8xf32> - %half_next13 = vector.mma %lhs1, %rhs13, %half_acc13 : vector<16xf16>, vector<16xf16>, vector<8xf32> - scf.schedule.fence - scf.yield %half_next10, %half_next11, %half_next12, %half_next13 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> + kernel.barrier scope(workgroup) ordering(acq_rel) + %second10, %second11, %second12, %second13 = scf.for %k_half1 = [%c0 to %c64 step %c16](%half_acc10 = %acc10 : vector<8xf32>, %half_acc11 = %acc11 : vector<8xf32>, %half_acc12 = %acc12 : vector<8xf32>, %half_acc13 = %acc13 : vector<8xf32>) -> (vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>) unroll { + %lhs1 = vector.fragment.load %weight_stage_view[%subgroup_channel_add, %k_half1] shape [%m, %k] : view<128x72xf16> -> vector<16xf16> + %rhs10 = vector.fragment.load %activation_fragment_view[%k_half1, %c0] shape [%k, %n] : view<64x64xf16, %activation_fragment_layout> -> vector<16xf16> + %rhs11 = vector.fragment.load %activation_fragment_view[%k_half1, %c16] shape [%k, %n] : view<64x64xf16, %activation_fragment_layout> -> vector<16xf16> + %half_next10 = vector.mma %lhs1, %rhs10, %half_acc10 : vector<16xf16>, vector<16xf16>, vector<8xf32> + %half_next11 = vector.mma %lhs1, %rhs11, %half_acc11 : vector<16xf16>, vector<16xf16>, vector<8xf32> + scf.schedule.fence + %rhs12 = vector.fragment.load %activation_fragment_view[%k_half1, %c32] shape [%k, %n] : view<64x64xf16, %activation_fragment_layout> -> vector<16xf16> + %rhs13 = vector.fragment.load %activation_fragment_view[%k_half1, %c48] shape [%k, %n] : view<64x64xf16, %activation_fragment_layout> -> vector<16xf16> + %half_next12 = vector.mma %lhs1, %rhs12, %half_acc12 : vector<16xf16>, vector<16xf16>, vector<8xf32> + %half_next13 = vector.mma %lhs1, %rhs13, %half_acc13 : vector<16xf16>, vector<16xf16>, vector<8xf32> + scf.schedule.fence + scf.yield %half_next10, %half_next11, %half_next12, %half_next13 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> + } + kernel.barrier scope(workgroup) ordering(acq_rel) + scf.yield %second10, %second11, %second12, %second13 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> } - kernel.barrier scope(workgroup) ordering(acq_rel) scf.yield %next00, %next01, %next02, %next03, %next10, %next11, %next12, %next13 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> } scf.yield %block_result00, %block_result01, %block_result02, %block_result03, %block_result10, %block_result11, %block_result12, %block_result13 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> @@ -1461,10 +1467,14 @@ template.def<@ggml.mul_mat_q6_k_packed.prefill_wave32.body> device @ggml_mul_mat template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result01, %c16, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result02, %c32, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result03, %c48, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) - template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result10, %c64, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) - template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result11, %c80, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) - template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result12, %c96, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) - template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result13, %c112, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) + %true = scalar.constant true : i1 + %double_half = scalar.xori %single_half, %true : i1 + scf.if %double_half { + template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result10, %c64, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) + template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result11, %c80, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) + template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result12, %c96, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) + template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result13, %c112, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) + } template.return } @@ -1490,7 +1500,36 @@ kernel.def target(@ggml_mul_mat_q6_k_prefill_gfx11_wave32) export("ggml_mul_mat_ %channel_tile = kernel.workgroup.id : index %token_tile = kernel.workgroup.id : index %input_is_f16 = scalar.constant false : i1 - template.apply<@ggml.mul_mat_q6_k_packed.prefill_wave32.body>(%input_is_f16, %bounded_token_count, %input_size, %output_size, %channel_tile, %token_tile, %input, %weight, %output) : (i1, index, index, index, index, index, buffer, buffer, buffer) + %single_half = scalar.constant false : i1 + template.apply<@ggml.mul_mat_q6_k_packed.prefill_wave32.body>(%input_is_f16, %single_half, %bounded_token_count, %input_size, %output_size, %channel_tile, %token_tile, %input, %weight, %output) : (i1, i1, index, index, index, index, index, buffer, buffer, buffer) + kernel.return +} + +// Use one M64 half per workgroup for short F32 prompts. +kernel.def target(@ggml_mul_mat_q6_k_prefill_gfx11_wave32) export("ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32") @ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32(%token_count: index) { + %output_size = config.get @ggml.mul_mat_q6_k_packed.output_size : index + %token_capacity = config.get @ggml.mul_mat_q6_k_packed.token_capacity : index + %c1 = index.constant 1 : index + %c127 = index.constant 127 : index + %c64 = index.constant 64 : index + %c128 = index.constant 128 : index + %c256 = index.constant 256 : index + %padded_output_size = index.add %output_size, %c127 : index + %output_tiles = index.div %padded_output_size, %c128 : index + %c63 = index.constant 63 : index + %padded_token_count = index.add %token_capacity, %c63 : index + %token_tiles = index.div %padded_token_count, %c64 : index + kernel.launch.config workgroups(%token_tiles, %output_tiles, %c1) workgroup_size(%c256, %c1, %c1) : index +} launch(%token_count: index, %input: buffer, %weight: buffer, %output: buffer) { + %input_size = config.get @ggml.mul_mat_q6_k_packed.input_size : index + %output_size = config.get @ggml.mul_mat_q6_k_packed.output_size : index + %token_capacity = config.get @ggml.mul_mat_q6_k_packed.token_capacity : index + %bounded_token_count = index.assume %token_count [range(%token_count, 64, 2048), le(%token_count, %token_capacity)] : index + %channel_tile = kernel.workgroup.id : index + %token_tile = kernel.workgroup.id : index + %input_is_f16 = scalar.constant false : i1 + %single_half = scalar.constant true : i1 + template.apply<@ggml.mul_mat_q6_k_packed.prefill_wave32.body>(%input_is_f16, %single_half, %bounded_token_count, %input_size, %output_size, %channel_tile, %token_tile, %input, %weight, %output) : (i1, i1, index, index, index, index, index, buffer, buffer, buffer) kernel.return } @@ -1586,7 +1625,8 @@ kernel.def target(@ggml_mul_mat_q6_k_prefill_gfx11_wave32) export("ggml_mul_mat_ func.call @ggml_mul_mat_quantized_f16_prefill_wave32(%weight_format, %prefill_binary_op, %paired, %packed_input, %paired, %bounded_token_count, %input_size, %output_size, %channel_tile, %token_tile, %input, %weight, %weight, %output, %output) : (index, index, i1, i1, i1, index, index, index, index, index, buffer, buffer, buffer, buffer, buffer) } else { %input_is_f16 = scalar.constant true : i1 - template.apply<@ggml.mul_mat_q6_k_packed.prefill_wave32.body>(%input_is_f16, %bounded_token_count, %input_size, %output_size, %channel_tile, %token_tile, %input, %weight, %output) : (i1, index, index, index, index, index, buffer, buffer, buffer) + %single_half = scalar.constant false : i1 + template.apply<@ggml.mul_mat_q6_k_packed.prefill_wave32.body>(%input_is_f16, %single_half, %bounded_token_count, %input_size, %output_size, %channel_tile, %token_tile, %input, %weight, %output) : (i1, i1, index, index, index, index, index, buffer, buffer, buffer) } kernel.return } diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom deleted file mode 100644 index 55da51a6161a..000000000000 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom +++ /dev/null @@ -1,242 +0,0 @@ -// Contracts GGML Q6_K weight rows directly with the Q8_1 x4 activation -// layout used by llama.cpp's Vulkan matrix path. One 210-byte Q6_K block -// represents 256 signed six-bit values: -// -// i8 ql[128]; // low four bits -// i8 qh[64]; // high two bits -// i8 scales[16]; // signed scale for each 16-value group -// f16 d; // block-wide scale -// -// A wave owns one output value. Each lane contracts eight values from every -// Q6_K block as two native signed dot4 packets, and the wave reduces those -// partials. The kernel is the raw-layout correctness and decode baseline shared -// by dense and routed projections; prefill schedules can reuse the inline -// packed-row primitive while staging weights across multiple activation rows. -// The shared packer is linked beside this module. -template.decl @ggml.linear_q6k_q8_1_x4.body(%publish_output: i1, %token_count: index, %token0: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer) -> (f32, index, index, i1, index, index) - -amdgpu.target @ggml_q6k_q8_gfx1151_wave64 {subgroup_size = 64} - -config.decl @ggml.linear_q6k_q8_1_x4.token_capacity : %value: index where [range(%value, 1, 2048)] - -config.decl @ggml.linear_q6k_q8_1_x4.output_capacity : %value: index where [range(%value, 1, 262144)] - -kernel.decl @ggml_quantize_q8_1_x4_f32(%token_count$15: index, %input_size$16: index) launch(%token_count$17: index, %input_size$18: index, %input: buffer, %output: buffer) - -func.decl @ggml_q8_1_x4_word(%q8_input: buffer, %row_byte_base: offset, %q8_block: index, %word_in_block: index) -> (vector<4xi8>, f32) - -func.decl @ggml_q6k_q8_1_x4_row_lane_wave64_block4(%input_size: index, %weight: buffer, %weight_row_byte_base: offset, %q8_input: buffer, %q8_row_byte_base: offset, %lane: index) -> (f32) - -// Dense raw-layout baseline. A 256-thread workgroup carries eight independent -// wave32 output rows so decode launches enough waves without duplicating Q6_K -// decoding within a row. -kernel.def export("ggml_linear_q6k_q8_1_x4") @ggml_linear_q6k_q8_1_x4(%token_count: index, %input_size: index, %output_size: index) { - %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index - %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index - %c8 = index.constant 8 : index - %c7 = index.constant 7 : index - %c1 = index.constant 1 : index - %workgroup_size = index.constant 256 : index - %padded_output_size = index.add %output_capacity, %c7 : index - %output_tiles = index.div %padded_output_size, %c8 : index - kernel.launch.config workgroups(%output_tiles, %token_capacity, %c1) workgroup_size(%workgroup_size, %c1, %c1) : index -} launch(%token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) { - %q8_noalias, %weight_noalias, %output_noalias = buffer.assume.noalias %q8_input, %weight, %output : buffer, buffer, buffer - %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index - %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index - %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048), le(%token_count, %token_capacity)] : index - %bounded_output_size = index.assume %output_size [range(%output_size, 1, 262144), le(%output_size, %output_capacity)] : index - %c0 = index.constant 0 : index - %token0 = kernel.workgroup.id : index - %valid_token = index.cmp ult, %token0, %bounded_token_count : index - %safe_token = scf.select %valid_token, %token0, %c0 : index - %dot, %token, %channel, %valid_channel, %launch_token_count, %output_bound = template.apply<@ggml.linear_q6k_q8_1_x4.body>(%valid_token, %bounded_token_count, %safe_token, %input_size, %bounded_output_size, %q8_noalias, %weight_noalias) : (i1, index, index, index, index, buffer, buffer) -> (f32, index, index, i1, index, index) - %c0_i32 = scalar.constant 0 : i32 - %c0_offset = index.constant 0 : offset - %lane = kernel.subgroup.lane.id : index - %lane_i32 = index.cast %lane : index to i32 - %is_lane_zero = scalar.cmpi eq, %lane_i32, %c0_i32 : i32 - scf.if %valid_token { - scf.if %valid_channel { - scf.if %is_lane_zero { - %output_view = buffer.view %output_noalias[%c0_offset] : buffer -> view<[%launch_token_count]x[%output_bound]xf32> - view.store %dot, %output_view[%token, %channel] : f32, view<[%launch_token_count]x[%output_bound]xf32> - } - } - } - kernel.return -} - -// Wave64 vocabulary schedule matching llama.cpp Vulkan's one-row workgroup and -// four-block concurrency on AMD targets with native wave64 execution. -kernel.def target(@ggml_q6k_q8_gfx1151_wave64) export("ggml_linear_q6k_q8_1_x4") @ggml_linear_q6k_q8_1_x4_wave64_block4(%token_count: index, %input_size: index, %output_size: index) { - %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index - %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index - %c1 = index.constant 1 : index - %subgroup_size = target.subgroup.size : index - kernel.launch.config workgroups(%output_capacity, %token_capacity, %c1) workgroup_size(%subgroup_size, %c1, %c1) : index -} launch(%token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) { - %q8_noalias, %weight_noalias, %output_noalias = buffer.assume.noalias %q8_input, %weight, %output : buffer, buffer, buffer - %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index - %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index - %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048), le(%token_count, %token_capacity)] : index - %bounded_input_size = index.assume %input_size [range(%input_size, 256, 32768), multiple_of(%input_size, 256)] : index - %bounded_output_size = index.assume %output_size [range(%output_size, 1, 262144), le(%output_size, %output_capacity)] : index - %token = kernel.workgroup.id : index - %channel = kernel.workgroup.id : index - %lane = kernel.subgroup.lane.id : index - %valid_token = index.cmp ult, %token, %bounded_token_count : index - %valid_channel = index.cmp ult, %channel, %bounded_output_size : index - %publish_output = scalar.andi %valid_token, %valid_channel : i1 - %c0 = index.constant 0 : index - %c128 = index.constant 128 : index - %c144_bytes = index.constant 144 : offset - %c210_bytes = index.constant 210 : offset - %c256 = index.constant 256 : index - %safe_token = scf.select %valid_token, %token, %c0 : index - %safe_channel = scf.select %valid_channel, %channel, %c0 : index - %q6_block_count = index.div %bounded_input_size, %c256 : index - %weight_row_bytes = index.scale %q6_block_count, %c210_bytes : index, offset -> offset - %weight_row_byte_base = index.scale %safe_channel, %weight_row_bytes : index, offset -> offset - %q8_group_count = index.div %bounded_input_size, %c128 : index - %q8_row_bytes = index.scale %q8_group_count, %c144_bytes : index, offset -> offset - %q8_row_byte_base = index.scale %safe_token, %q8_row_bytes : index, offset -> offset - %lane_acc = scf.if %publish_output -> (f32) { - %value = func.call @ggml_q6k_q8_1_x4_row_lane_wave64_block4(%bounded_input_size, %weight_noalias, %weight_row_byte_base, %q8_noalias, %q8_row_byte_base, %lane) : (index, buffer, offset, buffer, offset, index) -> (f32) - scf.yield %value : f32 - } else { - %c0_f32 = scalar.constant 0.0 : f32 - scf.yield %c0_f32 : f32 - } - %dot = kernel.subgroup.reduce %lane_acc : f32 - %is_lane_zero = index.cmp eq, %lane, %c0 : index - scf.if %publish_output { - scf.if %is_lane_zero { - %c0_offset = index.constant 0 : offset - %output_view = buffer.view %output_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]x[%bounded_output_size]xf32> - view.store %dot, %output_view[%token, %channel] : f32, view<[%bounded_token_count]x[%bounded_output_size]xf32> - } - } - kernel.return -} - -// Uniform packed bytes exercise both low/high Q6 fields and signed per-group -// scales. The output width crosses the eight-wave tile boundary, while K=2048 -// covers the exact mixed-format dense V contraction depth. -check.case public @ggml_linear_q6k_q8_1_x4_nonzero_tail_case { - %token_count = check.literal value(2) : index - %input_size = check.literal value(2048) : index - %output_size = check.literal value(9) : index - %input = check.generate.fill value(0.00390625) : tensor<2x2048xf32> - %q8_input = check.generate.fill value(0) : tensor<2x2304xi8> - %weight = check.generate.fill value(-86) : tensor<9x8x210xi8> - %output = check.generate.fill value(0.0) : tensor<2x9xf32> - %expected = check.generate.fill value(358.1715087890625) : tensor<2x9xf32> - kernel.launch @ggml_quantize_q8_1_x4_f32[%token_count, %input_size](%token_count, %input_size, %input, %q8_input) : [index, index](index, index, tensor<2x2048xf32>, tensor<2x2304xi8>) - kernel.launch @ggml_linear_q6k_q8_1_x4[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<2x2304xi8>, tensor<9x8x210xi8>, tensor<2x9xf32>) - check.expect.approximate actual(%output) expected(%expected) atol(0.25) rtol(9.9999999999999995e-07) nan(same) : tensor<2x9xf32>, tensor<2x9xf32> - check.return -} - -// The gfx1151 schedule has an exact target requirement, so its nonzero packed -// data case remains independently selectable from the generic gfx11 case. -check.case public @ggml_linear_q6k_q8_1_x4_wave64_block4_nonzero_tail_case { - %token_count = check.literal value(2) : index - %input_size = check.literal value(2048) : index - %output_size = check.literal value(9) : index - %input = check.generate.fill value(0.00390625) : tensor<2x2048xf32> - %q8_input = check.generate.fill value(0) : tensor<2x2304xi8> - %weight = check.generate.fill value(-86) : tensor<9x8x210xi8> - %output = check.generate.fill value(0.0) : tensor<2x9xf32> - %expected = check.generate.fill value(358.1715087890625) : tensor<2x9xf32> - kernel.launch @ggml_quantize_q8_1_x4_f32[%token_count, %input_size](%token_count, %input_size, %input, %q8_input) : [index, index](index, index, tensor<2x2048xf32>, tensor<2x2304xi8>) - kernel.launch @ggml_linear_q6k_q8_1_x4_wave64_block4[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<2x2304xi8>, tensor<9x8x210xi8>, tensor<2x9xf32>) - check.expect.approximate actual(%output) expected(%expected) atol(0.25) rtol(9.9999999999999995e-07) nan(same) : tensor<2x9xf32>, tensor<2x9xf32> - check.return -} - -check.case public @ggml_linear_q6k_q8_1_x4_benchmark_case { - %token_count = check.param.choice values([1, 17, 32, 63, 128, 129, 512]) name("token_count") : index - %input_size = check.literal value(768) : index - %output_size = check.literal value(2048) : index - %q8_input = check.generate.fill value(0) : tensor<[%token_count]x864xi8> - %weight = check.generate.fill value(0) : tensor<2048x3x210xi8> - %output = check.generate.fill value(1.0) : tensor<[%token_count]x2048xf32> - %expected = check.generate.fill value(0.0) : tensor<[%token_count]x2048xf32> - kernel.launch @ggml_linear_q6k_q8_1_x4[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<[%token_count]x864xi8>, tensor<2048x3x210xi8>, tensor<[%token_count]x2048xf32>) - check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<[%token_count]x2048xf32>, tensor<[%token_count]x2048xf32> - check.return -} - -// Production vocabulary shape used to compare the eight-row wave32 baseline -// with the one-row wave64 block schedule without model-runtime overhead. -check.case public @ggml_linear_q6k_q8_1_x4_vocabulary_wave32_benchmark_case { - %token_count = check.literal value(1) : index - %input_size = check.literal value(2048) : index - %output_size = check.literal value(151936) : index - %q8_input = check.generate.fill value(1) : tensor<1x2304xi8> - %weight = check.generate.fill value(0) : tensor<151936x8x210xi8> - %output = check.generate.fill value(1.0) : tensor<1x151936xf32> - %expected = check.generate.fill value(0.0) : tensor<1x151936xf32> - kernel.launch @ggml_linear_q6k_q8_1_x4[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<1x2304xi8>, tensor<151936x8x210xi8>, tensor<1x151936xf32>) - check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<1x151936xf32>, tensor<1x151936xf32> - check.return -} - -check.case public @ggml_linear_q6k_q8_1_x4_vocabulary_wave64_block4_benchmark_case { - %token_count = check.literal value(1) : index - %input_size = check.literal value(2048) : index - %output_size = check.literal value(151936) : index - %q8_input = check.generate.fill value(1) : tensor<1x2304xi8> - %weight = check.generate.fill value(0) : tensor<151936x8x210xi8> - %output = check.generate.fill value(1.0) : tensor<1x151936xf32> - %expected = check.generate.fill value(0.0) : tensor<1x151936xf32> - kernel.launch @ggml_linear_q6k_q8_1_x4_wave64_block4[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<1x2304xi8>, tensor<151936x8x210xi8>, tensor<1x151936xf32>) - check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<1x151936xf32>, tensor<1x151936xf32> - check.return -} - -// Exercises the complete activation-pack and dense Q6_K projection boundary at -// the K=2048, M=512 shape used by mixed-format attention V weights. The token -// buckets cover decode, awkward prefill tails, llama.cpp's 512-token -// microbatch, and larger schedules available to callers without host routing. -check.case public @ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case { - %token_count = check.param.choice values([1, 17, 32, 63, 128, 129, 512, 1024, 2048]) name("token_count") : index - %input_size = check.literal value(2048) : index - %output_size = check.literal value(512) : index - %input = check.generate.fill value(0.0) : tensor<[%token_count]x2048xf32> - %q8_input = check.generate.fill value(1) : tensor<[%token_count]x2304xi8> - %weight = check.generate.fill value(0) : tensor<512x8x210xi8> - %output = check.generate.fill value(1.0) : tensor<[%token_count]x512xf32> - %expected = check.generate.fill value(0.0) : tensor<[%token_count]x512xf32> - kernel.launch @ggml_quantize_q8_1_x4_f32[%token_count, %input_size](%token_count, %input_size, %input, %q8_input) : [index, index](index, index, tensor<[%token_count]x2048xf32>, tensor<[%token_count]x2304xi8>) - kernel.launch @ggml_linear_q6k_q8_1_x4[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<[%token_count]x2304xi8>, tensor<512x8x210xi8>, tensor<[%token_count]x512xf32>) - check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<[%token_count]x512xf32>, tensor<[%token_count]x512xf32> - check.return -} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_nonzero_tail_case> @ggml_linear_q6k_q8_1_x4_small - -check.benchmark<@ggml_linear_q6k_q8_1_x4_benchmark_case> @ggml_linear_q6k_q8_1_x4_decode {token_count = 1} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_vocabulary_wave32_benchmark_case> @ggml_linear_q6k_q8_1_x4_vocabulary_wave32 - -check.benchmark<@ggml_linear_q6k_q8_1_x4_vocabulary_wave64_block4_benchmark_case> @ggml_linear_q6k_q8_1_x4_vocabulary_wave64_block4 - -check.benchmark<@ggml_linear_q6k_q8_1_x4_benchmark_case> @ggml_linear_q6k_q8_1_x4_prefill_32 {token_count = 32} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_benchmark_case> @ggml_linear_q6k_q8_1_x4_prefill_128 {token_count = 128} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_benchmark_case> @ggml_linear_q6k_q8_1_x4_prefill_512 {token_count = 512} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6k_q8_1_x4_dense_v_decode {token_count = 1} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6k_q8_1_x4_dense_v_prefill_32 {token_count = 32} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6k_q8_1_x4_dense_v_prefill_128 {token_count = 128} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6k_q8_1_x4_dense_v_prefill_512 {token_count = 512} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6k_q8_1_x4_dense_v_prefill_1024 {token_count = 1024} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6k_q8_1_x4_dense_v_prefill_2048 {token_count = 2048} diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/q6_q8_reference_test.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/q6_q8_reference_test.loom new file mode 100644 index 000000000000..5b128c3ff948 --- /dev/null +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/q6_q8_reference_test.loom @@ -0,0 +1,70 @@ +// Test-only native reference for packed Q6/Q8 and multi-token QKV checks. +template.decl @ggml.linear_q6k_q8_1_x4.body(%publish_output: i1, %token_count: index, %token0: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer) -> (f32, index, index, i1, index, index) + +config.decl @ggml.linear_q6k_q8_1_x4.token_capacity : %value: index where [range(%value, 1, 2048)] + +config.decl @ggml.linear_q6k_q8_1_x4.output_capacity : %value: index where [range(%value, 1, 262144)] + +kernel.decl @ggml_quantize_q8_1_x4_f32(%token_count$15: index, %input_size$16: index) launch(%token_count$17: index, %input_size$18: index, %input: buffer, %output: buffer) + +func.decl @ggml_q8_1_x4_word(%q8_input: buffer, %row_byte_base: offset, %q8_block: index, %word_in_block: index) -> (vector<4xi8>, f32) + +// Dense raw-layout baseline. A 256-thread workgroup carries eight independent +// wave32 output rows so decode launches enough waves without duplicating Q6_K +// decoding within a row. +kernel.def @ggml_linear_q6k_q8_1_x4(%token_count: index, %input_size: index, %output_size: index) { + %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index + %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index + %c8 = index.constant 8 : index + %c7 = index.constant 7 : index + %c1 = index.constant 1 : index + %workgroup_size = index.constant 256 : index + %padded_output_size = index.add %output_capacity, %c7 : index + %output_tiles = index.div %padded_output_size, %c8 : index + kernel.launch.config workgroups(%output_tiles, %token_capacity, %c1) workgroup_size(%workgroup_size, %c1, %c1) : index +} launch(%token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) { + %q8_noalias, %weight_noalias, %output_noalias = buffer.assume.noalias %q8_input, %weight, %output : buffer, buffer, buffer + %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index + %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index + %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048), le(%token_count, %token_capacity)] : index + %bounded_output_size = index.assume %output_size [range(%output_size, 1, 262144), le(%output_size, %output_capacity)] : index + %c0 = index.constant 0 : index + %token0 = kernel.workgroup.id : index + %valid_token = index.cmp ult, %token0, %bounded_token_count : index + %safe_token = scf.select %valid_token, %token0, %c0 : index + %dot, %token, %channel, %valid_channel, %launch_token_count, %output_bound = template.apply<@ggml.linear_q6k_q8_1_x4.body>(%valid_token, %bounded_token_count, %safe_token, %input_size, %bounded_output_size, %q8_noalias, %weight_noalias) : (i1, index, index, index, index, buffer, buffer) -> (f32, index, index, i1, index, index) + %c0_i32 = scalar.constant 0 : i32 + %c0_offset = index.constant 0 : offset + %lane = kernel.subgroup.lane.id : index + %lane_i32 = index.cast %lane : index to i32 + %is_lane_zero = scalar.cmpi eq, %lane_i32, %c0_i32 : i32 + scf.if %valid_token { + scf.if %valid_channel { + scf.if %is_lane_zero { + %output_view = buffer.view %output_noalias[%c0_offset] : buffer -> view<[%launch_token_count]x[%output_bound]xf32> + view.store %dot, %output_view[%token, %channel] : f32, view<[%launch_token_count]x[%output_bound]xf32> + } + } + } + kernel.return +} + +// Uniform packed bytes exercise both low/high Q6 fields and signed per-group +// scales. The output width crosses the eight-wave tile boundary, while K=2048 +// covers the exact mixed-format dense V contraction depth. +check.case public @ggml_linear_q6k_q8_1_x4_nonzero_tail_case { + %token_count = check.literal value(2) : index + %input_size = check.literal value(2048) : index + %output_size = check.literal value(9) : index + %input = check.generate.fill value(0.00390625) : tensor<2x2048xf32> + %q8_input = check.generate.fill value(0) : tensor<2x2304xi8> + %weight = check.generate.fill value(-86) : tensor<9x8x210xi8> + %output = check.generate.fill value(0.0) : tensor<2x9xf32> + %expected = check.generate.fill value(358.1715087890625) : tensor<2x9xf32> + kernel.launch @ggml_quantize_q8_1_x4_f32[%token_count, %input_size](%token_count, %input_size, %input, %q8_input) : [index, index](index, index, tensor<2x2048xf32>, tensor<2x2304xi8>) + kernel.launch @ggml_linear_q6k_q8_1_x4[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<2x2304xi8>, tensor<9x8x210xi8>, tensor<2x9xf32>) + check.expect.approximate actual(%output) expected(%expected) atol(0.25) rtol(9.9999999999999995e-07) nan(same) : tensor<2x9xf32>, tensor<2x9xf32> + check.return +} + +check.benchmark<@ggml_linear_q6k_q8_1_x4_nonzero_tail_case> @ggml_linear_q6k_q8_1_x4_small diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json index 32fee80b9ee1..1469d87f1807 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json @@ -47,9 +47,6 @@ { "path": "../loom-libs/motifs/quantize_q8_1_x4_group.loom" }, - { - "path": "ggml/linear_q6k_q8_1_x4.loom" - }, { "path": "ggml/quantize_q8_1_x4.loom" }, @@ -119,135 +116,12 @@ { "path": "../qwen/attention_metadata.loom", "upstream_path": "experimental/qwen/kernels/attention_metadata.loom" + }, + { + "path": "ggml/q6_q8_reference_test.loom" } ], "exports": [ - { - "name": "ggml_linear_q6k_q8_1_x4", - "symbol": "ggml_linear_q6k_q8_1_x4", - "source": "ggml/linear_q6k_q8_1_x4.loom", - "workload_parameters": [ - { - "name": "token_count", - "type": "index" - }, - { - "name": "input_size", - "type": "index" - }, - { - "name": "output_size", - "type": "index" - } - ], - "launch_parameters": [ - { - "name": "token_count", - "type": "index" - }, - { - "name": "input_size", - "type": "index" - }, - { - "name": "output_size", - "type": "index" - } - ], - "bindings": [ - "q8_input", - "weight", - "output" - ], - "binding_access": [ - "read", - "read", - "read_write" - ], - "target_selector": "", - "compile_recipe": { - "mode": "archive", - "link_module": "linear_q6k_q8_1_x4_linked", - "primary_sources": [ - "ggml/linear_q6k_q8_1_x4.loom" - ], - "library_sources": [ - "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", - "../loom-libs/motifs/q6_k_f16.loom", - "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom" - ] - }, - "compile_dependencies": [ - "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", - "../loom-libs/motifs/q6_k_f16.loom", - "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom" - ] - }, - { - "name": "ggml_linear_q6k_q8_1_x4", - "symbol": "ggml_linear_q6k_q8_1_x4_wave64_block4", - "source": "ggml/linear_q6k_q8_1_x4.loom", - "workload_parameters": [ - { - "name": "token_count", - "type": "index" - }, - { - "name": "input_size", - "type": "index" - }, - { - "name": "output_size", - "type": "index" - } - ], - "launch_parameters": [ - { - "name": "token_count", - "type": "index" - }, - { - "name": "input_size", - "type": "index" - }, - { - "name": "output_size", - "type": "index" - } - ], - "bindings": [ - "q8_input", - "weight", - "output" - ], - "binding_access": [ - "read", - "read", - "read_write" - ], - "target_selector": "gfx1151", - "compile_recipe": { - "mode": "archive", - "link_module": "linear_q6k_q8_1_x4_linked", - "primary_sources": [ - "ggml/linear_q6k_q8_1_x4.loom" - ], - "library_sources": [ - "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", - "../loom-libs/motifs/q6_k_f16.loom", - "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom" - ] - }, - "compile_dependencies": [ - "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", - "../loom-libs/motifs/q6_k_f16.loom", - "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom" - ] - }, { "name": "ggml_quantize_q8_1_x4_f32", "symbol": "ggml_quantize_q8_1_x4_f32", @@ -1068,18 +942,6 @@ ":attention_qkv_quantized_linked" ] }, - { - "name": "linear_q6k_q8_1_x4_linked", - "srcs": [ - "ggml/linear_q6k_q8_1_x4.loom" - ], - "libraries": [ - "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", - "../loom-libs/motifs/q6_k_f16.loom", - "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom" - ] - }, { "name": "routed_down_quantized_f16_wmma_linked", "srcs": [ @@ -1127,7 +989,7 @@ "libraries": [ "../loom-libs/motifs/matmul/q4_q8_1_x4_dense.loom", "../loom-libs/motifs/matmul/q4_q8_1_x4_decode.loom", - "ggml/linear_q6k_q8_1_x4.loom", + "ggml/q6_q8_reference_test.loom", "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", "../loom-libs/motifs/q6_k_f16.loom", "../loom-libs/motifs/normalization/rmsnorm_q8_1_x4.loom", @@ -1140,6 +1002,18 @@ "../loom-libs/motifs/elementwise/binary_apply.loom", "../loom-libs/ops/moe_routing_tables.loom" ] + }, + { + "name": "q6_q8_reference_test_linked", + "srcs": [ + "ggml/q6_q8_reference_test.loom" + ], + "libraries": [ + "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", + "../loom-libs/motifs/q6_k_f16.loom", + "ggml/quantize_q8_1_x4.loom", + "../loom-libs/motifs/quantize_q8_1_x4_group.loom" + ] } ], "plan_cases": [ @@ -1307,17 +1181,6 @@ ], "source": "ggml/quantize_q8_1_x4.loom" }, - { - "name": "linear_q6k_q8_1_x4_plan_test", - "args": [ - "$(location :linear_q6k_q8_1_x4_linked)", - "--config=ggml.linear_q6k_q8_1_x4.output_capacity=151936", - "--config=ggml.linear_q6k_q8_1_x4.token_capacity=2048", - "--dry-run", - "--output-format=jsonl" - ], - "link_module": "linear_q6k_q8_1_x4_linked" - }, { "name": "routed_down_weighted_reduce_next_rmsnorm_f32_plan_test", "args": [ @@ -1428,6 +1291,19 @@ "--output-format=jsonl" ], "source": "../qwen/attention_metadata.loom" + }, + { + "name": "q6_q8_reference_tail_plan_test", + "args": [ + "$(location :q6_q8_reference_test_linked)", + "--benchmark=@ggml_linear_q6k_q8_1_x4_small", + "--config=ggml.linear_q6k_q8_1_x4.output_capacity=9", + "--config=ggml.linear_q6k_q8_1_x4.token_capacity=2", + "--config=ggml.quantize_q8_1_x4.group_capacity=32", + "--dry-run", + "--output-format=jsonl" + ], + "link_module": "q6_q8_reference_test_linked" } ] } diff --git a/src/llama-arch.h b/src/llama-arch.h index 0db00b72ee85..3bbddc4fc3c9 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -916,6 +916,7 @@ bool llm_arch_supports_mixed_batch(const llm_arch & arch); // This preference keeps compatible HRX consumers on one weight layout. Other architectures keep their existing dispatch choices. inline bool llm_arch_prefers_hrx_row64_weights(llm_arch arch) { switch (arch) { + case LLM_ARCH_QWEN3: case LLM_ARCH_QWEN35: return true; default: diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index 862e7d145518..72b4c17a05d6 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -4006,11 +4006,12 @@ static void schedule_qwen_terminal_q6k_q8_command(int64_t token_count) { const ggml::hrx::Dispatch & vocab_dispatch = scheduler.plan().dispatches[1]; REQUIRE(kernel_name_for_id(vocab_dispatch.kernel.kernel_id) == "qwen3_moe:ggml_linear_q6k_q8_1_x4"); REQUIRE(vocab_dispatch.kernel.integer_parameters.at("token_count") == token_count); - REQUIRE(vocab_dispatch.kernel.integer_parameters.at("input_size") == 2048); - REQUIRE(vocab_dispatch.kernel.integer_parameters.at("output_size") == 151936); - require_compile_parameter(vocab_dispatch, "ggml.linear_q6k_q8_1_x4.token_capacity", std::to_string(token_count)); - require_compile_parameter(vocab_dispatch, "ggml.linear_q6k_q8_1_x4.output_capacity", "151936"); - REQUIRE(vocab_dispatch.bindings.size() == 3); + require_compile_parameter(vocab_dispatch, "ggml.matmul.vector.input_size", "2048"); + require_compile_parameter(vocab_dispatch, "ggml.matmul.vector.output_size", "151936"); + require_compile_parameter(vocab_dispatch, "ggml.workload.token_capacity", std::to_string(token_count)); + require_compile_parameter(vocab_dispatch, "ggml.matmul.vector.weight_format", "6"); + require_compile_parameter(vocab_dispatch, "ggml.matmul.vector.activation_format", "9"); + REQUIRE(vocab_dispatch.bindings.size() == 4); REQUIRE(vocab_dispatch.bindings[0].value == q8_transient.value); REQUIRE(vocab_dispatch.bindings[0].length == q8_transient.size); @@ -4022,8 +4023,8 @@ static void schedule_qwen_terminal_q6k_q8_command(int64_t token_count) { REQUIRE(commands.commands[0].bindings.size() == 4); REQUIRE(commands.commands[0].bindings[3].name == "q8_output"); REQUIRE(commands.commands[0].bindings[3].origin == ggml::hrx::CommandBindingOrigin::Transient); - REQUIRE(commands.commands[1].bindings.size() == 3); - REQUIRE(commands.commands[1].bindings[0].name == "q8_input"); + REQUIRE(commands.commands[1].bindings.size() == 4); + REQUIRE(commands.commands[1].bindings[0].name == "input"); REQUIRE(commands.commands[1].bindings[0].origin == ggml::hrx::CommandBindingOrigin::Transient); REQUIRE(commands.commands[1].bindings[0].value == q8_transient.value); @@ -4282,7 +4283,8 @@ static void schedule_get_rows_q8_1_alternate_command(ggml_type embedding_weight_ const ggml::hrx::Dispatch & vocab_dispatch = scheduler.plan().dispatches[1]; const std::string vocab_kernel_name = kernel_name_for_id(vocab_dispatch.kernel.kernel_id); - REQUIRE(vocab_dispatch.bindings.size() == 3); + REQUIRE(vocab_dispatch.bindings.size() == + (vocab_kernel_name == "qwen3_moe:ggml_linear_q6k_q8_1_x4" ? 4 : 3)); if (vocab_kernel_name == "qwen3_moe:ggml_linear_q6k_q8_1_x4") { REQUIRE(vocab_dispatch.bindings[0].value == alternate->alternate_value); REQUIRE(vocab_dispatch.bindings[0].length == alternate->byte_count); @@ -4302,9 +4304,9 @@ static void schedule_get_rows_q8_1_alternate_command(ggml_type embedding_weight_ REQUIRE(commands.commands[0].bindings.size() == 4); REQUIRE(commands.commands[0].bindings[3].name == "next_output"); REQUIRE(commands.commands[0].bindings[3].origin == ggml::hrx::CommandBindingOrigin::Transient); - REQUIRE(commands.commands[1].bindings.size() == 3); - REQUIRE(commands.commands[1].bindings[0].name == - (vocab_kernel_name == "qwen3_moe:ggml_linear_q6k_q8_1_x4" ? "q8_input" : "input")); + REQUIRE(commands.commands[1].bindings.size() == + (vocab_kernel_name == "qwen3_moe:ggml_linear_q6k_q8_1_x4" ? 4 : 3)); + REQUIRE(commands.commands[1].bindings[0].name == "input"); ggml_free(ctx); } diff --git a/tests/hrx-loom-jit-test.cpp b/tests/hrx-loom-jit-test.cpp index df6581f9cae9..e815c5a126dd 100644 --- a/tests/hrx-loom-jit-test.cpp +++ b/tests/hrx-loom-jit-test.cpp @@ -554,11 +554,17 @@ static void run_targeted_export_materialize_case(HrxTestDevice & device) { ggml::hrx::Dispatch dispatch; dispatch.kernel.kernel_id = definition->id; dispatch.kernel.integer_parameters.emplace("token_count", 1); - dispatch.kernel.integer_parameters.emplace("input_size", 2048); - dispatch.kernel.integer_parameters.emplace("output_size", 151936); - dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.token_capacity", "1"); - dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.output_capacity", "151936"); - dispatch.bindings.resize(3); + dispatch.kernel.compile_parameters = { + { "ggml.workload.token_capacity", "1" }, + { "ggml.matmul.vector.input_size", "2048" }, + { "ggml.matmul.vector.output_size", "151936" }, + { "ggml.matmul.vector.weight_format", "6" }, + { "ggml.matmul.vector.activation_format", "9" }, + { "ggml.matmul.vector.output_accumulation", "0" }, + { "ggml.matmul.vector.output_unary_op", "23" }, + { "ggml.matmul.vector.publish_format", "0" }, + }; + dispatch.bindings.resize(4); ggml::hrx::KernelExecutableCache cache(ggml::hrx::LoomJitMode::Sync); std::vector constants; diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index 993f302387f7..9b5008884bcd 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -6028,13 +6028,17 @@ static void run_scale_add_f32_cpu_reference_case(bool strided_residual) { ggml_backend_free(hrx_backend); } +static void set_test_row64_policy(ggml_backend_t backend, bool prefer_row64); + static void run_get_rows_f32_cpu_reference_case(ggml_type weight_type, int64_t hidden_size = kQwenHiddenSize, int64_t vocabulary_count = 64, - int64_t token_count = 7) { + int64_t token_count = 7, + bool prefer_row64 = false) { ggml_backend_t cpu_backend = init_cpu_backend(); ggml_backend_t hrx_backend = ggml_backend_hrx_init(0); REQUIRE(hrx_backend != nullptr); + set_test_row64_policy(hrx_backend, prefer_row64); ggml_init_params params = {}; params.mem_size = 4 * 1024 * 1024; @@ -6060,7 +6064,23 @@ static void run_get_rows_f32_cpu_reference_case(ggml_type weight_type, ggml_build_forward_expand(cpu_graph, cpu_output); ggml_build_forward_expand(hrx_graph, hrx_output); - require_kernel_subsequence(scheduled_kernel_sequence(hrx_graph), { "loom_libs:ggml_get_rows_f32" }); + std::vector weight = make_matmul_weight_bytes(weight_type, hidden_size, vocabulary_count, 5); + auto imported_weight = std::unique_ptr( + nullptr, ggml_backend_buffer_free); + if (prefer_row64) { + imported_weight.reset(ggml_backend_dev_buffer_from_host_ptr( + ggml_backend_get_device(hrx_backend), weight.data(), weight.size(), weight.size())); + REQUIRE(imported_weight != nullptr); + REQUIRE(ggml_backend_tensor_alloc(imported_weight.get(), hrx_weight, weight.data()) == GGML_STATUS_SUCCESS); + ggml_backend_buffer_set_usage(imported_weight.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + const char * layout = weight_type == GGML_TYPE_Q4_K ? ggml::hrx::kQ4KPackedK256Row64Layout : + ggml::hrx::kQ6KPackedK256Row64ScaleRowLayout; + require_weight_layout(hrx_graph, { hrx_weight }, layout, { "gfx1151", true }, + "loom_libs:ggml_get_rows_f32", 1); + } + + require_kernel_subsequence(scheduled_kernel_sequence(hrx_graph, { "gfx1151", prefer_row64 }), + { "loom_libs:ggml_get_rows_f32" }); const ggml::hrx::KernelSpecialization specialization = scheduled_kernel_specialization(hrx_graph, "loom_libs:ggml_get_rows_f32"); REQUIRE(specialization.workload_specialization == ggml::hrx::WorkloadSpecialization::Dynamic); @@ -6076,12 +6096,14 @@ static void run_get_rows_f32_cpu_reference_case(ggml_type weight_type, REQUIRE(cpu_buffer != nullptr); REQUIRE(hrx_buffer != nullptr); - const std::vector weight = make_matmul_weight_bytes(weight_type, hidden_size, vocabulary_count, 5); std::vector ids(static_cast(token_count)); for (int64_t i = 0; i < token_count; ++i) { ids[static_cast(i)] = static_cast((i * 17 + 3) % vocabulary_count); } - set_tensor_pair_bytes(cpu_backend, cpu_weight, hrx_backend, hrx_weight, weight.data(), weight.size()); + ggml_backend_tensor_set(cpu_weight, weight.data(), 0, weight.size()); + if (!prefer_row64) { + ggml_backend_tensor_set(hrx_weight, weight.data(), 0, weight.size()); + } set_tensor_pair_bytes(cpu_backend, cpu_ids, hrx_backend, hrx_ids, ids.data(), ids.size() * sizeof(int32_t)); REQUIRE(ggml_backend_graph_compute(cpu_backend, cpu_graph) == GGML_STATUS_SUCCESS); @@ -6092,12 +6114,34 @@ static void run_get_rows_f32_cpu_reference_case(ggml_type weight_type, ggml_backend_buffer_free(cpu_buffer); ggml_backend_buffer_free(hrx_buffer); + imported_weight.reset(); ggml_free(cpu_ctx); ggml_free(hrx_ctx); ggml_backend_free(cpu_backend); ggml_backend_free(hrx_backend); } +static void run_get_rows_row64_alias_schedule_case() { + ggml_init_params params = {}; + params.mem_size = 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + ggml_tensor * weight = ggml_new_tensor_2d(ctx, GGML_TYPE_Q6_K, kQwenHiddenSize, 64); + ggml_tensor * view = ggml_view_2d(ctx, weight, kQwenHiddenSize, 64, weight->nb[1], 0); + ggml_tensor * ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); + ggml_tensor * rows = ggml_get_rows(ctx, view, ids); + ggml_cgraph * graph = ggml_new_graph(ctx); + ggml_build_forward_expand(graph, rows); + ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*graph); + REQUIRE(imported.valid()); + const ggml::hrx::Value * view_value = imported.graph.values().find_tensor(view); + REQUIRE(view_value != nullptr && view_value->alias_source.value >= 0); + require_weight_layout(graph, { view }, ggml::hrx::kNativeWeightLayout, { "gfx1151", true }, + "loom_libs:ggml_get_rows_f32", 1); + ggml_free(ctx); +} + static void run_get_rows_q8_1_zero_weight_case() { ggml_backend_t hrx_backend = ggml_backend_hrx_init(0); REQUIRE(hrx_backend != nullptr); @@ -6271,6 +6315,79 @@ static std::vector f32_matmul_reference(ggml_type type, const std::vector int64_t inputs, int64_t outputs, int64_t tokens); static void require_reference_close(const std::vector & actual, const std::vector & expected, float absolute_tolerance); +static void run_get_rows_rmsnorm_row64_cpu_reference_case(ggml_type embedding_type, int64_t row_count, + int64_t token_count) { + ggml_backend_t cpu_backend = init_cpu_backend(); + ggml_backend_t hrx_backend = ggml_backend_hrx_init(0); + REQUIRE(hrx_backend != nullptr); + set_test_row64_policy(hrx_backend, true); + + ggml_init_params params = {}; + params.mem_size = 8 * 1024 * 1024; + params.no_alloc = true; + ggml_context * cpu_ctx = ggml_init(params); + ggml_context * hrx_ctx = ggml_init(params); + REQUIRE(cpu_ctx != nullptr && hrx_ctx != nullptr); + + auto build_graph = [embedding_type, row_count, token_count](ggml_context * ctx) { + ggml_tensor * embedding = ggml_new_tensor_2d(ctx, embedding_type, kQwenHiddenSize, row_count); + ggml_tensor * ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, token_count); + ggml_tensor * norm_weight = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, kQwenHiddenSize); + ggml_tensor * projection = ggml_new_tensor_2d(ctx, GGML_TYPE_Q6_K, kQwenHiddenSize, 64); + ggml_tensor * rows = ggml_get_rows(ctx, embedding, ids); + ggml_tensor * normalized = build_rmsnorm_mul_graph(ctx, rows, norm_weight); + ggml_tensor * output = ggml_mul_mat(ctx, projection, normalized); + return std::array{ embedding, ids, norm_weight, projection, output }; + }; + const auto cpu = build_graph(cpu_ctx); + const auto hrx = build_graph(hrx_ctx); + ggml_cgraph * cpu_graph = ggml_new_graph(cpu_ctx); + ggml_cgraph * hrx_graph = ggml_new_graph(hrx_ctx); + ggml_build_forward_expand(cpu_graph, cpu[4]); + ggml_build_forward_expand(hrx_graph, hrx[4]); + + std::vector embedding = make_matmul_weight_bytes(embedding_type, kQwenHiddenSize, row_count, 5); + std::vector projection = make_matmul_weight_bytes(GGML_TYPE_Q6_K, kQwenHiddenSize, 64, 11); + auto embedding_buffer = import_test_weight(hrx_backend, hrx[0], embedding); + auto projection_buffer = allocate_test_weight(hrx_backend, hrx[3]); + const char * layout = embedding_type == GGML_TYPE_Q4_K ? ggml::hrx::kQ4KPackedK256Row64Layout : + ggml::hrx::kQ6KPackedK256Row64ScaleRowLayout; + require_weight_layout(hrx_graph, { hrx[0] }, layout, { "gfx1151", true }, + "loom_libs:ggml_get_rows_rmsnorm_binary_q8_1_x4_f16", 1); + + ggml_backend_buffer_t cpu_buffer = ggml_backend_alloc_ctx_tensors(cpu_ctx, cpu_backend); + ggml_backend_buffer_t hrx_buffer = ggml_backend_alloc_ctx_tensors(hrx_ctx, hrx_backend); + REQUIRE(cpu_buffer != nullptr && hrx_buffer != nullptr); + std::vector ids(static_cast(token_count)); + for (int64_t i = 0; i < token_count; ++i) { + ids[static_cast(i)] = static_cast((i * 37 + 3) % row_count); + } + const std::vector norm_weight = make_weight(kQwenHiddenSize); + ggml_backend_tensor_set(cpu[0], embedding.data(), 0, embedding.size()); + ggml_backend_tensor_set(cpu[3], projection.data(), 0, projection.size()); + ggml_backend_tensor_set(hrx[3], projection.data(), 0, projection.size()); + set_tensor_pair_bytes(cpu_backend, cpu[1], hrx_backend, hrx[1], ids.data(), ids.size() * sizeof(int32_t)); + set_tensor_pair_bytes(cpu_backend, cpu[2], hrx_backend, hrx[2], norm_weight.data(), + norm_weight.size() * sizeof(float)); + + REQUIRE(ggml_backend_graph_compute(cpu_backend, cpu_graph) == GGML_STATUS_SUCCESS); + REQUIRE(ggml_backend_graph_compute(hrx_backend, hrx_graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(cpu_backend); + ggml_backend_synchronize(hrx_backend); + require_close(get_f32_tensor(hrx_backend, hrx[4]), get_f32_tensor(cpu_backend, cpu[4]), 1.0f, 3.0e-2f); + REQUIRE(ggml_backend_graph_compute(hrx_backend, hrx_graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(hrx_backend); + require_weight_cache_stats(hrx_backend, TestWeightSource::Imported, 1); + + ggml_backend_buffer_free(cpu_buffer); + ggml_backend_buffer_free(hrx_buffer); + embedding_buffer.reset(); + projection_buffer.reset(); + ggml_free(cpu_ctx); + ggml_free(hrx_ctx); + ggml_backend_free(cpu_backend); + ggml_backend_free(hrx_backend); +} static void run_dense_matmul_cpu_reference_case(ggml_type weight_type, const char * expected_kernel, @@ -7349,10 +7466,12 @@ static void run_dense_matmul_zero_weight_case(ggml_type weight_type, ggml_backend_free(hrx_backend); } -static void run_endpoint_rmsnorm_q6k_q8_cpu_reference_case() { +static void run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(bool prefer_row64 = false, + bool tied_embedding = false, int64_t token_count = 1) { ggml_backend_t cpu_backend = init_cpu_backend(); ggml_backend_t hrx_backend = ggml_backend_hrx_init(0); REQUIRE(hrx_backend != nullptr); + set_test_row64_policy(hrx_backend, prefer_row64); ggml_init_params params = {}; params.mem_size = 32 * 1024 * 1024; @@ -7362,18 +7481,21 @@ static void run_endpoint_rmsnorm_q6k_q8_cpu_reference_case() { REQUIRE(cpu_ctx != nullptr); REQUIRE(hrx_ctx != nullptr); - constexpr int64_t token_count = 1; ggml_tensor * cpu_input = ggml_new_tensor_2d(cpu_ctx, GGML_TYPE_F32, kQwenHiddenSize, token_count); ggml_tensor * cpu_norm_w = ggml_new_tensor_1d(cpu_ctx, GGML_TYPE_F32, kQwenHiddenSize); ggml_tensor * cpu_weight = ggml_new_tensor_2d(cpu_ctx, GGML_TYPE_Q6_K, kQwenHiddenSize, kQwenVocabularyCount); ggml_tensor * cpu_norm = build_rmsnorm_mul_graph(cpu_ctx, cpu_input, cpu_norm_w); ggml_tensor * cpu_output = ggml_mul_mat(cpu_ctx, cpu_weight, cpu_norm); + ggml_tensor * cpu_ids = tied_embedding ? ggml_new_tensor_1d(cpu_ctx, GGML_TYPE_I32, 2) : nullptr; + ggml_tensor * cpu_rows = tied_embedding ? ggml_get_rows(cpu_ctx, cpu_weight, cpu_ids) : nullptr; ggml_tensor * hrx_input = ggml_new_tensor_2d(hrx_ctx, GGML_TYPE_F32, kQwenHiddenSize, token_count); ggml_tensor * hrx_norm_w = ggml_new_tensor_1d(hrx_ctx, GGML_TYPE_F32, kQwenHiddenSize); ggml_tensor * hrx_weight = ggml_new_tensor_2d(hrx_ctx, GGML_TYPE_Q6_K, kQwenHiddenSize, kQwenVocabularyCount); ggml_tensor * hrx_norm = build_rmsnorm_mul_graph(hrx_ctx, hrx_input, hrx_norm_w); ggml_tensor * hrx_output = ggml_mul_mat(hrx_ctx, hrx_weight, hrx_norm); + ggml_tensor * hrx_ids = tied_embedding ? ggml_new_tensor_1d(hrx_ctx, GGML_TYPE_I32, 2) : nullptr; + ggml_tensor * hrx_rows = tied_embedding ? ggml_get_rows(hrx_ctx, hrx_weight, hrx_ids) : nullptr; REQUIRE(cpu_output != nullptr); REQUIRE(hrx_output != nullptr); @@ -7381,12 +7503,36 @@ static void run_endpoint_rmsnorm_q6k_q8_cpu_reference_case() { ggml_cgraph * hrx_graph = ggml_new_graph(hrx_ctx); REQUIRE(cpu_graph != nullptr); REQUIRE(hrx_graph != nullptr); + if (tied_embedding) { + ggml_build_forward_expand(cpu_graph, cpu_rows); + ggml_build_forward_expand(hrx_graph, hrx_rows); + } ggml_build_forward_expand(cpu_graph, cpu_output); ggml_build_forward_expand(hrx_graph, hrx_output); - require_kernel_subsequence( - scheduled_kernel_sequence(hrx_graph), - { "qwen3_moe:qwen3_moe_rmsnorm_f32_quantize_q8_1_x4", "qwen3_moe:ggml_linear_q6k_q8_1_x4" }); + std::vector weight = make_quantized_rows(GGML_TYPE_Q6_K, kQwenHiddenSize, kQwenVocabularyCount, 9); + auto weight_buffer = std::unique_ptr( + nullptr, ggml_backend_buffer_free); + if (prefer_row64) { + weight_buffer = import_test_weight(hrx_backend, hrx_weight, weight); + require_weight_layout(hrx_graph, { hrx_weight }, ggml::hrx::kQ6KPackedK256Row64ScaleRowLayout, + { "gfx1151", true }, nullptr, tied_embedding ? 2 : 1); + } + + if (token_count == 1) { + require_kernel_subsequence( + scheduled_kernel_sequence(hrx_graph, { "gfx1151", prefer_row64 }), + { "qwen3_moe:qwen3_moe_rmsnorm_f32_quantize_q8_1_x4", + prefer_row64 ? "loom_libs:ggml_mul_mat_vector_f32_f32" : "qwen3_moe:ggml_linear_q6k_q8_1_x4" }); + } else { + require_kernel_subsequence( + scheduled_kernel_sequence(hrx_graph, { "gfx1151", prefer_row64 }), + { "loom_libs:ggml_mul_mat_tiled_input_f32_publish_f32" }); + } + if (tied_embedding) { + require_kernel_subsequence(scheduled_kernel_sequence(hrx_graph, { "gfx1151", prefer_row64 }), + { "loom_libs:ggml_get_rows_f32" }); + } ggml_backend_buffer_t cpu_buffer = ggml_backend_alloc_ctx_tensors(cpu_ctx, cpu_backend); ggml_backend_buffer_t hrx_buffer = ggml_backend_alloc_ctx_tensors(hrx_ctx, hrx_backend); @@ -7395,20 +7541,37 @@ static void run_endpoint_rmsnorm_q6k_q8_cpu_reference_case() { const std::vector input = make_pattern_f32(kQwenHiddenSize * token_count, 8, 0.01f); const std::vector norm_w = make_weight(kQwenHiddenSize); - const std::vector weight = make_quantized_rows(GGML_TYPE_Q6_K, kQwenHiddenSize, kQwenVocabularyCount, 9); set_tensor_pair_bytes(cpu_backend, cpu_input, hrx_backend, hrx_input, input.data(), input.size() * sizeof(float)); set_tensor_pair_bytes(cpu_backend, cpu_norm_w, hrx_backend, hrx_norm_w, norm_w.data(), norm_w.size() * sizeof(float)); - set_tensor_pair_bytes(cpu_backend, cpu_weight, hrx_backend, hrx_weight, weight.data(), weight.size()); + ggml_backend_tensor_set(cpu_weight, weight.data(), 0, weight.size()); + if (!prefer_row64) { + ggml_backend_tensor_set(hrx_weight, weight.data(), 0, weight.size()); + } + if (tied_embedding) { + const std::array ids = { 3, static_cast(kQwenVocabularyCount - 1) }; + set_tensor_pair_bytes(cpu_backend, cpu_ids, hrx_backend, hrx_ids, ids.data(), sizeof(ids)); + } REQUIRE(ggml_backend_graph_compute(cpu_backend, cpu_graph) == GGML_STATUS_SUCCESS); REQUIRE(ggml_backend_graph_compute(hrx_backend, hrx_graph) == GGML_STATUS_SUCCESS); ggml_backend_synchronize(cpu_backend); ggml_backend_synchronize(hrx_backend); require_close(get_f32_tensor(hrx_backend, hrx_output), get_f32_tensor(cpu_backend, cpu_output), 6.0e-1f, 3.0e-2f); + if (tied_embedding) { + require_close(get_f32_tensor(hrx_backend, hrx_rows), get_f32_tensor(cpu_backend, cpu_rows), 5.0e-4f); + } + if (prefer_row64) { + REQUIRE(ggml_backend_graph_compute(hrx_backend, hrx_graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(hrx_backend); + require_close(get_f32_tensor(hrx_backend, hrx_output), get_f32_tensor(cpu_backend, cpu_output), 6.0e-1f, + 3.0e-2f); + require_weight_cache_stats(hrx_backend, TestWeightSource::Imported, 1); + } ggml_backend_buffer_free(cpu_buffer); ggml_backend_buffer_free(hrx_buffer); + weight_buffer.reset(); ggml_free(cpu_ctx); ggml_free(hrx_ctx); ggml_backend_free(cpu_backend); @@ -8919,7 +9082,7 @@ struct ArchitectureRow64Case { static constexpr ArchitectureRow64Case kArchitectureRow64Cases[] = { { LLM_ARCH_QWEN35, true }, { LLM_ARCH_QWEN35MOE, false }, - { LLM_ARCH_QWEN3, false }, + { LLM_ARCH_QWEN3, true }, { LLM_ARCH_PHI3, false }, { LLM_ARCH_GEMMA4, false }, { LLM_ARCH_LLAMA, false }, @@ -8960,11 +9123,13 @@ static void run_qwen27_ssm_policy_route_case(ggml_type type, bool gathered) { ggml_free(graph.ctx); } -static void run_qwen27_matmul_policy_route_case(ggml_type type, int64_t tokens) { +static void run_qwen27_matmul_policy_route_case(ggml_type type, int64_t tokens, + int64_t k = 5120, int64_t n = 10240, + const char * target = "gfx1151") { ggml_context * ctx = ggml_init({ 4 * 1024 * 1024, nullptr, true }); REQUIRE(ctx != nullptr); - ggml_tensor * weight = ggml_new_tensor_2d(ctx, type, 5120, 10240); - ggml_tensor * input = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 5120, tokens); + ggml_tensor * weight = ggml_new_tensor_2d(ctx, type, k, n); + ggml_tensor * input = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, tokens); ggml_tensor * output = ggml_silu(ctx, ggml_mul_mat(ctx, weight, input)); ggml_cgraph * graph = ggml_new_graph(ctx); ggml_build_forward_expand(graph, output); @@ -8974,25 +9139,29 @@ static void run_qwen27_matmul_policy_route_case(ggml_type type, int64_t tokens) ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*graph); REQUIRE(imported.valid()); ggml::hrx::DispatchScheduler scheduler; - REQUIRE(scheduler.schedule_graph(imported.graph, { "gfx1151", prefer_row64 })); + REQUIRE(scheduler.schedule_graph(imported.graph, { target, prefer_row64 })); const ggml::hrx::Value * source = imported.graph.values().find_tensor(weight); REQUIRE(source != nullptr); const char * row64 = type == GGML_TYPE_Q4_K ? ggml::hrx::kQ4KPackedK256Row64Layout : ggml::hrx::kQ6KPackedK256Row64ScaleRowLayout; - const char * expected = policy.prefer_row64 ? row64 : ggml::hrx::kNativeWeightLayout; + // The existing Q4 prefill route already uses row64 with the policy off. + const bool legacy_row64 = type == GGML_TYPE_Q4_K && tokens == 512; + const char * expected = policy.prefer_row64 || legacy_row64 ? row64 : ggml::hrx::kNativeWeightLayout; size_t matched = 0; for (const auto & dispatch : scheduler.plan().dispatches) { for (const auto & binding : dispatch.bindings) { if (binding.value == source->id) { REQUIRE(binding.layout == expected); const bool q6_weight = type == GGML_TYPE_Q6_K; - const bool short_prompt = tokens == 66; + const bool short_prompt = tokens >= 64 && tokens < 128; const bool short_q6 = q6_weight && short_prompt; - if (short_q6) { + if (short_q6 && (policy.prefer_row64 || tokens == 66)) { + const bool use_m64 = n == 1024 && (k == 1024 || k == 3072); const char * expected_kernel = policy.prefer_row64 ? - "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_wave32" : + (use_m64 ? "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32" : + "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_wave32") : "loom_libs:ggml_mul_mat_tiled_input_f32_publish_f32"; - REQUIRE(kernel_name_for_id(dispatch.kernel.kernel_id, "gfx1151") == expected_kernel); + REQUIRE(kernel_name_for_id(dispatch.kernel.kernel_id, target) == expected_kernel); } ++matched; } @@ -9126,11 +9295,12 @@ static void require_f32_dot_reference(ggml_type type, const std::vector require_close(observed_samples, reference_samples, absolute_tolerance, 0.02f); } -static void run_dense_matmul_f32_reference_case(ggml_type type, int64_t tokens, int64_t k = 512, - int64_t n = 128) { +static std::vector run_dense_matmul_f32_reference_case(ggml_type type, int64_t tokens, int64_t k = 512, + int64_t n = 128, bool prefer_row64 = true, + bool check_reference = true) { auto backend = ggml_backend_hrx_init(0); REQUIRE(backend != nullptr); - set_test_row64_policy(backend, true); + set_test_row64_policy(backend, prefer_row64); auto * backend_state = static_cast(backend->context); REQUIRE(backend_state != nullptr && backend_state->device != nullptr); const std::string architecture = backend_state->device->architecture; @@ -9146,22 +9316,26 @@ static void run_dense_matmul_f32_reference_case(ggml_type type, int64_t tokens, ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*graph); REQUIRE(imported.valid()); ggml::hrx::DispatchScheduler scheduler; - REQUIRE(scheduler.schedule_graph(imported.graph, { architecture, true })); + REQUIRE(scheduler.schedule_graph(imported.graph, { architecture, prefer_row64 })); const ggml::hrx::Value * source = imported.graph.values().find_tensor(weight); REQUIRE(source != nullptr); - const char * expected_layout = type == GGML_TYPE_Q4_K ? ggml::hrx::kQ4KPackedK256Row64Layout : + const char * expected_layout = !prefer_row64 ? ggml::hrx::kNativeWeightLayout : + type == GGML_TYPE_Q4_K ? ggml::hrx::kQ4KPackedK256Row64Layout : ggml::hrx::kQ6KPackedK256Row64ScaleRowLayout; const bool q6_weight = type == GGML_TYPE_Q6_K; const bool short_minimum = tokens >= 64; const bool short_maximum = tokens < 128; - const bool short_q6 = q6_weight && short_minimum && short_maximum; + const bool short_q6 = prefer_row64 && q6_weight && short_minimum && short_maximum; const bool prefill_minimum = tokens >= 128; const bool prefill_aligned = tokens % 128 == 0; const bool aligned_prefill = prefill_minimum && prefill_aligned; const bool aligned_q6_prefill = q6_weight && aligned_prefill; + const bool short_m64 = short_q6 && n == 1024 && (k == 1024 || k == 3072); const char * expected_kernel = tokens == 1 ? "loom_libs:ggml_mul_mat_vector_f32_f32" : + short_m64 ? "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32" : short_q6 ? "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_wave32" : aligned_q6_prefill ? "loom_libs:ggml_mul_mat_q6_k_f16_wmma_prefill_wave32" : + tokens % 32 == 0 ? "loom_libs:ggml_mul_mat_tiled_input_f32_publish_f32_aligned" : "loom_libs:ggml_mul_mat_tiled_input_f32_publish_f32"; size_t matched = 0; for (const auto & dispatch : scheduler.plan().dispatches) { @@ -9185,14 +9359,16 @@ static void run_dense_matmul_f32_reference_case(ggml_type type, int64_t tokens, REQUIRE(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS); ggml_backend_synchronize(backend); const auto observed = get_f32_tensor(backend, output); - std::vector rows; - for (int64_t row = 0; row < n; ++row) { - rows.push_back(row); - } - const std::vector sample_tokens = tokens == 1 ? std::vector{ 0 } : - std::vector{ 0, 21, 43, tokens - 1 }; - require_f32_dot_reference(type, weights, input, observed, k, n, rows, sample_tokens, - tokens == 1 ? "vector" : short_q6 ? "short-prefill" : "tiled", true); + if (check_reference) { + std::vector rows; + for (int64_t row = 0; row < n; ++row) { + rows.push_back(row); + } + const std::vector sample_tokens = tokens == 1 ? std::vector{ 0 } : + std::vector{ 0, 21, 43, tokens - 1 }; + require_f32_dot_reference(type, weights, input, observed, k, n, rows, sample_tokens, + tokens == 1 ? "vector" : short_q6 ? "short-prefill" : "tiled", true); + } ggml_backend_hrx_cache_stats stats = {}; REQUIRE(ggml_backend_hrx_get_cache_stats(backend, &stats)); REQUIRE(stats.weight_duplicate_warnings == 0); @@ -9201,6 +9377,7 @@ static void run_dense_matmul_f32_reference_case(ggml_type type, int64_t tokens, weight_buffer.reset(); ggml_free(ctx); ggml_backend_free(backend); + return observed; } static void run_ssm_conv_prefill_recurrent_case(ggml_type type, bool gathered, bool clear_state, @@ -9336,6 +9513,25 @@ static void register_basic_ops_cases(Suite & suite) { suite.host_case("qwen27.ssm_policy_route." + type_name(type) + ".interior", [type] { run_qwen27_ssm_policy_route_case(type, true); }); } + for (const char * target : { "gfx1100", "gfx1151" }) { + for (const int64_t hidden_size : { 1024, 2048, 2560 }) { + for (const ggml_type type : { GGML_TYPE_Q4_K, GGML_TYPE_Q6_K }) { + for (const int64_t tokens : { 1, 29, 63, 64, 66, 127, 128, 129, 512 }) { + suite.host_case("qwen3.matmul_policy_route." + std::string(target) + "." + type_name(type) + + ".k" + std::to_string(hidden_size) + ".tokens" + std::to_string(tokens), + [target, hidden_size, type, tokens] { + run_qwen27_matmul_policy_route_case(type, tokens, hidden_size, 1024, target); + }); + } + } + } + for (const int64_t tokens : { 64, 66, 127 }) { + suite.host_case("qwen3.matmul_policy_route." + std::string(target) + ".q6_K.k3072.tokens" + + std::to_string(tokens), [target, tokens] { + run_qwen27_matmul_policy_route_case(GGML_TYPE_Q6_K, tokens, 3072, 1024, target); + }); + } + } suite.host_case("qwen27.q4_decode_policy_route", [] { run_qwen27_matmul_policy_route_case(GGML_TYPE_Q4_K, 1); }); @@ -9398,6 +9594,37 @@ static void register_basic_ops_cases(Suite & suite) { run_dense_matmul_f32_reference_case(type, 66); }); } + for (const int64_t hidden_size : { 1024, 2048, 2560 }) { + for (const ggml_type type : { GGML_TYPE_Q4_K, GGML_TYPE_Q6_K }) { + for (const int64_t tokens : { 1, 66, 128 }) { + // F16 accumulation at this width exceeds the F32 oracle in both layouts; compare them below. + if (type == GGML_TYPE_Q4_K && hidden_size == 2560 && tokens != 1) { + continue; + } + suite.device_case("qwen3.f32_reference." + type_name(type) + ".k" + + std::to_string(hidden_size) + ".tokens" + std::to_string(tokens), + [hidden_size, type, tokens] { + run_dense_matmul_f32_reference_case(type, tokens, hidden_size, 1024); + }); + } + } + } + suite.device_case("qwen3.q6_K.row64.vocabulary151936.f32_reference", [] { + run_dense_matmul_f32_reference_case(GGML_TYPE_Q6_K, 1, 1024, 151936); + }); + for (const int64_t tokens : { 64, 127 }) { + suite.device_case("qwen3.q6_K.k1024.m64_boundary.tokens" + std::to_string(tokens), [tokens] { + run_dense_matmul_f32_reference_case(GGML_TYPE_Q6_K, tokens, 1024, 1024); + }); + } + for (const int64_t tokens : { 66, 128 }) { + suite.device_case("qwen3.q4_K.k2560.native_row64_equivalence.tokens" + std::to_string(tokens), [tokens] { + const auto native = run_dense_matmul_f32_reference_case(GGML_TYPE_Q4_K, tokens, 2560, 1024, false, false); + const auto row64 = run_dense_matmul_f32_reference_case(GGML_TYPE_Q4_K, tokens, 2560, 1024, true, false); + REQUIRE(native.size() == row64.size()); + REQUIRE(std::memcmp(native.data(), row64.data(), native.size() * sizeof(float)) == 0); + }); + } for (const int64_t tokens : { 63, 64, 127, 128, 129 }) { suite.device_case("qwen27.q6_short_boundary.tokens" + std::to_string(tokens), [tokens] { run_dense_matmul_f32_reference_case(GGML_TYPE_Q6_K, tokens); @@ -9442,6 +9669,29 @@ static void register_basic_ops_cases(Suite & suite) { GGML_TYPE_BF16, GGML_TYPE_F32 }) { suite.device_case("get_rows_f32." + type_name(type), [type] { run_get_rows_f32_cpu_reference_case(type); }); } + for (const ggml_type type : { GGML_TYPE_Q4_K, GGML_TYPE_Q6_K }) { + suite.device_case("get_rows_f32." + type_name(type) + ".row64.tokens1", [type] { + run_get_rows_f32_cpu_reference_case(type, kQwenHiddenSize, 64, 1, true); + }); + suite.device_case("get_rows_f32." + type_name(type) + ".row64.tokens17", [type] { + run_get_rows_f32_cpu_reference_case(type, kQwenHiddenSize, 64, 17, true); + }); + } + suite.device_case("get_rows_f32.q6_k.row64.vocabulary151936.tokens1", [] { + run_get_rows_f32_cpu_reference_case(GGML_TYPE_Q6_K, kQwenHiddenSize, kQwenVocabularyCount, 1, true); + }); + suite.host_case("get_rows_f32.q6_k.row64.alias_uses_native", [] { run_get_rows_row64_alias_schedule_case(); }); + for (const ggml_type type : { GGML_TYPE_Q4_K, GGML_TYPE_Q6_K }) { + suite.device_case("get_rows_rmsnorm." + type_name(type) + ".row64.tokens1", [type] { + run_get_rows_rmsnorm_row64_cpu_reference_case(type, 64, 1); + }); + suite.device_case("get_rows_rmsnorm." + type_name(type) + ".row64.tokens17", [type] { + run_get_rows_rmsnorm_row64_cpu_reference_case(type, 64, 17); + }); + } + suite.device_case("get_rows_rmsnorm.q6_k.row64.vocabulary151936.tokens1", [] { + run_get_rows_rmsnorm_row64_cpu_reference_case(GGML_TYPE_Q6_K, kQwenVocabularyCount, 1); + }); suite.device_case("get_rows_f32.q1_0.rows2048", [] { run_get_rows_f32_cpu_reference_case(GGML_TYPE_Q1_0, 2048); }); suite.device_case("get_rows_f32.q5_1.rows640", [] { run_get_rows_f32_cpu_reference_case(GGML_TYPE_Q5_1, 640); }); suite.device_case("get_rows_q8_1.zero_weight", [] { run_get_rows_q8_1_zero_weight_case(); }); @@ -9888,6 +10138,14 @@ static void register_dense_matmul_cases(Suite & suite) { }); suite.device_case("endpoint_rmsnorm_q6k_q8.cpu_reference", [] { run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(); }); + suite.device_case("endpoint_rmsnorm_q6k_q8.row64.cpu_reference", + [] { run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(true); }); + suite.device_case("endpoint_rmsnorm_q6k_q8.row64.tied_embedding", + [] { run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(true, true); }); + suite.device_case("endpoint_rmsnorm_q6k_q8.cpu_reference.tokens17", + [] { run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(false, false, 17); }); + suite.device_case("endpoint_rmsnorm_q6k_q8.row64.cpu_reference.tokens17", + [] { run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(true, false, 17); }); suite.device_case("dense_matmul.q6_k.skinny.tokens5.outputs4096.input16384", [] { run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_skinny_input_f32_publish_f32", 5, 4096, 16384); @@ -9941,6 +10199,14 @@ static void register_dense_matmul_cases(Suite & suite) { GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_tiled_input_f32_bias_residual_publish_f32", 29, 1024, false, true, false, 3584, row64_postops); }); + for (const int64_t tokens : { 64, 66, 127 }) { + suite.device_case("qwen3.q6_row64_short_residual.k3072.tokens" + std::to_string(tokens), + [tokens, row64_postops] { + run_dense_matmul_postops_cpu_reference_case( + GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32", + tokens, 1024, false, true, false, 3072, row64_postops); + }); + } suite.device_case("minicpmv46.q4_row64_tiled_residual.tokens46", [row64_postops] { run_dense_matmul_postops_cpu_reference_case( GGML_TYPE_Q4_K, "loom_libs:ggml_mul_mat_tiled_input_f32_bias_residual_publish_f32",