From 92c1f6ca79dc1e379f24afac7d10cee9fdc7cc2b Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Wed, 11 Nov 2026 16:42:29 +0000 Subject: [PATCH 01/18] hrx: prefer row64 weights for dense Qwen3 Assisted-by: Codex --- src/llama-arch.h | 1 + tests/hrx-ops-test.cpp | 88 ++++++++++++++++++++++++++++++++---------- 2 files changed, 68 insertions(+), 21 deletions(-) diff --git a/src/llama-arch.h b/src/llama-arch.h index 0db00b72ee85..3bbddc4fc3c9 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -916,6 +916,7 @@ bool llm_arch_supports_mixed_batch(const llm_arch & arch); // This preference keeps compatible HRX consumers on one weight layout. Other architectures keep their existing dispatch choices. inline bool llm_arch_prefers_hrx_row64_weights(llm_arch arch) { switch (arch) { + case LLM_ARCH_QWEN3: case LLM_ARCH_QWEN35: return true; default: diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index 993f302387f7..213637bd8a64 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -8919,7 +8919,7 @@ struct ArchitectureRow64Case { static constexpr ArchitectureRow64Case kArchitectureRow64Cases[] = { { LLM_ARCH_QWEN35, true }, { LLM_ARCH_QWEN35MOE, false }, - { LLM_ARCH_QWEN3, false }, + { LLM_ARCH_QWEN3, true }, { LLM_ARCH_PHI3, false }, { LLM_ARCH_GEMMA4, false }, { LLM_ARCH_LLAMA, false }, @@ -8960,11 +8960,13 @@ static void run_qwen27_ssm_policy_route_case(ggml_type type, bool gathered) { ggml_free(graph.ctx); } -static void run_qwen27_matmul_policy_route_case(ggml_type type, int64_t tokens) { +static void run_qwen27_matmul_policy_route_case(ggml_type type, int64_t tokens, + int64_t k = 5120, int64_t n = 10240, + const char * target = "gfx1151") { ggml_context * ctx = ggml_init({ 4 * 1024 * 1024, nullptr, true }); REQUIRE(ctx != nullptr); - ggml_tensor * weight = ggml_new_tensor_2d(ctx, type, 5120, 10240); - ggml_tensor * input = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 5120, tokens); + ggml_tensor * weight = ggml_new_tensor_2d(ctx, type, k, n); + ggml_tensor * input = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, tokens); ggml_tensor * output = ggml_silu(ctx, ggml_mul_mat(ctx, weight, input)); ggml_cgraph * graph = ggml_new_graph(ctx); ggml_build_forward_expand(graph, output); @@ -8974,12 +8976,14 @@ static void run_qwen27_matmul_policy_route_case(ggml_type type, int64_t tokens) ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*graph); REQUIRE(imported.valid()); ggml::hrx::DispatchScheduler scheduler; - REQUIRE(scheduler.schedule_graph(imported.graph, { "gfx1151", prefer_row64 })); + REQUIRE(scheduler.schedule_graph(imported.graph, { target, prefer_row64 })); const ggml::hrx::Value * source = imported.graph.values().find_tensor(weight); REQUIRE(source != nullptr); const char * row64 = type == GGML_TYPE_Q4_K ? ggml::hrx::kQ4KPackedK256Row64Layout : ggml::hrx::kQ6KPackedK256Row64ScaleRowLayout; - const char * expected = policy.prefer_row64 ? row64 : ggml::hrx::kNativeWeightLayout; + // The existing Q4 prefill route already uses row64 with the policy off. + const bool legacy_row64 = type == GGML_TYPE_Q4_K && tokens == 512; + const char * expected = policy.prefer_row64 || legacy_row64 ? row64 : ggml::hrx::kNativeWeightLayout; size_t matched = 0; for (const auto & dispatch : scheduler.plan().dispatches) { for (const auto & binding : dispatch.bindings) { @@ -8992,7 +8996,7 @@ static void run_qwen27_matmul_policy_route_case(ggml_type type, int64_t tokens) const char * expected_kernel = policy.prefer_row64 ? "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_wave32" : "loom_libs:ggml_mul_mat_tiled_input_f32_publish_f32"; - REQUIRE(kernel_name_for_id(dispatch.kernel.kernel_id, "gfx1151") == expected_kernel); + REQUIRE(kernel_name_for_id(dispatch.kernel.kernel_id, target) == expected_kernel); } ++matched; } @@ -9126,11 +9130,12 @@ static void require_f32_dot_reference(ggml_type type, const std::vector require_close(observed_samples, reference_samples, absolute_tolerance, 0.02f); } -static void run_dense_matmul_f32_reference_case(ggml_type type, int64_t tokens, int64_t k = 512, - int64_t n = 128) { +static std::vector run_dense_matmul_f32_reference_case(ggml_type type, int64_t tokens, int64_t k = 512, + int64_t n = 128, bool prefer_row64 = true, + bool check_reference = true) { auto backend = ggml_backend_hrx_init(0); REQUIRE(backend != nullptr); - set_test_row64_policy(backend, true); + set_test_row64_policy(backend, prefer_row64); auto * backend_state = static_cast(backend->context); REQUIRE(backend_state != nullptr && backend_state->device != nullptr); const std::string architecture = backend_state->device->architecture; @@ -9146,15 +9151,16 @@ static void run_dense_matmul_f32_reference_case(ggml_type type, int64_t tokens, ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*graph); REQUIRE(imported.valid()); ggml::hrx::DispatchScheduler scheduler; - REQUIRE(scheduler.schedule_graph(imported.graph, { architecture, true })); + REQUIRE(scheduler.schedule_graph(imported.graph, { architecture, prefer_row64 })); const ggml::hrx::Value * source = imported.graph.values().find_tensor(weight); REQUIRE(source != nullptr); - const char * expected_layout = type == GGML_TYPE_Q4_K ? ggml::hrx::kQ4KPackedK256Row64Layout : + const char * expected_layout = !prefer_row64 ? ggml::hrx::kNativeWeightLayout : + type == GGML_TYPE_Q4_K ? ggml::hrx::kQ4KPackedK256Row64Layout : ggml::hrx::kQ6KPackedK256Row64ScaleRowLayout; const bool q6_weight = type == GGML_TYPE_Q6_K; const bool short_minimum = tokens >= 64; const bool short_maximum = tokens < 128; - const bool short_q6 = q6_weight && short_minimum && short_maximum; + const bool short_q6 = prefer_row64 && q6_weight && short_minimum && short_maximum; const bool prefill_minimum = tokens >= 128; const bool prefill_aligned = tokens % 128 == 0; const bool aligned_prefill = prefill_minimum && prefill_aligned; @@ -9162,6 +9168,7 @@ static void run_dense_matmul_f32_reference_case(ggml_type type, int64_t tokens, const char * expected_kernel = tokens == 1 ? "loom_libs:ggml_mul_mat_vector_f32_f32" : short_q6 ? "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_wave32" : aligned_q6_prefill ? "loom_libs:ggml_mul_mat_q6_k_f16_wmma_prefill_wave32" : + tokens % 32 == 0 ? "loom_libs:ggml_mul_mat_tiled_input_f32_publish_f32_aligned" : "loom_libs:ggml_mul_mat_tiled_input_f32_publish_f32"; size_t matched = 0; for (const auto & dispatch : scheduler.plan().dispatches) { @@ -9185,14 +9192,16 @@ static void run_dense_matmul_f32_reference_case(ggml_type type, int64_t tokens, REQUIRE(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS); ggml_backend_synchronize(backend); const auto observed = get_f32_tensor(backend, output); - std::vector rows; - for (int64_t row = 0; row < n; ++row) { - rows.push_back(row); - } - const std::vector sample_tokens = tokens == 1 ? std::vector{ 0 } : - std::vector{ 0, 21, 43, tokens - 1 }; - require_f32_dot_reference(type, weights, input, observed, k, n, rows, sample_tokens, - tokens == 1 ? "vector" : short_q6 ? "short-prefill" : "tiled", true); + if (check_reference) { + std::vector rows; + for (int64_t row = 0; row < n; ++row) { + rows.push_back(row); + } + const std::vector sample_tokens = tokens == 1 ? std::vector{ 0 } : + std::vector{ 0, 21, 43, tokens - 1 }; + require_f32_dot_reference(type, weights, input, observed, k, n, rows, sample_tokens, + tokens == 1 ? "vector" : short_q6 ? "short-prefill" : "tiled", true); + } ggml_backend_hrx_cache_stats stats = {}; REQUIRE(ggml_backend_hrx_get_cache_stats(backend, &stats)); REQUIRE(stats.weight_duplicate_warnings == 0); @@ -9201,6 +9210,7 @@ static void run_dense_matmul_f32_reference_case(ggml_type type, int64_t tokens, weight_buffer.reset(); ggml_free(ctx); ggml_backend_free(backend); + return observed; } static void run_ssm_conv_prefill_recurrent_case(ggml_type type, bool gathered, bool clear_state, @@ -9336,6 +9346,19 @@ static void register_basic_ops_cases(Suite & suite) { suite.host_case("qwen27.ssm_policy_route." + type_name(type) + ".interior", [type] { run_qwen27_ssm_policy_route_case(type, true); }); } + for (const char * target : { "gfx1100", "gfx1151" }) { + for (const int64_t hidden_size : { 1024, 2048, 2560 }) { + for (const ggml_type type : { GGML_TYPE_Q4_K, GGML_TYPE_Q6_K }) { + for (const int64_t tokens : { 1, 29, 63, 64, 66, 127, 128, 129, 512 }) { + suite.host_case("qwen3.matmul_policy_route." + std::string(target) + "." + type_name(type) + + ".k" + std::to_string(hidden_size) + ".tokens" + std::to_string(tokens), + [target, hidden_size, type, tokens] { + run_qwen27_matmul_policy_route_case(type, tokens, hidden_size, 1024, target); + }); + } + } + } + } suite.host_case("qwen27.q4_decode_policy_route", [] { run_qwen27_matmul_policy_route_case(GGML_TYPE_Q4_K, 1); }); @@ -9398,6 +9421,29 @@ static void register_basic_ops_cases(Suite & suite) { run_dense_matmul_f32_reference_case(type, 66); }); } + for (const int64_t hidden_size : { 1024, 2048, 2560 }) { + for (const ggml_type type : { GGML_TYPE_Q4_K, GGML_TYPE_Q6_K }) { + for (const int64_t tokens : { 1, 66, 128 }) { + // F16 accumulation at this width exceeds the F32 oracle in both layouts; compare them below. + if (type == GGML_TYPE_Q4_K && hidden_size == 2560 && tokens != 1) { + continue; + } + suite.device_case("qwen3.f32_reference." + type_name(type) + ".k" + + std::to_string(hidden_size) + ".tokens" + std::to_string(tokens), + [hidden_size, type, tokens] { + run_dense_matmul_f32_reference_case(type, tokens, hidden_size, 1024); + }); + } + } + } + for (const int64_t tokens : { 66, 128 }) { + suite.device_case("qwen3.q4_K.k2560.native_row64_equivalence.tokens" + std::to_string(tokens), [tokens] { + const auto native = run_dense_matmul_f32_reference_case(GGML_TYPE_Q4_K, tokens, 2560, 1024, false, false); + const auto row64 = run_dense_matmul_f32_reference_case(GGML_TYPE_Q4_K, tokens, 2560, 1024, true, false); + REQUIRE(native.size() == row64.size()); + REQUIRE(std::memcmp(native.data(), row64.data(), native.size() * sizeof(float)) == 0); + }); + } for (const int64_t tokens : { 63, 64, 127, 128, 129 }) { suite.device_case("qwen27.q6_short_boundary.tokens" + std::to_string(tokens), [tokens] { run_dense_matmul_f32_reference_case(GGML_TYPE_Q6_K, tokens); From 9c27b0a99ec9c6104a50b0714f2e61b58d76373d Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Wed, 11 Nov 2026 16:41:21 +0000 Subject: [PATCH 02/18] hrx: support row64 tied embedding and vocabulary projection Assisted-by: Codex --- .../common/dispatch-get-rows.cpp | 38 ++- .../qwen/dispatch-qwen-matmul.cpp | 19 +- .../qwen/dispatch-qwen-rmsnorm.cpp | 17 +- .../kernels/loom-libs/motifs/dequant.loom | 15 +- .../motifs/matmul/q6_q8_1_x4_decode.loom | 39 ++++ .../qwen_moe/ggml/linear_q6k_q8_1_x4.loom | 219 ++++++++++++++++++ .../kernels/qwen_moe/manifest.json | 145 +++++++++++- tests/hrx-ops-test.cpp | 203 +++++++++++++++- 8 files changed, 663 insertions(+), 32 deletions(-) diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp index 9ef56a6d0d15..4f3bae392d6c 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp @@ -103,12 +103,21 @@ struct GetRowsMatch { int64_t token_count = 0; int64_t row_count = 0; int64_t hidden_size = 0; + std::string weight_layout = kNativeWeightLayout; bool matched() const { return ids != nullptr && weight != nullptr && output != nullptr && weight_format_value >= 0; } }; +static DispatchBinding get_rows_weight_binding(const GetRowsMatch & match) { + if (match.weight_layout == kNativeWeightLayout) { + return { match.weight->id, 0, match.weight->byte_count }; + } + return { match.weight->id, 0, match.weight->byte_count, match.weight_layout, match.weight->type, + match.hidden_size, match.row_count, match.weight->byte_count }; +} + struct GetRowsRmsNormMatch { GetRowsMatch rows; const GraphNode * rms_node = nullptr; @@ -153,7 +162,7 @@ static void add_common_integer_parameters(Dispatch & dispatch, const GetRowsMatc static void add_primary_bindings(Dispatch & dispatch, const GetRowsMatch & match) { dispatch.bindings.push_back({ match.ids->id, 0, match.ids->byte_count }); - dispatch.bindings.push_back({ match.weight->id, 0, match.weight->byte_count }); + dispatch.bindings.push_back(get_rows_weight_binding(match)); dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count }); } @@ -256,8 +265,10 @@ static bool match_get_rows_scalar_3d_f32_dispatch(const DispatchMatchContext & c return true; } -static GetRowsMatch match_get_rows_f32(const Graph & graph, const GraphNode * node) { +static GetRowsMatch match_get_rows_f32(const DispatchMatchContext & context) { GetRowsMatch match; + const Graph & graph = context.graph; + const GraphNode * node = context.root_node; if (node == nullptr || node->op != GGML_OP_GET_ROWS || node->inputs.size() != 2) { return match; } @@ -293,12 +304,23 @@ static GetRowsMatch match_get_rows_f32(const Graph & graph, const GraphNode * no match.token_count = token_count; match.row_count = row_count; match.hidden_size = hidden_size; + if (context.row64_weight_policy && weight->alias_source.value < 0 && row_count % 64 == 0 && + hidden_size % 256 == 0 && + dispatch_weight_layout_eligible(*weight)) { + if (weight->type == GGML_TYPE_Q4_K) { + match.weight_format_value = common_mul_mat_format_config_value(CommonMulMatWeightFormat::Q4KRow64); + match.weight_layout = kQ4KPackedK256Row64Layout; + } else if (weight->type == GGML_TYPE_Q6_K) { + match.weight_format_value = common_mul_mat_format_config_value(CommonMulMatWeightFormat::Q6KRow64); + match.weight_layout = kQ6KPackedK256Row64ScaleRowLayout; + } + } return match; } static GetRowsScaleMatch match_get_rows_scale_f32(const DispatchMatchContext & context) { GetRowsScaleMatch match; - match.rows = match_get_rows_f32(context.graph, context.root_node); + match.rows = match_get_rows_f32(context); if (!match.rows.matched() || !context.graph.has_index()) { return {}; } @@ -345,7 +367,7 @@ static GetRowsScaleMatch match_get_rows_scale_f32(const DispatchMatchContext & c static GetRowsRmsNormMatch match_get_rows_rmsnorm_binary(const DispatchMatchContext & context) { GetRowsRmsNormMatch match; - match.rows = match_get_rows_f32(context.graph, context.root_node); + match.rows = match_get_rows_f32(context); if (!match.rows.matched() || !context.graph.has_index() || match.rows.hidden_size < 128 || match.rows.hidden_size > 32768 || match.rows.hidden_size % 128 != 0) { return {}; @@ -433,7 +455,7 @@ static Dispatch make_get_rows_next_dispatch(const GetRowsMatch & match, } static bool match_get_rows_f32_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) { - const GetRowsMatch match = match_get_rows_f32(context.graph, context.root_node); + const GetRowsMatch match = match_get_rows_f32(context); if (!match.matched()) { return false; } @@ -458,7 +480,7 @@ static bool match_get_rows_scale_f32_dispatch(const DispatchMatchContext & conte dispatch.kernel.compile_parameters.emplace("ggml.get_rows_scale_f32.scale", to_config_value(match.scale)); dispatch.kernel.compile_parameters.emplace("ggml.get_rows_scale_f32.bias", to_config_value(match.bias)); dispatch.bindings.push_back({ match.rows.ids->id, 0, match.rows.ids->byte_count }); - dispatch.bindings.push_back({ match.rows.weight->id, 0, match.rows.weight->byte_count }); + dispatch.bindings.push_back(get_rows_weight_binding(match.rows)); dispatch.bindings.push_back({ match.rows.output->id, 0, match.rows.output->byte_count }); dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count }); @@ -469,7 +491,7 @@ static bool match_get_rows_scale_f32_dispatch(const DispatchMatchContext & conte } static bool match_get_rows_f32_next_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) { - const GetRowsMatch match = match_get_rows_f32(context.graph, context.root_node); + const GetRowsMatch match = match_get_rows_f32(context); if (!match.matched()) { return false; } @@ -558,7 +580,7 @@ static bool match_get_rows_rmsnorm_binary_dispatch(const DispatchMatchContext & add_common_compile_parameters(dispatch, match.rows); dispatch.kernel.compile_parameters.emplace("ggml.get_rows_rmsnorm.rms_epsilon", to_config_value(match.epsilon)); dispatch.bindings.push_back({ match.rows.ids->id, 0, match.rows.ids->byte_count }); - dispatch.bindings.push_back({ match.rows.weight->id, 0, match.rows.weight->byte_count }); + dispatch.bindings.push_back(get_rows_weight_binding(match.rows)); dispatch.bindings.push_back({ match.norm_weight->id, 0, match.norm_weight->byte_count }); dispatch.bindings.push_back({ match.rows.output->id, 0, match.rows.output->byte_count }); dispatch.bindings.push_back({ match.normalized_output->id, 0, match.normalized_output->byte_count }); diff --git a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp index fcd7597bb721..3c3dabaaf5da 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp @@ -18,6 +18,8 @@ static constexpr KernelCatalogRef kQwenDenseLinearQ4KQ8NextQ8Kernel = GGML_HRX_KERNEL_REF("qwen3_moe", "qwen3_moe_dense_linear_q4k_q8_1_x4_next_q8"); static constexpr KernelCatalogRef kGgmlLinearQ6KQ8_1X4Kernel = GGML_HRX_KERNEL_REF("qwen3_moe", "ggml_linear_q6k_q8_1_x4"); +static constexpr KernelCatalogRef kGgmlLinearQ6KQ8_1X4Row64Kernel = + GGML_HRX_KERNEL_REF("qwen3_moe", "ggml_linear_q6k_q8_1_x4_row64"); static constexpr int64_t kQwenHiddenSize = kQwen30BMoeDispatchProfile.hidden_size; static constexpr int64_t kQwenVocabularyCount = 151936; @@ -338,9 +340,12 @@ bool qwen_accepts_q6k_q8_input(const DispatchMatchContext & context, static void build_qwen_matmul_dispatch(const QwenMatmulMatch & match, DispatchMatch & dispatch_match, - size_t root_index) { + size_t root_index, + bool row64_weight_policy) { Dispatch dispatch; - dispatch.kernel = make_kernel_specialization(match.kernel); + const bool row64_weight = row64_weight_policy && match.weight->alias_source.value < 0 && + dispatch_weight_layout_eligible(*match.weight); + dispatch.kernel = make_kernel_specialization(row64_weight ? kGgmlLinearQ6KQ8_1X4Row64Kernel : match.kernel); dispatch.kernel.integer_parameters.emplace("token_count", match.token_count); dispatch.kernel.integer_parameters.emplace("input_size", match.input_size); dispatch.kernel.integer_parameters.emplace("output_size", match.output_size); @@ -349,7 +354,13 @@ static void build_qwen_matmul_dispatch(const QwenMatmulMatch & match, dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.output_capacity", to_config_value(match.output_size)); dispatch.bindings.push_back({ match.input_value, 0, match.input_bytes }); - dispatch.bindings.push_back({ match.weight->id, 0, match.weight->byte_count }); + if (row64_weight) { + dispatch.bindings.push_back({ match.weight->id, 0, match.weight->byte_count, + kQ6KPackedK256Row64ScaleRowLayout, match.weight->type, match.input_size, + match.output_size, match.weight->byte_count }); + } else { + dispatch.bindings.push_back({ match.weight->id, 0, match.weight->byte_count }); + } dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count }); dispatch_match.covered_nodes.push_back(root_index); @@ -361,7 +372,7 @@ static bool match_qwen_q6k_q8_dispatch(const DispatchMatchContext & context, Dis if (!match.matched()) { return false; } - build_qwen_matmul_dispatch(match, dispatch_match, context.root_index); + build_qwen_matmul_dispatch(match, dispatch_match, context.root_index, context.row64_weight_policy); return true; } diff --git a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-rmsnorm.cpp b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-rmsnorm.cpp index dce10fa60c64..7e01675f2661 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-rmsnorm.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-rmsnorm.cpp @@ -19,6 +19,8 @@ static constexpr KernelCatalogRef kQwenRmsNormF32QuantizeQ8_1X4Kernel = GGML_HRX_KERNEL_REF("qwen3_moe", "qwen3_moe_rmsnorm_f32_quantize_q8_1_x4"); static constexpr KernelCatalogRef kGgmlLinearQ6KQ8_1X4Kernel = GGML_HRX_KERNEL_REF("qwen3_moe", "ggml_linear_q6k_q8_1_x4"); +static constexpr KernelCatalogRef kGgmlLinearQ6KQ8_1X4Row64Kernel = + GGML_HRX_KERNEL_REF("qwen3_moe", "ggml_linear_q6k_q8_1_x4_row64"); static constexpr float kQwenRmsNormEpsilon = kQwen30BMoeDispatchProfile.rms_norm_epsilon; static constexpr int64_t kQwenHiddenSize = kQwen30BMoeDispatchProfile.hidden_size; static constexpr int64_t kQwenVocabularyCount = 151936; @@ -285,7 +287,10 @@ static bool match_qwen_rmsnorm_f32_quantize_q8_1_x4_dispatch(const DispatchMatch rms_dispatch.bindings.push_back({ q8_value, 0, q8_byte_count }); Dispatch projection_dispatch; - projection_dispatch.kernel = make_kernel_specialization(kGgmlLinearQ6KQ8_1X4Kernel); + const bool row64_weight = context.row64_weight_policy && projection_match.weight->alias_source.value < 0 && + dispatch_weight_layout_eligible(*projection_match.weight); + projection_dispatch.kernel = make_kernel_specialization( + row64_weight ? kGgmlLinearQ6KQ8_1X4Row64Kernel : kGgmlLinearQ6KQ8_1X4Kernel); projection_dispatch.kernel.integer_parameters.emplace("token_count", rms_match.token_count); projection_dispatch.kernel.integer_parameters.emplace("input_size", rms_match.hidden_size); projection_dispatch.kernel.integer_parameters.emplace("output_size", kQwenVocabularyCount); @@ -294,7 +299,15 @@ static bool match_qwen_rmsnorm_f32_quantize_q8_1_x4_dispatch(const DispatchMatch projection_dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.output_capacity", to_config_value(kQwenVocabularyCount)); projection_dispatch.bindings.push_back({ q8_value, 0, q8_byte_count }); - projection_dispatch.bindings.push_back({ projection_match.weight->id, 0, projection_match.weight->byte_count }); + if (row64_weight) { + projection_dispatch.bindings.push_back({ projection_match.weight->id, 0, + projection_match.weight->byte_count, + kQ6KPackedK256Row64ScaleRowLayout, projection_match.weight->type, + rms_match.hidden_size, kQwenVocabularyCount, + projection_match.weight->byte_count }); + } else { + projection_dispatch.bindings.push_back({ projection_match.weight->id, 0, projection_match.weight->byte_count }); + } projection_dispatch.bindings.push_back({ projection_match.output->id, 0, projection_match.output->byte_count }); match.covered_nodes.push_back(rms_match.rms_node_index); diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/dequant.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/dequant.loom index 3914d5f07574..15b9e70d9ee2 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/dequant.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/dequant.loom @@ -2710,7 +2710,7 @@ func.def inline @ggml_q6k_f32_vector4(%weight: buffer, %weight_row_byte_base: of func.return %values : vector<4xf32> } -func.def inline @ggml_q6k_row64_tile_lane_f32_vector4(%weight: buffer, %tile: index, %row_lane: index, %q6_group: index, %packet: index) -> (vector<4xf32>) { +func.def inline @ggml_q6k_row64_tile_lane_i8_scale(%weight: buffer, %tile: index, %row_lane: index, %q6_group: index, %packet: index) -> (vector<4xi8>, f32) { %c2 = index.constant 2 : index %c4 = index.constant 4 : index %tile_bytes = index.constant 13440 : offset @@ -2719,7 +2719,7 @@ func.def inline @ggml_q6k_row64_tile_lane_f32_vector4(%weight: buffer, %tile: in %c4_i32v = vector.splati 4 : vector<1xi32> %nibble_mask = vector.splati 252645135 : vector<1xi32> %high_mask = vector.splati 50529027 : vector<1xi32> - %c32_f32v = vector.splatf 32.0 : vector<4xf32> + %c32_i8v = vector.splati 32 : vector<4xi8> %bounded_group = index.assume %q6_group [range(%q6_group, 0, 7)] : index %bounded_packet = index.assume %packet [range(%packet, 0, 7)] : index %tile_base = index.scale %tile, %tile_bytes : index, offset -> offset @@ -2748,15 +2748,20 @@ func.def inline @ggml_q6k_row64_tile_lane_f32_vector4(%weight: buffer, %tile: in %qh = vector.shli %qh_low, %c4_i32v : vector<1xi32> %code = vector.ori %ql, %qh : vector<1xi32> %code_i8 = vector.bitcast %code : vector<1xi32> to vector<4xi8> - %code_f32 = vector.uitofp %code_i8 : vector<4xi8> to vector<4xf32> - %centered = vector.subf %code_f32, %c32_f32v : vector<4xf32> + %centered = vector.subi %code_i8, %c32_i8v : vector<4xi8> %scale_i8 = view.load %scale_view[%row_lane, %scale_packet_half, %bounded_group] : view<64x2x8xi8> -> i8 %d_f16 = view.load %d_view[%row_lane] : view<64xf16> -> f16 %scale = scalar.sitofp %scale_i8 : i8 to f32 %d = scalar.extf %d_f16 : f16 to f32 %combined_scale = scalar.mulf %scale, %d : f32 + func.return %centered, %combined_scale : vector<4xi8>, f32 +} + +func.def inline @ggml_q6k_row64_tile_lane_f32_vector4(%weight: buffer, %tile: index, %row_lane: index, %q6_group: index, %packet: index) -> (vector<4xf32>) { + %centered, %combined_scale = func.call @ggml_q6k_row64_tile_lane_i8_scale(%weight, %tile, %row_lane, %q6_group, %packet) : (buffer, index, index, index, index) -> (vector<4xi8>, f32) + %centered_f32 = vector.sitofp %centered : vector<4xi8> to vector<4xf32> %combined_scale_vector = vector.splat %combined_scale : vector<4xf32> - %values = vector.mulf %centered, %combined_scale_vector : vector<4xf32> + %values = vector.mulf %centered_f32, %combined_scale_vector : vector<4xf32> func.return %values : vector<4xf32> } diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom index c906e25b77bc..5b2c2ddf5540 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom @@ -7,6 +7,45 @@ func.decl @ggml_q8_1_x4_word(%q8_input: buffer, %row_byte_base: offset, %q8_bloc func.decl @ggml_q6k_f16_vector4(%weight: buffer, %weight_row_byte_base: offset, %q6_block: index, %q6_group: index, %packet: index) -> (vector<4xf16>) +func.decl @ggml_q6k_row64_tile_lane_i8_scale(%weight: buffer, %tile: index, %row_lane: index, %q6_group: index, %packet: index) -> (vector<4xi8>, f32) + +func.def inline @ggml_q6k_q8_1_x4_row64_row_lane(%input_size: index, %row: index, %weight: buffer, %q8_input: buffer, %q8_row_byte_base: offset, %lane: index) -> (f32) { + %bounded_input_size = index.assume %input_size [range(%input_size, 256, 32768), multiple_of(%input_size, 256)] : index + %bounded_lane = index.assume %lane [range(%lane, 0, 31)] : index + %c0 = index.constant 0 : index + %c1 = index.constant 1 : index + %c4 = index.constant 4 : index + %c8 = index.constant 8 : index + %c64 = index.constant 64 : index + %c256 = index.constant 256 : index + %zero = scalar.constant 0.0 : f32 + %zero_i32v = vector.constant 0 : vector<1xi32> + %block_count = index.div %bounded_input_size, %c256 : index + %row_group = index.div %row, %c64 : index + %row_lane = index.rem %row, %c64 : index + %group_low = index.div %bounded_lane, %c8 : index + %packet = index.rem %bounded_lane, %c8 : index + %sum = scf.for %block = [%c0 to %block_count step %c1](%block_acc = %zero : f32) -> (f32) { + %block_q8_base = index.mul %block, %c8 : index + %tile = index.madd %row_group, %block_count, %block : index + %partial = scf.for %half = [%c0 to %c8 step %c4](%half_acc = %block_acc : f32) -> (f32) { + %group = index.add %group_low, %half : index + %weight_values, %weight_scale = func.call @ggml_q6k_row64_tile_lane_i8_scale(%weight, %tile, %row_lane, %group, %packet) : (buffer, index, index, index, index) -> (vector<4xi8>, f32) + %q8_block = index.add %block_q8_base, %group : index + %q8_values, %q8_d = func.call @ggml_q8_1_x4_word(%q8_input, %q8_row_byte_base, %q8_block, %packet) : (buffer, offset, index, index) -> (vector<4xi8>, f32) + %dot_i32v = vector.dot4i %weight_values, %q8_values, %zero_i32v : vector<4xi8>, vector<4xi8>, vector<1xi32> + %dot_i32 = vector.extract %dot_i32v[0] : vector<1xi32> -> i32 + %dot = scalar.sitofp %dot_i32 : i32 to f32 + %scaled_weight = scalar.mulf %dot, %weight_scale : f32 + %scaled = scalar.mulf %scaled_weight, %q8_d : f32 + %next = scalar.addf %half_acc, %scaled : f32 + scf.yield %next : f32 + } + scf.yield %partial : f32 + } + func.return %sum : f32 +} + func.def inline @ggml_q6k_sign_extend_dot4(%code: vector<1xi32>) -> (vector<4xi8>) { %c1_i32v = vector.splati 1 : vector<1xi32> %c2_i32v = vector.splati 2 : vector<1xi32> diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom index 55da51a6161a..86bef8d4559f 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom @@ -27,6 +27,8 @@ func.decl @ggml_q8_1_x4_word(%q8_input: buffer, %row_byte_base: offset, %q8_bloc func.decl @ggml_q6k_q8_1_x4_row_lane_wave64_block4(%input_size: index, %weight: buffer, %weight_row_byte_base: offset, %q8_input: buffer, %q8_row_byte_base: offset, %lane: index) -> (f32) +func.decl @ggml_q6k_q8_1_x4_row64_row_lane(%input_size: index, %row: index, %weight: buffer, %q8_input: buffer, %q8_row_byte_base: offset, %lane: index) -> (f32) + // Dense raw-layout baseline. A 256-thread workgroup carries eight independent // wave32 output rows so decode launches enough waves without duplicating Q6_K // decoding within a row. @@ -67,6 +69,59 @@ kernel.def export("ggml_linear_q6k_q8_1_x4") @ggml_linear_q6k_q8_1_x4(%token_cou kernel.return } +kernel.def export("ggml_linear_q6k_q8_1_x4_row64") @ggml_linear_q6k_q8_1_x4_row64(%token_count: index, %input_size: index, %output_size: index) { + %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index + %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index + %c8 = index.constant 8 : index + %c7 = index.constant 7 : index + %c1 = index.constant 1 : index + %workgroup_size = index.constant 256 : index + %padded_output_size = index.add %output_capacity, %c7 : index + %output_tiles = index.div %padded_output_size, %c8 : index + kernel.launch.config workgroups(%output_tiles, %token_capacity, %c1) workgroup_size(%workgroup_size, %c1, %c1) : index +} launch(%token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) { + %q8_noalias, %weight_noalias, %output_noalias = buffer.assume.noalias %q8_input, %weight, %output : buffer, buffer, buffer + %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index + %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index + %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048), le(%token_count, %token_capacity)] : index + %bounded_input_size = index.assume %input_size [range(%input_size, 256, 32768), multiple_of(%input_size, 256)] : index + %bounded_output_size = index.assume %output_size [range(%output_size, 1, 262144), le(%output_size, %output_capacity), multiple_of(%output_size, 64)] : index + %c0 = index.constant 0 : index + %c8 = index.constant 8 : index + %c128 = index.constant 128 : index + %c144_bytes = index.constant 144 : offset + %token = kernel.workgroup.id : index + %channel_tile = kernel.workgroup.id : index + %subgroup = kernel.subgroup.id : index + %lane = kernel.subgroup.lane.id : index + %channel = index.madd %channel_tile, %c8, %subgroup : index + %valid_token = index.cmp ult, %token, %bounded_token_count : index + %valid_channel = index.cmp ult, %channel, %bounded_output_size : index + %publish = scalar.andi %valid_token, %valid_channel : i1 + %safe_token = scf.select %valid_token, %token, %c0 : index + %safe_channel = scf.select %valid_channel, %channel, %c0 : index + %q8_groups = index.div %bounded_input_size, %c128 : index + %q8_row_bytes = index.scale %q8_groups, %c144_bytes : index, offset -> offset + %q8_row_byte_base = index.scale %safe_token, %q8_row_bytes : index, offset -> offset + %lane_acc = scf.if %publish -> (f32) { + %value = func.call @ggml_q6k_q8_1_x4_row64_row_lane(%bounded_input_size, %safe_channel, %weight_noalias, %q8_noalias, %q8_row_byte_base, %lane) : (index, index, buffer, buffer, offset, index) -> (f32) + scf.yield %value : f32 + } else { + %zero = scalar.constant 0.0 : f32 + scf.yield %zero : f32 + } + %dot = kernel.subgroup.reduce %lane_acc : f32 + %is_lane_zero = index.cmp eq, %lane, %c0 : index + scf.if %publish { + scf.if %is_lane_zero { + %c0_offset = index.constant 0 : offset + %output_view = buffer.view %output_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]x[%bounded_output_size]xf32> + view.store %dot, %output_view[%token, %channel] : f32, view<[%bounded_token_count]x[%bounded_output_size]xf32> + } + } + kernel.return +} + // Wave64 vocabulary schedule matching llama.cpp Vulkan's one-row workgroup and // four-block concurrency on AMD targets with native wave64 execution. kernel.def target(@ggml_q6k_q8_gfx1151_wave64) export("ggml_linear_q6k_q8_1_x4") @ggml_linear_q6k_q8_1_x4_wave64_block4(%token_count: index, %input_size: index, %output_size: index) { @@ -183,6 +238,19 @@ check.case public @ggml_linear_q6k_q8_1_x4_vocabulary_wave32_benchmark_case { check.return } +check.case public @ggml_linear_q6k_q8_1_x4_vocabulary_prefill_benchmark_case { + %token_count = check.literal value(17) : index + %input_size = check.literal value(2048) : index + %output_size = check.literal value(151936) : index + %q8_input = check.generate.fill value(1) : tensor<17x2304xi8> + %weight = check.generate.fill value(0) : tensor<151936x8x210xi8> + %output = check.generate.fill value(1.0) : tensor<17x151936xf32> + %expected = check.generate.fill value(0.0) : tensor<17x151936xf32> + kernel.launch @ggml_linear_q6k_q8_1_x4[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<17x2304xi8>, tensor<151936x8x210xi8>, tensor<17x151936xf32>) + check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<17x151936xf32>, tensor<17x151936xf32> + check.return +} + check.case public @ggml_linear_q6k_q8_1_x4_vocabulary_wave64_block4_benchmark_case { %token_count = check.literal value(1) : index %input_size = check.literal value(2048) : index @@ -196,6 +264,32 @@ check.case public @ggml_linear_q6k_q8_1_x4_vocabulary_wave64_block4_benchmark_ca check.return } +check.case public @ggml_linear_q6k_q8_1_x4_row64_vocabulary_benchmark_case { + %token_count = check.literal value(1) : index + %input_size = check.literal value(2048) : index + %output_size = check.literal value(151936) : index + %q8_input = check.generate.fill value(1) : tensor<1x2304xi8> + %weight = check.generate.fill value(0) : tensor<2374x8x13440xi8> + %output = check.generate.fill value(1.0) : tensor<1x151936xf32> + %expected = check.generate.fill value(0.0) : tensor<1x151936xf32> + kernel.launch @ggml_linear_q6k_q8_1_x4_row64[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<1x2304xi8>, tensor<2374x8x13440xi8>, tensor<1x151936xf32>) + check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<1x151936xf32>, tensor<1x151936xf32> + check.return +} + +check.case public @ggml_linear_q6k_q8_1_x4_row64_vocabulary_prefill_benchmark_case { + %token_count = check.literal value(17) : index + %input_size = check.literal value(2048) : index + %output_size = check.literal value(151936) : index + %q8_input = check.generate.fill value(1) : tensor<17x2304xi8> + %weight = check.generate.fill value(0) : tensor<2374x8x13440xi8> + %output = check.generate.fill value(1.0) : tensor<17x151936xf32> + %expected = check.generate.fill value(0.0) : tensor<17x151936xf32> + kernel.launch @ggml_linear_q6k_q8_1_x4_row64[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<17x2304xi8>, tensor<2374x8x13440xi8>, tensor<17x151936xf32>) + check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<17x151936xf32>, tensor<17x151936xf32> + check.return +} + // Exercises the complete activation-pack and dense Q6_K projection boundary at // the K=2048, M=512 shape used by mixed-format attention V weights. The token // buckets cover decode, awkward prefill tails, llama.cpp's 512-token @@ -221,8 +315,14 @@ check.benchmark<@ggml_linear_q6k_q8_1_x4_benchmark_case> @ggml_linear_q6k_q8_1_x check.benchmark<@ggml_linear_q6k_q8_1_x4_vocabulary_wave32_benchmark_case> @ggml_linear_q6k_q8_1_x4_vocabulary_wave32 +check.benchmark<@ggml_linear_q6k_q8_1_x4_vocabulary_prefill_benchmark_case> @ggml_linear_q6k_q8_1_x4_vocabulary_prefill_17 + check.benchmark<@ggml_linear_q6k_q8_1_x4_vocabulary_wave64_block4_benchmark_case> @ggml_linear_q6k_q8_1_x4_vocabulary_wave64_block4 +check.benchmark<@ggml_linear_q6k_q8_1_x4_row64_vocabulary_benchmark_case> @ggml_linear_q6k_q8_1_x4_row64_vocabulary_decode + +check.benchmark<@ggml_linear_q6k_q8_1_x4_row64_vocabulary_prefill_benchmark_case> @ggml_linear_q6k_q8_1_x4_row64_vocabulary_prefill_17 + check.benchmark<@ggml_linear_q6k_q8_1_x4_benchmark_case> @ggml_linear_q6k_q8_1_x4_prefill_32 {token_count = 32} check.benchmark<@ggml_linear_q6k_q8_1_x4_benchmark_case> @ggml_linear_q6k_q8_1_x4_prefill_128 {token_count = 128} @@ -240,3 +340,122 @@ check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6 check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6k_q8_1_x4_dense_v_prefill_1024 {token_count = 1024} check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6k_q8_1_x4_dense_v_prefill_2048 {token_count = 2048} + +amdgpu.target @ggml_q6k_row64_gfx1100_wave64 {subgroup_size = 64} + +func.decl @ggml_q6k_row64_tile_lane_i8_scale(%weight: buffer, %tile: index, %row_lane: index, %q6_group: index, %packet: index) -> (vector<4xi8>, f32) + +func.def inline @ggml_q6k_q8_1_x4_row64_row_lane_wave64(%input_size: index, %row: index, %weight: buffer, %q8_input: buffer, %q8_row_byte_base: offset, %lane: index) -> (f32) { + %bounded_input_size = index.assume %input_size [range(%input_size, 256, 32768), multiple_of(%input_size, 256)] : index + %bounded_lane = index.assume %lane [range(%lane, 0, 63)] : index + %c0 = index.constant 0 : index + %c1 = index.constant 1 : index + %c8 = index.constant 8 : index + %c64 = index.constant 64 : index + %c256 = index.constant 256 : index + %zero = scalar.constant 0.0 : f32 + %zero_i32v = vector.constant 0 : vector<1xi32> + %block_count = index.div %bounded_input_size, %c256 : index + %row_group = index.div %row, %c64 : index + %row_lane = index.rem %row, %c64 : index + %group = index.div %bounded_lane, %c8 : index + %packet = index.rem %bounded_lane, %c8 : index + %sum = scf.for %block = [%c0 to %block_count step %c1](%block_acc = %zero : f32) -> (f32) { + %tile = index.madd %row_group, %block_count, %block : index + %block_q8_base = index.mul %block, %c8 : index + %weight_values, %weight_scale = func.call @ggml_q6k_row64_tile_lane_i8_scale(%weight, %tile, %row_lane, %group, %packet) : (buffer, index, index, index, index) -> (vector<4xi8>, f32) + %q8_block = index.add %block_q8_base, %group : index + %q8_values, %q8_d = func.call @ggml_q8_1_x4_word(%q8_input, %q8_row_byte_base, %q8_block, %packet) : (buffer, offset, index, index) -> (vector<4xi8>, f32) + %dot_i32v = vector.dot4i %weight_values, %q8_values, %zero_i32v : vector<4xi8>, vector<4xi8>, vector<1xi32> + %dot_i32 = vector.extract %dot_i32v[0] : vector<1xi32> -> i32 + %dot = scalar.sitofp %dot_i32 : i32 to f32 + %scaled_weight = scalar.mulf %dot, %weight_scale : f32 + %scaled = scalar.mulf %scaled_weight, %q8_d : f32 + %next = scalar.addf %block_acc, %scaled : f32 + scf.yield %next : f32 + } + func.return %sum : f32 +} + +kernel.def target(@ggml_q6k_row64_gfx1100_wave64) export("ggml_linear_q6k_q8_1_x4_row64") @ggml_linear_q6k_q8_1_x4_row64_wave64(%token_count: index, %input_size: index, %output_size: index) { + %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index + %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index + %c3 = index.constant 3 : index + %c4 = index.constant 4 : index + %c1 = index.constant 1 : index + %workgroup_size = index.constant 256 : index + %padded_output_size = index.add %output_capacity, %c3 : index + %output_tiles = index.div %padded_output_size, %c4 : index + kernel.launch.config workgroups(%output_tiles, %token_capacity, %c1) workgroup_size(%workgroup_size, %c1, %c1) : index +} launch(%token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) { + %q8_noalias, %weight_noalias, %output_noalias = buffer.assume.noalias %q8_input, %weight, %output : buffer, buffer, buffer + %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index + %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index + %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048), le(%token_count, %token_capacity)] : index + %bounded_input_size = index.assume %input_size [range(%input_size, 256, 32768), multiple_of(%input_size, 256)] : index + %bounded_output_size = index.assume %output_size [range(%output_size, 1, 262144), le(%output_size, %output_capacity), multiple_of(%output_size, 64)] : index + %c0 = index.constant 0 : index + %c4 = index.constant 4 : index + %c128 = index.constant 128 : index + %c144_bytes = index.constant 144 : offset + %token = kernel.workgroup.id : index + %channel_tile = kernel.workgroup.id : index + %subgroup = kernel.subgroup.id : index + %lane = kernel.subgroup.lane.id : index + %channel = index.madd %channel_tile, %c4, %subgroup : index + %valid_token = index.cmp ult, %token, %bounded_token_count : index + %valid_channel = index.cmp ult, %channel, %bounded_output_size : index + %publish = scalar.andi %valid_token, %valid_channel : i1 + %safe_token = scf.select %valid_token, %token, %c0 : index + %safe_channel = scf.select %valid_channel, %channel, %c0 : index + %q8_groups = index.div %bounded_input_size, %c128 : index + %q8_row_bytes = index.scale %q8_groups, %c144_bytes : index, offset -> offset + %q8_row_byte_base = index.scale %safe_token, %q8_row_bytes : index, offset -> offset + %lane_acc = scf.if %publish -> (f32) { + %value = func.call @ggml_q6k_q8_1_x4_row64_row_lane_wave64(%bounded_input_size, %safe_channel, %weight_noalias, %q8_noalias, %q8_row_byte_base, %lane) : (index, index, buffer, buffer, offset, index) -> (f32) + scf.yield %value : f32 + } else { + %zero = scalar.constant 0.0 : f32 + scf.yield %zero : f32 + } + %dot = kernel.subgroup.reduce %lane_acc : f32 + %is_lane_zero = index.cmp eq, %lane, %c0 : index + scf.if %publish { + scf.if %is_lane_zero { + %c0_offset = index.constant 0 : offset + %output_view = buffer.view %output_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]x[%bounded_output_size]xf32> + view.store %dot, %output_view[%token, %channel] : f32, view<[%bounded_token_count]x[%bounded_output_size]xf32> + } + } + kernel.return +} + +check.case public @ggml_linear_q6k_q8_1_x4_row64_wave64_benchmark_case { + %token_count = check.literal value(1) : index + %input_size = check.literal value(2048) : index + %output_size = check.literal value(151936) : index + %q8_input = check.generate.fill value(1) : tensor<1x2304xi8> + %weight = check.generate.fill value(0) : tensor<2374x8x13440xi8> + %output = check.generate.fill value(1.0) : tensor<1x151936xf32> + %expected = check.generate.fill value(0.0) : tensor<1x151936xf32> + kernel.launch @ggml_linear_q6k_q8_1_x4_row64_wave64[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<1x2304xi8>, tensor<2374x8x13440xi8>, tensor<1x151936xf32>) + check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<1x151936xf32>, tensor<1x151936xf32> + check.return +} + +check.benchmark<@ggml_linear_q6k_q8_1_x4_row64_wave64_benchmark_case> @ggml_linear_q6k_q8_1_x4_row64_wave64_decode + +check.case public @ggml_linear_q6k_q8_1_x4_row64_wave64_prefill_benchmark_case { + %token_count = check.literal value(17) : index + %input_size = check.literal value(2048) : index + %output_size = check.literal value(151936) : index + %q8_input = check.generate.fill value(1) : tensor<17x2304xi8> + %weight = check.generate.fill value(0) : tensor<2374x8x13440xi8> + %output = check.generate.fill value(1.0) : tensor<17x151936xf32> + %expected = check.generate.fill value(0.0) : tensor<17x151936xf32> + kernel.launch @ggml_linear_q6k_q8_1_x4_row64_wave64[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<17x2304xi8>, tensor<2374x8x13440xi8>, tensor<17x151936xf32>) + check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<17x151936xf32>, tensor<17x151936xf32> + check.return +} + +check.benchmark<@ggml_linear_q6k_q8_1_x4_row64_wave64_prefill_benchmark_case> @ggml_linear_q6k_q8_1_x4_row64_wave64_prefill_17 diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json index 32fee80b9ee1..23a760375e66 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json @@ -2,6 +2,9 @@ "schema": "ggml-hrx-kernel-corpus-v1", "upstream_revision": "local", "files": [ + { + "path": "../loom-libs/motifs/dequant.loom" + }, { "path": "../loom-libs/motifs/q6_k_f16.loom" }, @@ -175,14 +178,146 @@ "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", "../loom-libs/motifs/q6_k_f16.loom", "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom" + "../loom-libs/motifs/quantize_q8_1_x4_group.loom", + "../loom-libs/motifs/dequant.loom" ] }, "compile_dependencies": [ "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", "../loom-libs/motifs/q6_k_f16.loom", "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom" + "../loom-libs/motifs/quantize_q8_1_x4_group.loom", + "../loom-libs/motifs/dequant.loom" + ] + }, + { + "name": "ggml_linear_q6k_q8_1_x4_row64", + "symbol": "ggml_linear_q6k_q8_1_x4_row64", + "source": "ggml/linear_q6k_q8_1_x4.loom", + "workload_parameters": [ + { + "name": "token_count", + "type": "index" + }, + { + "name": "input_size", + "type": "index" + }, + { + "name": "output_size", + "type": "index" + } + ], + "launch_parameters": [ + { + "name": "token_count", + "type": "index" + }, + { + "name": "input_size", + "type": "index" + }, + { + "name": "output_size", + "type": "index" + } + ], + "bindings": [ + "q8_input", + "weight", + "output" + ], + "binding_access": [ + "read", + "read", + "read_write" + ], + "target_selector": "", + "compile_recipe": { + "mode": "archive", + "link_module": "linear_q6k_q8_1_x4_linked", + "primary_sources": [ + "ggml/linear_q6k_q8_1_x4.loom" + ], + "library_sources": [ + "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", + "../loom-libs/motifs/q6_k_f16.loom", + "ggml/quantize_q8_1_x4.loom", + "../loom-libs/motifs/quantize_q8_1_x4_group.loom", + "../loom-libs/motifs/dequant.loom" + ] + }, + "compile_dependencies": [ + "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", + "../loom-libs/motifs/q6_k_f16.loom", + "ggml/quantize_q8_1_x4.loom", + "../loom-libs/motifs/quantize_q8_1_x4_group.loom", + "../loom-libs/motifs/dequant.loom" + ] + }, + { + "name": "ggml_linear_q6k_q8_1_x4_row64", + "symbol": "ggml_linear_q6k_q8_1_x4_row64_wave64", + "source": "ggml/linear_q6k_q8_1_x4.loom", + "workload_parameters": [ + { + "name": "token_count", + "type": "index" + }, + { + "name": "input_size", + "type": "index" + }, + { + "name": "output_size", + "type": "index" + } + ], + "launch_parameters": [ + { + "name": "token_count", + "type": "index" + }, + { + "name": "input_size", + "type": "index" + }, + { + "name": "output_size", + "type": "index" + } + ], + "bindings": [ + "q8_input", + "weight", + "output" + ], + "binding_access": [ + "read", + "read", + "read_write" + ], + "target_selector": "gfx1100", + "compile_recipe": { + "mode": "archive", + "link_module": "linear_q6k_q8_1_x4_linked", + "primary_sources": [ + "ggml/linear_q6k_q8_1_x4.loom" + ], + "library_sources": [ + "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", + "../loom-libs/motifs/q6_k_f16.loom", + "ggml/quantize_q8_1_x4.loom", + "../loom-libs/motifs/quantize_q8_1_x4_group.loom", + "../loom-libs/motifs/dequant.loom" + ] + }, + "compile_dependencies": [ + "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", + "../loom-libs/motifs/q6_k_f16.loom", + "ggml/quantize_q8_1_x4.loom", + "../loom-libs/motifs/quantize_q8_1_x4_group.loom", + "../loom-libs/motifs/dequant.loom" ] }, { @@ -238,14 +373,16 @@ "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", "../loom-libs/motifs/q6_k_f16.loom", "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom" + "../loom-libs/motifs/quantize_q8_1_x4_group.loom", + "../loom-libs/motifs/dequant.loom" ] }, "compile_dependencies": [ "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", "../loom-libs/motifs/q6_k_f16.loom", "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom" + "../loom-libs/motifs/quantize_q8_1_x4_group.loom", + "../loom-libs/motifs/dequant.loom" ] }, { diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index 213637bd8a64..3aa9d1ad8e18 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -6028,13 +6028,17 @@ static void run_scale_add_f32_cpu_reference_case(bool strided_residual) { ggml_backend_free(hrx_backend); } +static void set_test_row64_policy(ggml_backend_t backend, bool prefer_row64); + static void run_get_rows_f32_cpu_reference_case(ggml_type weight_type, int64_t hidden_size = kQwenHiddenSize, int64_t vocabulary_count = 64, - int64_t token_count = 7) { + int64_t token_count = 7, + bool prefer_row64 = false) { ggml_backend_t cpu_backend = init_cpu_backend(); ggml_backend_t hrx_backend = ggml_backend_hrx_init(0); REQUIRE(hrx_backend != nullptr); + set_test_row64_policy(hrx_backend, prefer_row64); ggml_init_params params = {}; params.mem_size = 4 * 1024 * 1024; @@ -6060,7 +6064,23 @@ static void run_get_rows_f32_cpu_reference_case(ggml_type weight_type, ggml_build_forward_expand(cpu_graph, cpu_output); ggml_build_forward_expand(hrx_graph, hrx_output); - require_kernel_subsequence(scheduled_kernel_sequence(hrx_graph), { "loom_libs:ggml_get_rows_f32" }); + std::vector weight = make_matmul_weight_bytes(weight_type, hidden_size, vocabulary_count, 5); + auto imported_weight = std::unique_ptr( + nullptr, ggml_backend_buffer_free); + if (prefer_row64) { + imported_weight.reset(ggml_backend_dev_buffer_from_host_ptr( + ggml_backend_get_device(hrx_backend), weight.data(), weight.size(), weight.size())); + REQUIRE(imported_weight != nullptr); + REQUIRE(ggml_backend_tensor_alloc(imported_weight.get(), hrx_weight, weight.data()) == GGML_STATUS_SUCCESS); + ggml_backend_buffer_set_usage(imported_weight.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + const char * layout = weight_type == GGML_TYPE_Q4_K ? ggml::hrx::kQ4KPackedK256Row64Layout : + ggml::hrx::kQ6KPackedK256Row64ScaleRowLayout; + require_weight_layout(hrx_graph, { hrx_weight }, layout, { "gfx1151", true }, + "loom_libs:ggml_get_rows_f32", 1); + } + + require_kernel_subsequence(scheduled_kernel_sequence(hrx_graph, { "gfx1151", prefer_row64 }), + { "loom_libs:ggml_get_rows_f32" }); const ggml::hrx::KernelSpecialization specialization = scheduled_kernel_specialization(hrx_graph, "loom_libs:ggml_get_rows_f32"); REQUIRE(specialization.workload_specialization == ggml::hrx::WorkloadSpecialization::Dynamic); @@ -6076,12 +6096,14 @@ static void run_get_rows_f32_cpu_reference_case(ggml_type weight_type, REQUIRE(cpu_buffer != nullptr); REQUIRE(hrx_buffer != nullptr); - const std::vector weight = make_matmul_weight_bytes(weight_type, hidden_size, vocabulary_count, 5); std::vector ids(static_cast(token_count)); for (int64_t i = 0; i < token_count; ++i) { ids[static_cast(i)] = static_cast((i * 17 + 3) % vocabulary_count); } - set_tensor_pair_bytes(cpu_backend, cpu_weight, hrx_backend, hrx_weight, weight.data(), weight.size()); + ggml_backend_tensor_set(cpu_weight, weight.data(), 0, weight.size()); + if (!prefer_row64) { + ggml_backend_tensor_set(hrx_weight, weight.data(), 0, weight.size()); + } set_tensor_pair_bytes(cpu_backend, cpu_ids, hrx_backend, hrx_ids, ids.data(), ids.size() * sizeof(int32_t)); REQUIRE(ggml_backend_graph_compute(cpu_backend, cpu_graph) == GGML_STATUS_SUCCESS); @@ -6092,12 +6114,34 @@ static void run_get_rows_f32_cpu_reference_case(ggml_type weight_type, ggml_backend_buffer_free(cpu_buffer); ggml_backend_buffer_free(hrx_buffer); + imported_weight.reset(); ggml_free(cpu_ctx); ggml_free(hrx_ctx); ggml_backend_free(cpu_backend); ggml_backend_free(hrx_backend); } +static void run_get_rows_row64_alias_schedule_case() { + ggml_init_params params = {}; + params.mem_size = 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + ggml_tensor * weight = ggml_new_tensor_2d(ctx, GGML_TYPE_Q6_K, kQwenHiddenSize, 64); + ggml_tensor * view = ggml_view_2d(ctx, weight, kQwenHiddenSize, 64, weight->nb[1], 0); + ggml_tensor * ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); + ggml_tensor * rows = ggml_get_rows(ctx, view, ids); + ggml_cgraph * graph = ggml_new_graph(ctx); + ggml_build_forward_expand(graph, rows); + ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*graph); + REQUIRE(imported.valid()); + const ggml::hrx::Value * view_value = imported.graph.values().find_tensor(view); + REQUIRE(view_value != nullptr && view_value->alias_source.value >= 0); + require_weight_layout(graph, { view }, ggml::hrx::kNativeWeightLayout, { "gfx1151", true }, + "loom_libs:ggml_get_rows_f32", 1); + ggml_free(ctx); +} + static void run_get_rows_q8_1_zero_weight_case() { ggml_backend_t hrx_backend = ggml_backend_hrx_init(0); REQUIRE(hrx_backend != nullptr); @@ -6271,6 +6315,79 @@ static std::vector f32_matmul_reference(ggml_type type, const std::vector int64_t inputs, int64_t outputs, int64_t tokens); static void require_reference_close(const std::vector & actual, const std::vector & expected, float absolute_tolerance); +static void run_get_rows_rmsnorm_row64_cpu_reference_case(ggml_type embedding_type, int64_t row_count, + int64_t token_count) { + ggml_backend_t cpu_backend = init_cpu_backend(); + ggml_backend_t hrx_backend = ggml_backend_hrx_init(0); + REQUIRE(hrx_backend != nullptr); + set_test_row64_policy(hrx_backend, true); + + ggml_init_params params = {}; + params.mem_size = 8 * 1024 * 1024; + params.no_alloc = true; + ggml_context * cpu_ctx = ggml_init(params); + ggml_context * hrx_ctx = ggml_init(params); + REQUIRE(cpu_ctx != nullptr && hrx_ctx != nullptr); + + auto build_graph = [embedding_type, row_count, token_count](ggml_context * ctx) { + ggml_tensor * embedding = ggml_new_tensor_2d(ctx, embedding_type, kQwenHiddenSize, row_count); + ggml_tensor * ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, token_count); + ggml_tensor * norm_weight = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, kQwenHiddenSize); + ggml_tensor * projection = ggml_new_tensor_2d(ctx, GGML_TYPE_Q6_K, kQwenHiddenSize, 64); + ggml_tensor * rows = ggml_get_rows(ctx, embedding, ids); + ggml_tensor * normalized = build_rmsnorm_mul_graph(ctx, rows, norm_weight); + ggml_tensor * output = ggml_mul_mat(ctx, projection, normalized); + return std::array{ embedding, ids, norm_weight, projection, output }; + }; + const auto cpu = build_graph(cpu_ctx); + const auto hrx = build_graph(hrx_ctx); + ggml_cgraph * cpu_graph = ggml_new_graph(cpu_ctx); + ggml_cgraph * hrx_graph = ggml_new_graph(hrx_ctx); + ggml_build_forward_expand(cpu_graph, cpu[4]); + ggml_build_forward_expand(hrx_graph, hrx[4]); + + std::vector embedding = make_matmul_weight_bytes(embedding_type, kQwenHiddenSize, row_count, 5); + std::vector projection = make_matmul_weight_bytes(GGML_TYPE_Q6_K, kQwenHiddenSize, 64, 11); + auto embedding_buffer = import_test_weight(hrx_backend, hrx[0], embedding); + auto projection_buffer = allocate_test_weight(hrx_backend, hrx[3]); + const char * layout = embedding_type == GGML_TYPE_Q4_K ? ggml::hrx::kQ4KPackedK256Row64Layout : + ggml::hrx::kQ6KPackedK256Row64ScaleRowLayout; + require_weight_layout(hrx_graph, { hrx[0] }, layout, { "gfx1151", true }, + "loom_libs:ggml_get_rows_rmsnorm_binary_q8_1_x4_f16", 1); + + ggml_backend_buffer_t cpu_buffer = ggml_backend_alloc_ctx_tensors(cpu_ctx, cpu_backend); + ggml_backend_buffer_t hrx_buffer = ggml_backend_alloc_ctx_tensors(hrx_ctx, hrx_backend); + REQUIRE(cpu_buffer != nullptr && hrx_buffer != nullptr); + std::vector ids(static_cast(token_count)); + for (int64_t i = 0; i < token_count; ++i) { + ids[static_cast(i)] = static_cast((i * 37 + 3) % row_count); + } + const std::vector norm_weight = make_weight(kQwenHiddenSize); + ggml_backend_tensor_set(cpu[0], embedding.data(), 0, embedding.size()); + ggml_backend_tensor_set(cpu[3], projection.data(), 0, projection.size()); + ggml_backend_tensor_set(hrx[3], projection.data(), 0, projection.size()); + set_tensor_pair_bytes(cpu_backend, cpu[1], hrx_backend, hrx[1], ids.data(), ids.size() * sizeof(int32_t)); + set_tensor_pair_bytes(cpu_backend, cpu[2], hrx_backend, hrx[2], norm_weight.data(), + norm_weight.size() * sizeof(float)); + + REQUIRE(ggml_backend_graph_compute(cpu_backend, cpu_graph) == GGML_STATUS_SUCCESS); + REQUIRE(ggml_backend_graph_compute(hrx_backend, hrx_graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(cpu_backend); + ggml_backend_synchronize(hrx_backend); + require_close(get_f32_tensor(hrx_backend, hrx[4]), get_f32_tensor(cpu_backend, cpu[4]), 1.0f, 3.0e-2f); + REQUIRE(ggml_backend_graph_compute(hrx_backend, hrx_graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(hrx_backend); + require_weight_cache_stats(hrx_backend, TestWeightSource::Imported, 1); + + ggml_backend_buffer_free(cpu_buffer); + ggml_backend_buffer_free(hrx_buffer); + embedding_buffer.reset(); + projection_buffer.reset(); + ggml_free(cpu_ctx); + ggml_free(hrx_ctx); + ggml_backend_free(cpu_backend); + ggml_backend_free(hrx_backend); +} static void run_dense_matmul_cpu_reference_case(ggml_type weight_type, const char * expected_kernel, @@ -7349,10 +7466,12 @@ static void run_dense_matmul_zero_weight_case(ggml_type weight_type, ggml_backend_free(hrx_backend); } -static void run_endpoint_rmsnorm_q6k_q8_cpu_reference_case() { +static void run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(bool prefer_row64 = false, + bool tied_embedding = false) { ggml_backend_t cpu_backend = init_cpu_backend(); ggml_backend_t hrx_backend = ggml_backend_hrx_init(0); REQUIRE(hrx_backend != nullptr); + set_test_row64_policy(hrx_backend, prefer_row64); ggml_init_params params = {}; params.mem_size = 32 * 1024 * 1024; @@ -7368,12 +7487,16 @@ static void run_endpoint_rmsnorm_q6k_q8_cpu_reference_case() { ggml_tensor * cpu_weight = ggml_new_tensor_2d(cpu_ctx, GGML_TYPE_Q6_K, kQwenHiddenSize, kQwenVocabularyCount); ggml_tensor * cpu_norm = build_rmsnorm_mul_graph(cpu_ctx, cpu_input, cpu_norm_w); ggml_tensor * cpu_output = ggml_mul_mat(cpu_ctx, cpu_weight, cpu_norm); + ggml_tensor * cpu_ids = tied_embedding ? ggml_new_tensor_1d(cpu_ctx, GGML_TYPE_I32, 2) : nullptr; + ggml_tensor * cpu_rows = tied_embedding ? ggml_get_rows(cpu_ctx, cpu_weight, cpu_ids) : nullptr; ggml_tensor * hrx_input = ggml_new_tensor_2d(hrx_ctx, GGML_TYPE_F32, kQwenHiddenSize, token_count); ggml_tensor * hrx_norm_w = ggml_new_tensor_1d(hrx_ctx, GGML_TYPE_F32, kQwenHiddenSize); ggml_tensor * hrx_weight = ggml_new_tensor_2d(hrx_ctx, GGML_TYPE_Q6_K, kQwenHiddenSize, kQwenVocabularyCount); ggml_tensor * hrx_norm = build_rmsnorm_mul_graph(hrx_ctx, hrx_input, hrx_norm_w); ggml_tensor * hrx_output = ggml_mul_mat(hrx_ctx, hrx_weight, hrx_norm); + ggml_tensor * hrx_ids = tied_embedding ? ggml_new_tensor_1d(hrx_ctx, GGML_TYPE_I32, 2) : nullptr; + ggml_tensor * hrx_rows = tied_embedding ? ggml_get_rows(hrx_ctx, hrx_weight, hrx_ids) : nullptr; REQUIRE(cpu_output != nullptr); REQUIRE(hrx_output != nullptr); @@ -7381,12 +7504,30 @@ static void run_endpoint_rmsnorm_q6k_q8_cpu_reference_case() { ggml_cgraph * hrx_graph = ggml_new_graph(hrx_ctx); REQUIRE(cpu_graph != nullptr); REQUIRE(hrx_graph != nullptr); + if (tied_embedding) { + ggml_build_forward_expand(cpu_graph, cpu_rows); + ggml_build_forward_expand(hrx_graph, hrx_rows); + } ggml_build_forward_expand(cpu_graph, cpu_output); ggml_build_forward_expand(hrx_graph, hrx_output); + std::vector weight = make_quantized_rows(GGML_TYPE_Q6_K, kQwenHiddenSize, kQwenVocabularyCount, 9); + auto weight_buffer = std::unique_ptr( + nullptr, ggml_backend_buffer_free); + if (prefer_row64) { + weight_buffer = import_test_weight(hrx_backend, hrx_weight, weight); + require_weight_layout(hrx_graph, { hrx_weight }, ggml::hrx::kQ6KPackedK256Row64ScaleRowLayout, + { "gfx1151", true }, nullptr, tied_embedding ? 2 : 1); + } + require_kernel_subsequence( - scheduled_kernel_sequence(hrx_graph), - { "qwen3_moe:qwen3_moe_rmsnorm_f32_quantize_q8_1_x4", "qwen3_moe:ggml_linear_q6k_q8_1_x4" }); + scheduled_kernel_sequence(hrx_graph, { "gfx1151", prefer_row64 }), + { "qwen3_moe:qwen3_moe_rmsnorm_f32_quantize_q8_1_x4", + prefer_row64 ? "qwen3_moe:ggml_linear_q6k_q8_1_x4_row64" : "qwen3_moe:ggml_linear_q6k_q8_1_x4" }); + if (tied_embedding) { + require_kernel_subsequence(scheduled_kernel_sequence(hrx_graph, { "gfx1151", prefer_row64 }), + { "loom_libs:ggml_get_rows_f32" }); + } ggml_backend_buffer_t cpu_buffer = ggml_backend_alloc_ctx_tensors(cpu_ctx, cpu_backend); ggml_backend_buffer_t hrx_buffer = ggml_backend_alloc_ctx_tensors(hrx_ctx, hrx_backend); @@ -7395,20 +7536,37 @@ static void run_endpoint_rmsnorm_q6k_q8_cpu_reference_case() { const std::vector input = make_pattern_f32(kQwenHiddenSize * token_count, 8, 0.01f); const std::vector norm_w = make_weight(kQwenHiddenSize); - const std::vector weight = make_quantized_rows(GGML_TYPE_Q6_K, kQwenHiddenSize, kQwenVocabularyCount, 9); set_tensor_pair_bytes(cpu_backend, cpu_input, hrx_backend, hrx_input, input.data(), input.size() * sizeof(float)); set_tensor_pair_bytes(cpu_backend, cpu_norm_w, hrx_backend, hrx_norm_w, norm_w.data(), norm_w.size() * sizeof(float)); - set_tensor_pair_bytes(cpu_backend, cpu_weight, hrx_backend, hrx_weight, weight.data(), weight.size()); + ggml_backend_tensor_set(cpu_weight, weight.data(), 0, weight.size()); + if (!prefer_row64) { + ggml_backend_tensor_set(hrx_weight, weight.data(), 0, weight.size()); + } + if (tied_embedding) { + const std::array ids = { 3, static_cast(kQwenVocabularyCount - 1) }; + set_tensor_pair_bytes(cpu_backend, cpu_ids, hrx_backend, hrx_ids, ids.data(), sizeof(ids)); + } REQUIRE(ggml_backend_graph_compute(cpu_backend, cpu_graph) == GGML_STATUS_SUCCESS); REQUIRE(ggml_backend_graph_compute(hrx_backend, hrx_graph) == GGML_STATUS_SUCCESS); ggml_backend_synchronize(cpu_backend); ggml_backend_synchronize(hrx_backend); require_close(get_f32_tensor(hrx_backend, hrx_output), get_f32_tensor(cpu_backend, cpu_output), 6.0e-1f, 3.0e-2f); + if (tied_embedding) { + require_close(get_f32_tensor(hrx_backend, hrx_rows), get_f32_tensor(cpu_backend, cpu_rows), 5.0e-4f); + } + if (prefer_row64) { + REQUIRE(ggml_backend_graph_compute(hrx_backend, hrx_graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(hrx_backend); + require_close(get_f32_tensor(hrx_backend, hrx_output), get_f32_tensor(cpu_backend, cpu_output), 6.0e-1f, + 3.0e-2f); + require_weight_cache_stats(hrx_backend, TestWeightSource::Imported, 1); + } ggml_backend_buffer_free(cpu_buffer); ggml_backend_buffer_free(hrx_buffer); + weight_buffer.reset(); ggml_free(cpu_ctx); ggml_free(hrx_ctx); ggml_backend_free(cpu_backend); @@ -9488,6 +9646,29 @@ static void register_basic_ops_cases(Suite & suite) { GGML_TYPE_BF16, GGML_TYPE_F32 }) { suite.device_case("get_rows_f32." + type_name(type), [type] { run_get_rows_f32_cpu_reference_case(type); }); } + for (const ggml_type type : { GGML_TYPE_Q4_K, GGML_TYPE_Q6_K }) { + suite.device_case("get_rows_f32." + type_name(type) + ".row64.tokens1", [type] { + run_get_rows_f32_cpu_reference_case(type, kQwenHiddenSize, 64, 1, true); + }); + suite.device_case("get_rows_f32." + type_name(type) + ".row64.tokens17", [type] { + run_get_rows_f32_cpu_reference_case(type, kQwenHiddenSize, 64, 17, true); + }); + } + suite.device_case("get_rows_f32.q6_k.row64.vocabulary151936.tokens1", [] { + run_get_rows_f32_cpu_reference_case(GGML_TYPE_Q6_K, kQwenHiddenSize, kQwenVocabularyCount, 1, true); + }); + suite.host_case("get_rows_f32.q6_k.row64.alias_uses_native", [] { run_get_rows_row64_alias_schedule_case(); }); + for (const ggml_type type : { GGML_TYPE_Q4_K, GGML_TYPE_Q6_K }) { + suite.device_case("get_rows_rmsnorm." + type_name(type) + ".row64.tokens1", [type] { + run_get_rows_rmsnorm_row64_cpu_reference_case(type, 64, 1); + }); + suite.device_case("get_rows_rmsnorm." + type_name(type) + ".row64.tokens17", [type] { + run_get_rows_rmsnorm_row64_cpu_reference_case(type, 64, 17); + }); + } + suite.device_case("get_rows_rmsnorm.q6_k.row64.vocabulary151936.tokens1", [] { + run_get_rows_rmsnorm_row64_cpu_reference_case(GGML_TYPE_Q6_K, kQwenVocabularyCount, 1); + }); suite.device_case("get_rows_f32.q1_0.rows2048", [] { run_get_rows_f32_cpu_reference_case(GGML_TYPE_Q1_0, 2048); }); suite.device_case("get_rows_f32.q5_1.rows640", [] { run_get_rows_f32_cpu_reference_case(GGML_TYPE_Q5_1, 640); }); suite.device_case("get_rows_q8_1.zero_weight", [] { run_get_rows_q8_1_zero_weight_case(); }); @@ -9934,6 +10115,10 @@ static void register_dense_matmul_cases(Suite & suite) { }); suite.device_case("endpoint_rmsnorm_q6k_q8.cpu_reference", [] { run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(); }); + suite.device_case("endpoint_rmsnorm_q6k_q8.row64.cpu_reference", + [] { run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(true); }); + suite.device_case("endpoint_rmsnorm_q6k_q8.row64.tied_embedding", + [] { run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(true, true); }); suite.device_case("dense_matmul.q6_k.skinny.tokens5.outputs4096.input16384", [] { run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_skinny_input_f32_publish_f32", 5, 4096, 16384); From fdabd06be411157178e7b2e98b41862c50632ac4 Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Wed, 11 Nov 2026 17:07:46 +0000 Subject: [PATCH 03/18] test: cover Q6 row64 short residual numerics (lemonade-ws-aj0j.6) Assisted-by: Codex --- tests/hrx-ops-test.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index 3aa9d1ad8e18..b13b152fd699 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -10172,6 +10172,14 @@ static void register_dense_matmul_cases(Suite & suite) { GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_tiled_input_f32_bias_residual_publish_f32", 29, 1024, false, true, false, 3584, row64_postops); }); + for (const int64_t tokens : { 64, 66, 127 }) { + suite.device_case("qwen3.q6_row64_short_residual.k3072.tokens" + std::to_string(tokens), + [tokens, row64_postops] { + run_dense_matmul_postops_cpu_reference_case( + GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_wave32", + tokens, 1024, false, true, false, 3072, row64_postops); + }); + } suite.device_case("minicpmv46.q4_row64_tiled_residual.tokens46", [row64_postops] { run_dense_matmul_postops_cpu_reference_case( GGML_TYPE_Q4_K, "loom_libs:ggml_mul_mat_tiled_input_f32_bias_residual_publish_f32", From 1a7db48f0e5c27ecd94df905d75d0710b48bba24 Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Wed, 11 Nov 2026 17:19:49 +0000 Subject: [PATCH 04/18] ggml-hrx: use M64 Q6 row64 tiles for short Qwen3 prefills Measured direct Q6 kernel gains at the captured PromptBridge shapes and verified independent CPU output references before routing. Preserve M128 for other dimensions and prompt lengths. Bead: lemonade-ws-aj0j.6 Assisted-by: Codex --- .../common/dispatch-mul-mat.cpp | 9 +- .../kernels/loom-libs/manifest.json | 47 +++++++ .../ops/mul_mat_q6_k_packed_f16_wmma.loom | 126 ++++++++++++------ tests/hrx-ops-test.cpp | 23 +++- 4 files changed, 156 insertions(+), 49 deletions(-) diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp index 848c0b98ddfa..ab09d0cde83c 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp @@ -103,6 +103,8 @@ static constexpr int64_t kMulMatVectorPackedMaxOutputSize = 26 static constexpr int64_t kMulMatVectorSelectedOutputsMaxOutputSize = 262144; static constexpr KernelCatalogRef kMulMatQ6KF32WmmaPrefillWave32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_q6_k_f32_wmma_prefill_wave32"); +static constexpr KernelCatalogRef kMulMatQ6KF32WmmaPrefillM64Wave32Kernel = + GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32"); static constexpr KernelCatalogRef kMulMatQ6KF16WmmaPrefillWave32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_q6_k_f16_wmma_prefill_wave32"); static constexpr KernelCatalogRef kSelectSymmetricI4K32GroupsKernel = @@ -903,10 +905,13 @@ static bool match_q6_k_prefill_wave32_dispatch(const DispatchMatchContext & cont return false; } - // Short prompts use masked M128 tiles without a separate activation copy. + // Short prompts use masked WMMA tiles without a separate activation copy. if (short_row64) { + // M64 halves reduce register pressure at these measured short shapes. + const bool use_m64 = match.output_size == 1024 && (match.input_size == 1024 || match.input_size == 3072); return build_q6_k_prefill_wave32_dispatch(match, match.input->id, match.input->byte_count, - kMulMatQ6KF32WmmaPrefillWave32Kernel, row64_weight, + use_m64 ? kMulMatQ6KF32WmmaPrefillM64Wave32Kernel : + kMulMatQ6KF32WmmaPrefillWave32Kernel, row64_weight, context.root_index, dispatch_match); } diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json index e88e8e69c243..15538f76d2d1 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json @@ -6131,6 +6131,53 @@ "motifs/elementwise/binary_apply.loom" ] }, + { + "name": "ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32", + "family": "loom_libs", + "symbol": "ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32", + "source": "ops/mul_mat_q6_k_packed_f16_wmma.loom", + "workload_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "launch_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "bindings": [ + "input", + "weight", + "output" + ], + "binding_access": [ + "read", + "read", + "write" + ], + "target_selector": "", + "compile_recipe": { + "mode": "direct", + "primary_sources": [ + "ops/mul_mat_q6_k_packed_f16_wmma.loom" + ], + "library_sources": [ + "motifs/q6_k_f16.loom", + "motifs/dequant.loom", + "motifs/mul_mat_quantized_f16_prefill.loom", + "motifs/elementwise/binary_apply.loom" + ] + }, + "compile_dependencies": [ + "motifs/q6_k_f16.loom", + "motifs/dequant.loom", + "motifs/mul_mat_quantized_f16_prefill.loom", + "motifs/elementwise/binary_apply.loom" + ] + }, { "name": "ggml_mul_mat_q6_k_f16_wmma_prefill_wave32", "family": "loom_libs", diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom index 36ca4f94b66f..ede8a3d08561 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom @@ -28,7 +28,7 @@ template.decl @ggml.mul_mat_q6_k_i8_prepacked.body(%token_count: index, %input_s template.decl @ggml.mul_mat_q6_k_packed.selected_refine.body(%token_count: index, %input_size0: index, %output_size0: index, %scale_row_layout: index, %candidate_count0: index, %input: buffer, %weight: buffer, %candidates: buffer, %output: buffer) -template.decl @ggml.mul_mat_q6_k_packed.prefill_wave32.body(%input_is_f16: i1, %token_count: index, %input_size0: index, %output_size0: index, %channel_tile: index, %token_tile: index, %input: buffer, %weight: buffer, %output: buffer) +template.decl @ggml.mul_mat_q6_k_packed.prefill_wave32.body(%input_is_f16: i1, %single_half: i1, %token_count: index, %input_size0: index, %output_size0: index, %channel_tile: index, %token_tile: index, %input: buffer, %weight: buffer, %output: buffer) amdgpu.target @ggml_mul_mat_q6_k_prefill_gfx11_wave32 {subgroup_size = 32} @@ -1280,7 +1280,7 @@ template.def<@ggml.mul_mat_q6_k_packed.publish_f32_wave32> device @ggml_mul_mat_ template.return } -template.def<@ggml.mul_mat_q6_k_packed.prefill_wave32.body> device @ggml_mul_mat_q6_k_packed_prefill_wave32_body(%input_is_f16: i1, %token_count: index, %input_size0: index, %output_size0: index, %channel_tile: index, %token_tile: index, %input: buffer, %weight: buffer, %output: buffer) { +template.def<@ggml.mul_mat_q6_k_packed.prefill_wave32.body> device @ggml_mul_mat_q6_k_packed_prefill_wave32_body(%input_is_f16: i1, %single_half: i1, %token_count: index, %input_size0: index, %output_size0: index, %channel_tile: index, %token_tile: index, %input: buffer, %weight: buffer, %output: buffer) { %output_accumulation = config.get @ggml.mul_mat_q6_k_packed.output_accumulation : index %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048)] : index %bounded_input_size = index.assume %input_size0 [range(%input_size0, 256, 32768), multiple_of(%input_size0, 256)] : index @@ -1323,7 +1323,8 @@ template.def<@ggml.mul_mat_q6_k_packed.prefill_wave32.body> device @ggml_mul_mat %activation_fragment_layout = encoding.layout.strided [1, %c72] : encoding %activation_fragment_view = buffer.view %activation_stage[%c0_offset] : buffer -> view<64x64xf16, %activation_fragment_layout> %channel_tile_base = index.mul %channel_tile, %c128 : index - %token_tile_base = index.mul %token_tile, %c128 : index + %token_tile_width = scf.select %single_half, %c64, %c128 : index + %token_tile_base = index.mul %token_tile, %token_tile_width : index %second_token_tile_base = index.add %token_tile_base, %c64 : index %load_half = index.rem %workitem, %c2 : index %load_packet = index.mul %load_half, %c4 : index @@ -1415,44 +1416,49 @@ template.def<@ggml.mul_mat_q6_k_packed.prefill_wave32.body> device @ggml_mul_mat scf.yield %half_next00, %half_next01, %half_next02, %half_next03 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> } kernel.barrier scope(workgroup) ordering(acq_rel) - scf.for %stage_half1 = [%c0 to %c2 step %c1] unroll { - %stage_k_base1 = index.mul %stage_half1, %c32 : index - %token1 = index.add %second_token_tile_base, %activation_load_row : index - %valid_token1 = index.cmp ult, %token1, %bounded_token_count : index - %activation_k1 = index.add %stage_k_base1, %activation_load_k : index - %activation_values1 = scf.if %valid_token1 -> (vector<8xf16>) { - %bounded_token1, %input_token_count1 = index.assume %token1, %bounded_token_count [lt(%token1, %bounded_token_count)] : index, index - %input_k1 = index.add %k_origin, %activation_k1 : index - %loaded1 = scf.if %input_is_f16 -> (vector<8xf16>) { - %narrow1 = vector.load %input_f16_view[%bounded_token1, %input_k1] : view<[%bounded_token_count]x[%bounded_input_size]xf16> -> vector<8xf16> - scf.yield %narrow1 : vector<8xf16> + %next10, %next11, %next12, %next13 = scf.if %single_half -> (vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>) { + scf.yield %acc10, %acc11, %acc12, %acc13 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> + } else { + scf.for %stage_half1 = [%c0 to %c2 step %c1] unroll { + %stage_k_base1 = index.mul %stage_half1, %c32 : index + %token1 = index.add %second_token_tile_base, %activation_load_row : index + %valid_token1 = index.cmp ult, %token1, %bounded_token_count : index + %activation_k1 = index.add %stage_k_base1, %activation_load_k : index + %activation_values1 = scf.if %valid_token1 -> (vector<8xf16>) { + %bounded_token1, %input_token_count1 = index.assume %token1, %bounded_token_count [lt(%token1, %bounded_token_count)] : index, index + %input_k1 = index.add %k_origin, %activation_k1 : index + %loaded1 = scf.if %input_is_f16 -> (vector<8xf16>) { + %narrow1 = vector.load %input_f16_view[%bounded_token1, %input_k1] : view<[%bounded_token_count]x[%bounded_input_size]xf16> -> vector<8xf16> + scf.yield %narrow1 : vector<8xf16> + } else { + %wide1 = vector.load %input_f32_view[%bounded_token1, %input_k1] : view<[%bounded_token_count]x[%bounded_input_size]xf32> -> vector<8xf32> + %narrow1 = vector.fptrunc %wide1 : vector<8xf32> to vector<8xf16> + scf.yield %narrow1 : vector<8xf16> + } + scf.yield %loaded1 : vector<8xf16> } else { - %wide1 = vector.load %input_f32_view[%bounded_token1, %input_k1] : view<[%bounded_token_count]x[%bounded_input_size]xf32> -> vector<8xf32> - %narrow1 = vector.fptrunc %wide1 : vector<8xf32> to vector<8xf16> - scf.yield %narrow1 : vector<8xf16> + scf.yield %c0_f16x8 : vector<8xf16> } - scf.yield %loaded1 : vector<8xf16> - } else { - scf.yield %c0_f16x8 : vector<8xf16> + vector.store %activation_values1, %activation_stage_physical_view[%activation_load_row, %activation_k1] : vector<8xf16>, view<64x72xf16> } - vector.store %activation_values1, %activation_stage_physical_view[%activation_load_row, %activation_k1] : vector<8xf16>, view<64x72xf16> - } - kernel.barrier scope(workgroup) ordering(acq_rel) - %next10, %next11, %next12, %next13 = scf.for %k_half1 = [%c0 to %c64 step %c16](%half_acc10 = %acc10 : vector<8xf32>, %half_acc11 = %acc11 : vector<8xf32>, %half_acc12 = %acc12 : vector<8xf32>, %half_acc13 = %acc13 : vector<8xf32>) -> (vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>) unroll { - %lhs1 = vector.fragment.load %weight_stage_view[%subgroup_channel_add, %k_half1] shape [%m, %k] : view<128x72xf16> -> vector<16xf16> - %rhs10 = vector.fragment.load %activation_fragment_view[%k_half1, %c0] shape [%k, %n] : view<64x64xf16, %activation_fragment_layout> -> vector<16xf16> - %rhs11 = vector.fragment.load %activation_fragment_view[%k_half1, %c16] shape [%k, %n] : view<64x64xf16, %activation_fragment_layout> -> vector<16xf16> - %half_next10 = vector.mma %lhs1, %rhs10, %half_acc10 : vector<16xf16>, vector<16xf16>, vector<8xf32> - %half_next11 = vector.mma %lhs1, %rhs11, %half_acc11 : vector<16xf16>, vector<16xf16>, vector<8xf32> - scf.schedule.fence - %rhs12 = vector.fragment.load %activation_fragment_view[%k_half1, %c32] shape [%k, %n] : view<64x64xf16, %activation_fragment_layout> -> vector<16xf16> - %rhs13 = vector.fragment.load %activation_fragment_view[%k_half1, %c48] shape [%k, %n] : view<64x64xf16, %activation_fragment_layout> -> vector<16xf16> - %half_next12 = vector.mma %lhs1, %rhs12, %half_acc12 : vector<16xf16>, vector<16xf16>, vector<8xf32> - %half_next13 = vector.mma %lhs1, %rhs13, %half_acc13 : vector<16xf16>, vector<16xf16>, vector<8xf32> - scf.schedule.fence - scf.yield %half_next10, %half_next11, %half_next12, %half_next13 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> + kernel.barrier scope(workgroup) ordering(acq_rel) + %second10, %second11, %second12, %second13 = scf.for %k_half1 = [%c0 to %c64 step %c16](%half_acc10 = %acc10 : vector<8xf32>, %half_acc11 = %acc11 : vector<8xf32>, %half_acc12 = %acc12 : vector<8xf32>, %half_acc13 = %acc13 : vector<8xf32>) -> (vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>) unroll { + %lhs1 = vector.fragment.load %weight_stage_view[%subgroup_channel_add, %k_half1] shape [%m, %k] : view<128x72xf16> -> vector<16xf16> + %rhs10 = vector.fragment.load %activation_fragment_view[%k_half1, %c0] shape [%k, %n] : view<64x64xf16, %activation_fragment_layout> -> vector<16xf16> + %rhs11 = vector.fragment.load %activation_fragment_view[%k_half1, %c16] shape [%k, %n] : view<64x64xf16, %activation_fragment_layout> -> vector<16xf16> + %half_next10 = vector.mma %lhs1, %rhs10, %half_acc10 : vector<16xf16>, vector<16xf16>, vector<8xf32> + %half_next11 = vector.mma %lhs1, %rhs11, %half_acc11 : vector<16xf16>, vector<16xf16>, vector<8xf32> + scf.schedule.fence + %rhs12 = vector.fragment.load %activation_fragment_view[%k_half1, %c32] shape [%k, %n] : view<64x64xf16, %activation_fragment_layout> -> vector<16xf16> + %rhs13 = vector.fragment.load %activation_fragment_view[%k_half1, %c48] shape [%k, %n] : view<64x64xf16, %activation_fragment_layout> -> vector<16xf16> + %half_next12 = vector.mma %lhs1, %rhs12, %half_acc12 : vector<16xf16>, vector<16xf16>, vector<8xf32> + %half_next13 = vector.mma %lhs1, %rhs13, %half_acc13 : vector<16xf16>, vector<16xf16>, vector<8xf32> + scf.schedule.fence + scf.yield %half_next10, %half_next11, %half_next12, %half_next13 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> + } + kernel.barrier scope(workgroup) ordering(acq_rel) + scf.yield %second10, %second11, %second12, %second13 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> } - kernel.barrier scope(workgroup) ordering(acq_rel) scf.yield %next00, %next01, %next02, %next03, %next10, %next11, %next12, %next13 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> } scf.yield %block_result00, %block_result01, %block_result02, %block_result03, %block_result10, %block_result11, %block_result12, %block_result13 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> @@ -1461,10 +1467,14 @@ template.def<@ggml.mul_mat_q6_k_packed.prefill_wave32.body> device @ggml_mul_mat template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result01, %c16, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result02, %c32, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result03, %c48, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) - template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result10, %c64, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) - template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result11, %c80, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) - template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result12, %c96, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) - template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result13, %c112, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) + %true = scalar.constant true : i1 + %double_half = scalar.xori %single_half, %true : i1 + scf.if %double_half { + template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result10, %c64, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) + template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result11, %c80, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) + template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result12, %c96, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) + template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result13, %c112, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) + } template.return } @@ -1490,7 +1500,36 @@ kernel.def target(@ggml_mul_mat_q6_k_prefill_gfx11_wave32) export("ggml_mul_mat_ %channel_tile = kernel.workgroup.id : index %token_tile = kernel.workgroup.id : index %input_is_f16 = scalar.constant false : i1 - template.apply<@ggml.mul_mat_q6_k_packed.prefill_wave32.body>(%input_is_f16, %bounded_token_count, %input_size, %output_size, %channel_tile, %token_tile, %input, %weight, %output) : (i1, index, index, index, index, index, buffer, buffer, buffer) + %single_half = scalar.constant false : i1 + template.apply<@ggml.mul_mat_q6_k_packed.prefill_wave32.body>(%input_is_f16, %single_half, %bounded_token_count, %input_size, %output_size, %channel_tile, %token_tile, %input, %weight, %output) : (i1, i1, index, index, index, index, index, buffer, buffer, buffer) + kernel.return +} + +// Use one M64 half per workgroup for short F32 prompts. +kernel.def target(@ggml_mul_mat_q6_k_prefill_gfx11_wave32) export("ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32") @ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32(%token_count: index) { + %output_size = config.get @ggml.mul_mat_q6_k_packed.output_size : index + %token_capacity = config.get @ggml.mul_mat_q6_k_packed.token_capacity : index + %c1 = index.constant 1 : index + %c127 = index.constant 127 : index + %c64 = index.constant 64 : index + %c128 = index.constant 128 : index + %c256 = index.constant 256 : index + %padded_output_size = index.add %output_size, %c127 : index + %output_tiles = index.div %padded_output_size, %c128 : index + %c63 = index.constant 63 : index + %padded_token_count = index.add %token_capacity, %c63 : index + %token_tiles = index.div %padded_token_count, %c64 : index + kernel.launch.config workgroups(%token_tiles, %output_tiles, %c1) workgroup_size(%c256, %c1, %c1) : index +} launch(%token_count: index, %input: buffer, %weight: buffer, %output: buffer) { + %input_size = config.get @ggml.mul_mat_q6_k_packed.input_size : index + %output_size = config.get @ggml.mul_mat_q6_k_packed.output_size : index + %token_capacity = config.get @ggml.mul_mat_q6_k_packed.token_capacity : index + %bounded_token_count = index.assume %token_count [range(%token_count, 64, 2048), le(%token_count, %token_capacity)] : index + %channel_tile = kernel.workgroup.id : index + %token_tile = kernel.workgroup.id : index + %input_is_f16 = scalar.constant false : i1 + %single_half = scalar.constant true : i1 + template.apply<@ggml.mul_mat_q6_k_packed.prefill_wave32.body>(%input_is_f16, %single_half, %bounded_token_count, %input_size, %output_size, %channel_tile, %token_tile, %input, %weight, %output) : (i1, i1, index, index, index, index, index, buffer, buffer, buffer) kernel.return } @@ -1586,7 +1625,8 @@ kernel.def target(@ggml_mul_mat_q6_k_prefill_gfx11_wave32) export("ggml_mul_mat_ func.call @ggml_mul_mat_quantized_f16_prefill_wave32(%weight_format, %prefill_binary_op, %paired, %packed_input, %paired, %bounded_token_count, %input_size, %output_size, %channel_tile, %token_tile, %input, %weight, %weight, %output, %output) : (index, index, i1, i1, i1, index, index, index, index, index, buffer, buffer, buffer, buffer, buffer) } else { %input_is_f16 = scalar.constant true : i1 - template.apply<@ggml.mul_mat_q6_k_packed.prefill_wave32.body>(%input_is_f16, %bounded_token_count, %input_size, %output_size, %channel_tile, %token_tile, %input, %weight, %output) : (i1, index, index, index, index, index, buffer, buffer, buffer) + %single_half = scalar.constant false : i1 + template.apply<@ggml.mul_mat_q6_k_packed.prefill_wave32.body>(%input_is_f16, %single_half, %bounded_token_count, %input_size, %output_size, %channel_tile, %token_tile, %input, %weight, %output) : (i1, i1, index, index, index, index, index, buffer, buffer, buffer) } kernel.return } diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index b13b152fd699..ec1a23ffb346 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -9148,11 +9148,13 @@ static void run_qwen27_matmul_policy_route_case(ggml_type type, int64_t tokens, if (binding.value == source->id) { REQUIRE(binding.layout == expected); const bool q6_weight = type == GGML_TYPE_Q6_K; - const bool short_prompt = tokens == 66; + const bool short_prompt = tokens >= 64 && tokens < 128; const bool short_q6 = q6_weight && short_prompt; - if (short_q6) { + if (short_q6 && (policy.prefer_row64 || tokens == 66)) { + const bool use_m64 = n == 1024 && (k == 1024 || k == 3072); const char * expected_kernel = policy.prefer_row64 ? - "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_wave32" : + (use_m64 ? "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32" : + "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_wave32") : "loom_libs:ggml_mul_mat_tiled_input_f32_publish_f32"; REQUIRE(kernel_name_for_id(dispatch.kernel.kernel_id, target) == expected_kernel); } @@ -9323,7 +9325,9 @@ static std::vector run_dense_matmul_f32_reference_case(ggml_type type, in const bool prefill_aligned = tokens % 128 == 0; const bool aligned_prefill = prefill_minimum && prefill_aligned; const bool aligned_q6_prefill = q6_weight && aligned_prefill; + const bool short_m64 = short_q6 && n == 1024 && (k == 1024 || k == 3072); const char * expected_kernel = tokens == 1 ? "loom_libs:ggml_mul_mat_vector_f32_f32" : + short_m64 ? "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32" : short_q6 ? "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_wave32" : aligned_q6_prefill ? "loom_libs:ggml_mul_mat_q6_k_f16_wmma_prefill_wave32" : tokens % 32 == 0 ? "loom_libs:ggml_mul_mat_tiled_input_f32_publish_f32_aligned" : @@ -9516,6 +9520,12 @@ static void register_basic_ops_cases(Suite & suite) { } } } + for (const int64_t tokens : { 64, 66, 127 }) { + suite.host_case("qwen3.matmul_policy_route." + std::string(target) + ".q6_K.k3072.tokens" + + std::to_string(tokens), [target, tokens] { + run_qwen27_matmul_policy_route_case(GGML_TYPE_Q6_K, tokens, 3072, 1024, target); + }); + } } suite.host_case("qwen27.q4_decode_policy_route", [] { run_qwen27_matmul_policy_route_case(GGML_TYPE_Q4_K, 1); @@ -9594,6 +9604,11 @@ static void register_basic_ops_cases(Suite & suite) { } } } + for (const int64_t tokens : { 64, 127 }) { + suite.device_case("qwen3.q6_K.k1024.m64_boundary.tokens" + std::to_string(tokens), [tokens] { + run_dense_matmul_f32_reference_case(GGML_TYPE_Q6_K, tokens, 1024, 1024); + }); + } for (const int64_t tokens : { 66, 128 }) { suite.device_case("qwen3.q4_K.k2560.native_row64_equivalence.tokens" + std::to_string(tokens), [tokens] { const auto native = run_dense_matmul_f32_reference_case(GGML_TYPE_Q4_K, tokens, 2560, 1024, false, false); @@ -10176,7 +10191,7 @@ static void register_dense_matmul_cases(Suite & suite) { suite.device_case("qwen3.q6_row64_short_residual.k3072.tokens" + std::to_string(tokens), [tokens, row64_postops] { run_dense_matmul_postops_cpu_reference_case( - GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_wave32", + GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32", tokens, 1024, false, true, false, 3072, row64_postops); }); } From 4f114e802c4f54a3d574e2eb702f6045c6a8d727 Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Wed, 11 Nov 2026 22:59:26 +0000 Subject: [PATCH 05/18] ggml-hrx: reuse Q6 row64 decode loads Assisted-by: Codex --- .../motifs/matmul/vector_decode_pair.loom | 208 ++++++++++++------ tests/hrx-ops-test.cpp | 3 + 2 files changed, 146 insertions(+), 65 deletions(-) diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_decode_pair.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_decode_pair.loom index f5e982b2adbf..c7ca1b407a6a 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_decode_pair.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_decode_pair.loom @@ -14,6 +14,68 @@ func.decl @ggml_iq3s_grid_f32_halves() -> (vector<32xf32>, vector<32xf32>, vecto func.decl @ggml_dequant_f32_vector4(%iq4nl_table: vector<16xi8>, %iq3s_grid_low0: vector<32xf32>, %iq3s_grid_low1: vector<32xf32>, %iq3s_grid_low2: vector<32xf32>, %iq3s_grid_low3: vector<32xf32>, %iq3s_grid_low4: vector<32xf32>, %iq3s_grid_low5: vector<32xf32>, %iq3s_grid_low6: vector<32xf32>, %iq3s_grid_low7: vector<32xf32>, %iq3s_grid_low8: vector<32xf32>, %iq3s_grid_low9: vector<32xf32>, %iq3s_grid_low10: vector<32xf32>, %iq3s_grid_low11: vector<32xf32>, %iq3s_grid_low12: vector<32xf32>, %iq3s_grid_low13: vector<32xf32>, %iq3s_grid_low14: vector<32xf32>, %iq3s_grid_low15: vector<32xf32>, %iq3s_grid_high0: vector<32xf32>, %iq3s_grid_high1: vector<32xf32>, %iq3s_grid_high2: vector<32xf32>, %iq3s_grid_high3: vector<32xf32>, %iq3s_grid_high4: vector<32xf32>, %iq3s_grid_high5: vector<32xf32>, %iq3s_grid_high6: vector<32xf32>, %iq3s_grid_high7: vector<32xf32>, %iq3s_grid_high8: vector<32xf32>, %iq3s_grid_high9: vector<32xf32>, %iq3s_grid_high10: vector<32xf32>, %iq3s_grid_high11: vector<32xf32>, %iq3s_grid_high12: vector<32xf32>, %iq3s_grid_high13: vector<32xf32>, %iq3s_grid_high14: vector<32xf32>, %iq3s_grid_high15: vector<32xf32>, %weight_format: index, %weight: buffer, %row_byte_base: offset, %row: index, %input_size: index, %quant_block: index, %quant_group: index, %packet: index, %k: index) -> (vector<4xf32>) +// The four groups share a high-bit word, two low-bit words, and one block scale. +func.def inline @ggml_q6k_row64_pair_load_decode_state(%weight: buffer, %row: index, %input_size: index, %block: index, %packet: index, %half: index) -> (f32, vector<4xi8>, vector<1xi32>, vector<1xi32>, vector<1xi32>) { + %c0 = index.constant 0 : index + %c1 = index.constant 1 : index + %c2 = index.constant 2 : index + %c4 = index.constant 4 : index + %c64 = index.constant 64 : index + %c256 = index.constant 256 : index + %tile_bytes = index.constant 13440 : offset + %scale_offset = index.constant 128 : offset + %raw_offset = index.constant 1152 : offset + %blocks = index.div %input_size, %c256 : index + %row_group = index.div %row, %c64 : index + %row_lane = index.rem %row, %c64 : index + %tile = index.madd %row_group, %blocks, %block : index + %tile_base = index.scale %tile, %tile_bytes : index, offset -> offset + %scale_base = index.add %tile_base, %scale_offset : offset + %raw_base = index.add %tile_base, %raw_offset : offset + %raw = buffer.view %weight[%raw_base] : buffer -> view<2x3x64x8xi32> + %scales = buffer.view %weight[%scale_base] : buffer -> view<64x2x8xi8> + %ds = buffer.view %weight[%tile_base] : buffer -> view<64xf16> + %packet_half = index.div %packet, %c4 : index + %group_base = index.mul %half, %c4 : index + %d16 = view.load %ds[%row_lane] : view<64xf16> -> f16 + %d = scalar.extf %d16 : f16 to f32 + %s = vector.load %scales[%row_lane, %packet_half, %group_base] : view<64x2x8xi8> -> vector<4xi8> + %ql0 = vector.load %raw[%half, %c0, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<1xi32> + %ql1 = vector.load %raw[%half, %c1, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<1xi32> + %qh = vector.load %raw[%half, %c2, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<1xi32> + func.return %d, %s, %ql0, %ql1, %qh : f32, vector<4xi8>, vector<1xi32>, vector<1xi32>, vector<1xi32> +} + +func.def inline @ggml_q6k_row64_pair_preloaded_f32_vector4(%group: index, %d: f32, %scale_i8: i8, %ql_word: vector<1xi32>, %qh_word: vector<1xi32>) -> (vector<4xf32>) { + %c2 = index.constant 2 : index + %c4 = index.constant 4 : index + %c4v = vector.constant 4 : vector<1xi32> + %nibble_mask = vector.constant 252645135 : vector<1xi32> + %high_mask = vector.constant 50529027 : vector<1xi32> + %nibble = index.div %group, %c2 : index + %ls = index.mul %nibble, %c4 : index + %hs = index.mul %group, %c2 : index + %ls32 = index.cast %ls : index to i32 + %hs32 = index.cast %hs : index to i32 + %lsv = vector.splat %ls32 : vector<1xi32> + %hsv = vector.splat %hs32 : vector<1xi32> + %lshift = vector.shrui %ql_word, %lsv : vector<1xi32> + %hshift = vector.shrui %qh_word, %hsv : vector<1xi32> + %ql = vector.andi %lshift, %nibble_mask : vector<1xi32> + %qh_low = vector.andi %hshift, %high_mask : vector<1xi32> + %qh = vector.shli %qh_low, %c4v : vector<1xi32> + %codes = vector.ori %ql, %qh : vector<1xi32> + %bytes = vector.bitcast %codes : vector<1xi32> to vector<4xi8> + %c32 = vector.constant 32.0 : vector<4xf32> + %unbiased = vector.uitofp %bytes : vector<4xi8> to vector<4xf32> + %wide = vector.subf %unbiased, %c32 : vector<4xf32> + %scale = scalar.sitofp %scale_i8 : i8 to f32 + %combined = scalar.mulf %scale, %d : f32 + %sv = vector.splat %combined : vector<4xf32> + %result = vector.mulf %wide, %sv : vector<4xf32> + func.return %result : vector<4xf32> +} + func.def inline @ggml_q5k_pair_load_decode_state(%weight: buffer, %row_base: offset, %block: index, %packet: index, %group0: index) -> (vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>, vector<1xi32>) { %c0 = index.constant 0 : index %c4 = index.constant 4 : index @@ -238,81 +300,97 @@ func.def inline @ggml_matmul_vector_decode_pair_block_row(%iq4nl_table: vector<1 } %pair_q5_format = index.constant 5 : index %pair_q5 = index.cmp eq, %weight_format, %pair_q5_format : index - %weight0, %weight1, %weight2, %weight3 = scf.if %pair_q5 -> (vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32>) { - %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload0, %payload2 = func.call @ggml_q5k_pair_load_decode_state(%weight, %row_base, %safe_block, %packet, %group0) : (buffer, offset, index, index, index) -> (vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>, vector<1xi32>) - %weight0 = scf.if %valid_k0 -> (vector<4xf32>) { - %result = func.call @ggml_q5k_pair_preloaded_f32_vector4(%group0, %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload0) : (index, vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>) -> (vector<4xf32>) - scf.yield %result : vector<4xf32> - } else { - scf.yield %zero : vector<4xf32> - } - %weight1 = scf.if %valid_k1 -> (vector<4xf32>) { - %result = func.call @ggml_q5k_pair_preloaded_f32_vector4(%group1, %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload0) : (index, vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>) -> (vector<4xf32>) - scf.yield %result : vector<4xf32> - } else { - scf.yield %zero : vector<4xf32> - } - %weight2 = scf.if %valid_k2 -> (vector<4xf32>) { - %result = func.call @ggml_q5k_pair_preloaded_f32_vector4(%group2, %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload2) : (index, vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>) -> (vector<4xf32>) - scf.yield %result : vector<4xf32> - } else { - scf.yield %zero : vector<4xf32> - } - %weight3 = scf.if %valid_k3 -> (vector<4xf32>) { - %result = func.call @ggml_q5k_pair_preloaded_f32_vector4(%group3, %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload2) : (index, vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>) -> (vector<4xf32>) - scf.yield %result : vector<4xf32> - } else { - scf.yield %zero : vector<4xf32> - } - scf.yield %weight0, %weight1, %weight2, %weight3 : vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32> + %q6_row64_format = index.constant 46 : index + %pair_q6 = index.cmp eq, %weight_format, %q6_row64_format : index + %weight0, %weight1, %weight2, %weight3 = scf.if %pair_q6 -> (vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32>) { + %d, %scales, %ql0, %ql1, %qh = func.call @ggml_q6k_row64_pair_load_decode_state(%weight, %row, %input_size, %safe_block, %packet, %vector_half) : (buffer, index, index, index, index, index) -> (f32, vector<4xi8>, vector<1xi32>, vector<1xi32>, vector<1xi32>) + %s0 = vector.extract %scales[0] : vector<4xi8> -> i8 + %w0 = func.call @ggml_q6k_row64_pair_preloaded_f32_vector4(%c0, %d, %s0, %ql0, %qh) : (index, f32, i8, vector<1xi32>, vector<1xi32>) -> (vector<4xf32>) + %s1 = vector.extract %scales[1] : vector<4xi8> -> i8 + %w1 = func.call @ggml_q6k_row64_pair_preloaded_f32_vector4(%c1, %d, %s1, %ql1, %qh) : (index, f32, i8, vector<1xi32>, vector<1xi32>) -> (vector<4xf32>) + %s2 = vector.extract %scales[2] : vector<4xi8> -> i8 + %w2 = func.call @ggml_q6k_row64_pair_preloaded_f32_vector4(%c2, %d, %s2, %ql0, %qh) : (index, f32, i8, vector<1xi32>, vector<1xi32>) -> (vector<4xf32>) + %s3 = vector.extract %scales[3] : vector<4xi8> -> i8 + %w3 = func.call @ggml_q6k_row64_pair_preloaded_f32_vector4(%c3, %d, %s3, %ql1, %qh) : (index, f32, i8, vector<1xi32>, vector<1xi32>) -> (vector<4xf32>) + scf.yield %w0, %w1, %w2, %w3 : vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32> } else { - %weight0 = scf.if %valid_k0 -> (vector<4xf32>) { - %decoded = scf.if %metadata_enabled -> (vector<4xf32>) { - %v = func.call @ggml_q1_0_preloaded_f32_vector4(%weight, %row_base, %k0, %cached0) : (buffer, offset, index, vector<1xf16>) -> (vector<4xf32>) - scf.yield %v : vector<4xf32> + %weight0, %weight1, %weight2, %weight3 = scf.if %pair_q5 -> (vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32>) { + %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload0, %payload2 = func.call @ggml_q5k_pair_load_decode_state(%weight, %row_base, %safe_block, %packet, %group0) : (buffer, offset, index, index, index) -> (vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>, vector<1xi32>) + %weight0 = scf.if %valid_k0 -> (vector<4xf32>) { + %result = func.call @ggml_q5k_pair_preloaded_f32_vector4(%group0, %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload0) : (index, vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>) -> (vector<4xf32>) + scf.yield %result : vector<4xf32> } else { - %fallback = func.call @ggml_dequant_f32_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight, %row_base, %row, %input_size, %safe_block, %group0, %packet, %k0) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index, index) -> (vector<4xf32>) - scf.yield %fallback : vector<4xf32> + scf.yield %zero : vector<4xf32> } - scf.yield %decoded : vector<4xf32> - } else { - scf.yield %zero : vector<4xf32> - } - %weight1 = scf.if %valid_k1 -> (vector<4xf32>) { - %decoded = scf.if %metadata_enabled -> (vector<4xf32>) { - %v = func.call @ggml_q1_0_preloaded_f32_vector4(%weight, %row_base, %k1, %cached0) : (buffer, offset, index, vector<1xf16>) -> (vector<4xf32>) - scf.yield %v : vector<4xf32> + %weight1 = scf.if %valid_k1 -> (vector<4xf32>) { + %result = func.call @ggml_q5k_pair_preloaded_f32_vector4(%group1, %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload0) : (index, vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>) -> (vector<4xf32>) + scf.yield %result : vector<4xf32> } else { - %fallback = func.call @ggml_dequant_f32_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight, %row_base, %row, %input_size, %safe_block, %group1, %packet, %k1) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index, index) -> (vector<4xf32>) - scf.yield %fallback : vector<4xf32> + scf.yield %zero : vector<4xf32> } - scf.yield %decoded : vector<4xf32> - } else { - scf.yield %zero : vector<4xf32> - } - %weight2 = scf.if %valid_k2 -> (vector<4xf32>) { - %decoded = scf.if %metadata_enabled -> (vector<4xf32>) { - %v = func.call @ggml_q1_0_preloaded_f32_vector4(%weight, %row_base, %k2, %cached0) : (buffer, offset, index, vector<1xf16>) -> (vector<4xf32>) - scf.yield %v : vector<4xf32> + %weight2 = scf.if %valid_k2 -> (vector<4xf32>) { + %result = func.call @ggml_q5k_pair_preloaded_f32_vector4(%group2, %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload2) : (index, vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>) -> (vector<4xf32>) + scf.yield %result : vector<4xf32> } else { - %fallback = func.call @ggml_dequant_f32_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight, %row_base, %row, %input_size, %safe_block, %group2, %packet, %k2) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index, index) -> (vector<4xf32>) - scf.yield %fallback : vector<4xf32> + scf.yield %zero : vector<4xf32> } - scf.yield %decoded : vector<4xf32> - } else { - scf.yield %zero : vector<4xf32> - } - %weight3 = scf.if %valid_k3 -> (vector<4xf32>) { - %decoded = scf.if %metadata_enabled -> (vector<4xf32>) { - %v = func.call @ggml_q1_0_preloaded_f32_vector4(%weight, %row_base, %k3, %cached0) : (buffer, offset, index, vector<1xf16>) -> (vector<4xf32>) - scf.yield %v : vector<4xf32> + %weight3 = scf.if %valid_k3 -> (vector<4xf32>) { + %result = func.call @ggml_q5k_pair_preloaded_f32_vector4(%group3, %pair_q5k_dm, %pair_q5k_scales, %pair_q5k_qh_bytes, %payload2) : (index, vector<2xf16>, vector<3xi32>, vector<4xi8>, vector<1xi32>) -> (vector<4xf32>) + scf.yield %result : vector<4xf32> } else { - %fallback = func.call @ggml_dequant_f32_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight, %row_base, %row, %input_size, %safe_block, %group3, %packet, %k3) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index, index) -> (vector<4xf32>) - scf.yield %fallback : vector<4xf32> + scf.yield %zero : vector<4xf32> } - scf.yield %decoded : vector<4xf32> + scf.yield %weight0, %weight1, %weight2, %weight3 : vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32> } else { - scf.yield %zero : vector<4xf32> + %weight0 = scf.if %valid_k0 -> (vector<4xf32>) { + %decoded = scf.if %metadata_enabled -> (vector<4xf32>) { + %v = func.call @ggml_q1_0_preloaded_f32_vector4(%weight, %row_base, %k0, %cached0) : (buffer, offset, index, vector<1xf16>) -> (vector<4xf32>) + scf.yield %v : vector<4xf32> + } else { + %fallback = func.call @ggml_dequant_f32_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight, %row_base, %row, %input_size, %safe_block, %group0, %packet, %k0) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index, index) -> (vector<4xf32>) + scf.yield %fallback : vector<4xf32> + } + scf.yield %decoded : vector<4xf32> + } else { + scf.yield %zero : vector<4xf32> + } + %weight1 = scf.if %valid_k1 -> (vector<4xf32>) { + %decoded = scf.if %metadata_enabled -> (vector<4xf32>) { + %v = func.call @ggml_q1_0_preloaded_f32_vector4(%weight, %row_base, %k1, %cached0) : (buffer, offset, index, vector<1xf16>) -> (vector<4xf32>) + scf.yield %v : vector<4xf32> + } else { + %fallback = func.call @ggml_dequant_f32_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight, %row_base, %row, %input_size, %safe_block, %group1, %packet, %k1) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index, index) -> (vector<4xf32>) + scf.yield %fallback : vector<4xf32> + } + scf.yield %decoded : vector<4xf32> + } else { + scf.yield %zero : vector<4xf32> + } + %weight2 = scf.if %valid_k2 -> (vector<4xf32>) { + %decoded = scf.if %metadata_enabled -> (vector<4xf32>) { + %v = func.call @ggml_q1_0_preloaded_f32_vector4(%weight, %row_base, %k2, %cached0) : (buffer, offset, index, vector<1xf16>) -> (vector<4xf32>) + scf.yield %v : vector<4xf32> + } else { + %fallback = func.call @ggml_dequant_f32_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight, %row_base, %row, %input_size, %safe_block, %group2, %packet, %k2) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index, index) -> (vector<4xf32>) + scf.yield %fallback : vector<4xf32> + } + scf.yield %decoded : vector<4xf32> + } else { + scf.yield %zero : vector<4xf32> + } + %weight3 = scf.if %valid_k3 -> (vector<4xf32>) { + %decoded = scf.if %metadata_enabled -> (vector<4xf32>) { + %v = func.call @ggml_q1_0_preloaded_f32_vector4(%weight, %row_base, %k3, %cached0) : (buffer, offset, index, vector<1xf16>) -> (vector<4xf32>) + scf.yield %v : vector<4xf32> + } else { + %fallback = func.call @ggml_dequant_f32_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight, %row_base, %row, %input_size, %safe_block, %group3, %packet, %k3) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index, index) -> (vector<4xf32>) + scf.yield %fallback : vector<4xf32> + } + scf.yield %decoded : vector<4xf32> + } else { + scf.yield %zero : vector<4xf32> + } + scf.yield %weight0, %weight1, %weight2, %weight3 : vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32> } scf.yield %weight0, %weight1, %weight2, %weight3 : vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32> } diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index ec1a23ffb346..7704d2848189 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -9604,6 +9604,9 @@ static void register_basic_ops_cases(Suite & suite) { } } } + suite.device_case("qwen3.q6_K.row64.vocabulary151936.f32_reference", [] { + run_dense_matmul_f32_reference_case(GGML_TYPE_Q6_K, 1, 1024, 151936); + }); for (const int64_t tokens : { 64, 127 }) { suite.device_case("qwen3.q6_K.k1024.m64_boundary.tokens" + std::to_string(tokens), [tokens] { run_dense_matmul_f32_reference_case(GGML_TYPE_Q6_K, tokens, 1024, 1024); From 834ebe5bf9a4f70f8ef81a063c0bba6354fc41b4 Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Mon, 16 Nov 2026 00:38:16 +0000 Subject: [PATCH 06/18] hrx: use generic gfx11 row64 Q6 projection --- .../qwen_moe/ggml/linear_q6k_q8_1_x4.loom | 4 +- .../kernels/qwen_moe/manifest.json | 67 +------------------ 2 files changed, 3 insertions(+), 68 deletions(-) diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom index 86bef8d4559f..7274574e966f 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom @@ -341,7 +341,7 @@ check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6 check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6k_q8_1_x4_dense_v_prefill_2048 {token_count = 2048} -amdgpu.target @ggml_q6k_row64_gfx1100_wave64 {subgroup_size = 64} +amdgpu.target @ggml_q6k_row64_gfx11_wave64 {subgroup_size = 64} func.decl @ggml_q6k_row64_tile_lane_i8_scale(%weight: buffer, %tile: index, %row_lane: index, %q6_group: index, %packet: index) -> (vector<4xi8>, f32) @@ -377,7 +377,7 @@ func.def inline @ggml_q6k_q8_1_x4_row64_row_lane_wave64(%input_size: index, %row func.return %sum : f32 } -kernel.def target(@ggml_q6k_row64_gfx1100_wave64) export("ggml_linear_q6k_q8_1_x4_row64") @ggml_linear_q6k_q8_1_x4_row64_wave64(%token_count: index, %input_size: index, %output_size: index) { +kernel.def target(@ggml_q6k_row64_gfx11_wave64) export("ggml_linear_q6k_q8_1_x4_row64") @ggml_linear_q6k_q8_1_x4_row64_wave64(%token_count: index, %input_size: index, %output_size: index) { %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index %c3 = index.constant 3 : index diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json index 23a760375e66..eb59e8392a31 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json @@ -190,71 +190,6 @@ "../loom-libs/motifs/dequant.loom" ] }, - { - "name": "ggml_linear_q6k_q8_1_x4_row64", - "symbol": "ggml_linear_q6k_q8_1_x4_row64", - "source": "ggml/linear_q6k_q8_1_x4.loom", - "workload_parameters": [ - { - "name": "token_count", - "type": "index" - }, - { - "name": "input_size", - "type": "index" - }, - { - "name": "output_size", - "type": "index" - } - ], - "launch_parameters": [ - { - "name": "token_count", - "type": "index" - }, - { - "name": "input_size", - "type": "index" - }, - { - "name": "output_size", - "type": "index" - } - ], - "bindings": [ - "q8_input", - "weight", - "output" - ], - "binding_access": [ - "read", - "read", - "read_write" - ], - "target_selector": "", - "compile_recipe": { - "mode": "archive", - "link_module": "linear_q6k_q8_1_x4_linked", - "primary_sources": [ - "ggml/linear_q6k_q8_1_x4.loom" - ], - "library_sources": [ - "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", - "../loom-libs/motifs/q6_k_f16.loom", - "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom", - "../loom-libs/motifs/dequant.loom" - ] - }, - "compile_dependencies": [ - "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", - "../loom-libs/motifs/q6_k_f16.loom", - "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom", - "../loom-libs/motifs/dequant.loom" - ] - }, { "name": "ggml_linear_q6k_q8_1_x4_row64", "symbol": "ggml_linear_q6k_q8_1_x4_row64_wave64", @@ -297,7 +232,7 @@ "read", "read_write" ], - "target_selector": "gfx1100", + "target_selector": "", "compile_recipe": { "mode": "archive", "link_module": "linear_q6k_q8_1_x4_linked", From a27a4bef89107504966f23b8f02bd9797c2efc26 Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Mon, 16 Nov 2026 00:39:08 +0000 Subject: [PATCH 07/18] hrx: adapt row64 additions to upstream vector splat syntax --- .../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom | 2 +- .../loom-libs/motifs/matmul/vector_decode_pair.loom | 8 ++++---- .../kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom index 5b2c2ddf5540..6be2c8f2b710 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom @@ -19,7 +19,7 @@ func.def inline @ggml_q6k_q8_1_x4_row64_row_lane(%input_size: index, %row: index %c64 = index.constant 64 : index %c256 = index.constant 256 : index %zero = scalar.constant 0.0 : f32 - %zero_i32v = vector.constant 0 : vector<1xi32> + %zero_i32v = vector.splati 0 : vector<1xi32> %block_count = index.div %bounded_input_size, %c256 : index %row_group = index.div %row, %c64 : index %row_lane = index.rem %row, %c64 : index diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_decode_pair.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_decode_pair.loom index c7ca1b407a6a..d827e19929c1 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_decode_pair.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_decode_pair.loom @@ -49,9 +49,9 @@ func.def inline @ggml_q6k_row64_pair_load_decode_state(%weight: buffer, %row: in func.def inline @ggml_q6k_row64_pair_preloaded_f32_vector4(%group: index, %d: f32, %scale_i8: i8, %ql_word: vector<1xi32>, %qh_word: vector<1xi32>) -> (vector<4xf32>) { %c2 = index.constant 2 : index %c4 = index.constant 4 : index - %c4v = vector.constant 4 : vector<1xi32> - %nibble_mask = vector.constant 252645135 : vector<1xi32> - %high_mask = vector.constant 50529027 : vector<1xi32> + %c4v = vector.splati 4 : vector<1xi32> + %nibble_mask = vector.splati 252645135 : vector<1xi32> + %high_mask = vector.splati 50529027 : vector<1xi32> %nibble = index.div %group, %c2 : index %ls = index.mul %nibble, %c4 : index %hs = index.mul %group, %c2 : index @@ -66,7 +66,7 @@ func.def inline @ggml_q6k_row64_pair_preloaded_f32_vector4(%group: index, %d: f3 %qh = vector.shli %qh_low, %c4v : vector<1xi32> %codes = vector.ori %ql, %qh : vector<1xi32> %bytes = vector.bitcast %codes : vector<1xi32> to vector<4xi8> - %c32 = vector.constant 32.0 : vector<4xf32> + %c32 = vector.splatf 32.0 : vector<4xf32> %unbiased = vector.uitofp %bytes : vector<4xi8> to vector<4xf32> %wide = vector.subf %unbiased, %c32 : vector<4xf32> %scale = scalar.sitofp %scale_i8 : i8 to f32 diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom index 7274574e966f..c86b2e06d771 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom @@ -354,7 +354,7 @@ func.def inline @ggml_q6k_q8_1_x4_row64_row_lane_wave64(%input_size: index, %row %c64 = index.constant 64 : index %c256 = index.constant 256 : index %zero = scalar.constant 0.0 : f32 - %zero_i32v = vector.constant 0 : vector<1xi32> + %zero_i32v = vector.splati 0 : vector<1xi32> %block_count = index.div %bounded_input_size, %c256 : index %row_group = index.div %row, %c64 : index %row_lane = index.rem %row, %c64 : index From 3649b968a1ee2e27df4a7458e7d3a89387c856a0 Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Mon, 16 Nov 2026 00:45:31 +0000 Subject: [PATCH 08/18] hrx: package row64 performance reproduction for gfx11 --- .../tools/benchmarks/row64-161/README.md | 66 ++++++++++++++++ .../tools/benchmarks/row64-161/native.json | 71 +++++++++++++++++ .../tools/benchmarks/row64-161/native.loom | 21 +++++ .../tools/benchmarks/row64-161/row64.json | 71 +++++++++++++++++ .../tools/benchmarks/row64-161/row64.loom | 21 +++++ .../tools/benchmarks/row64-161/run.py | 77 +++++++++++++++++++ 6 files changed, 327 insertions(+) create mode 100644 ggml/src/ggml-hrx/tools/benchmarks/row64-161/README.md create mode 100644 ggml/src/ggml-hrx/tools/benchmarks/row64-161/native.json create mode 100644 ggml/src/ggml-hrx/tools/benchmarks/row64-161/native.loom create mode 100644 ggml/src/ggml-hrx/tools/benchmarks/row64-161/row64.json create mode 100644 ggml/src/ggml-hrx/tools/benchmarks/row64-161/row64.loom create mode 100644 ggml/src/ggml-hrx/tools/benchmarks/row64-161/run.py diff --git a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/README.md b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/README.md new file mode 100644 index 000000000000..eb5ce88f2cd7 --- /dev/null +++ b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/README.md @@ -0,0 +1,66 @@ +# Issue 161 performance reproduction + +Compare PR196 (branch `users/astgeorg/qwen3-row64-161`) against upstream +`4a5e53048` after rebasing. The candidate includes the generic gfx11 wave64 +Q6 x Q8 row64 projection. This experiment does not include F32 accumulation +or activation-LDS reuse from the separate accuracy investigation. + +Build both sources as RelWithDebInfo with HRX +`8d78611874afe8b7aaaf0227e28e4c3bd632bc99` and the same ROCm SDK/compiler. +The reference SDK is `10.1.0a20260822-1100`. Use each machine's supported +SDK if needed and report the difference. Enable `GGML_HRX`, set +`HRX_SOURCE_DIR` and `IREE_ROCM_PATH`; build `llama-bench` and `hrx-ops-test`. +The CMake dependency build also produces loom-link and loom-benchmark. +Pin `ROCR_VISIBLE_DEVICES` to the actual gfx1100 or gfx1151 UUID; set the +SDK library directory in `LD_LIBRARY_PATH`. Record GPU, power mode, clocks, +RAM, compiler, exact source commits, model hashes and binary hashes. + +Run from the candidate checkout, supplying absolute paths: + +```sh +python3 ggml/src/ggml-hrx/tools/benchmarks/row64-161/run.py \ + --baseline-source "$BASELINE_SOURCE" --baseline-build "$BASELINE_BUILD" \ + --candidate-source "$CANDIDATE_SOURCE" --candidate-build "$CANDIDATE_BUILD" \ + --models "$MODELS" --output "$RESULTS" +``` + +The runner starts with kernel measurements, then runs the same six models +and pp66/pp128/pp512/tg32 scenarios as the original sweep. Model names are +listed in run.py; use the existing shared model store. ABBA ordering, +five samples per run, ten samples per build/scenario, batch/ubatch 512, +four CPU threads, flash attention, GPU offload and default warmup match +the original measurement. Summarize pooled median throughput changes and +individual run medians; retain all samples and failures. Small differences +are screening results, not statistical significance claims. + +Kernel fixtures capture the original generic vocabulary projection at +K=1024, N=151936, one token. Measure native/row64 on both baseline and +candidate in forward/reverse order. This adds a native candidate control; +it excludes discarded prototype variants. Four binding sets total +515,383,296 bytes, batch eight, five warmups, minimum 30 iterations. +Report device replay timing separately from normalized host completion, +alongside VGPR/LDS and instruction counts. Fixtures contain zero data and +have no numerical oracle. Run relevant nonzero hrx-ops-test coverage +separately before interpreting timings as acceptance evidence. + +Also capture model command programs with `GGML_HRX_DUMP_COMMAND_PROGRAM_DIR` +to verify selection. The generic F32 vocabulary benchmark above does not +exercise the newly generalized Q6 x Q8 wave64 projection. Compile/run its +`ggml_linear_q6k_q8_1_x4_row64_wave64_decode` and +`ggml_linear_q6k_q8_1_x4_row64_wave64_prefill_17` Loom benchmarks from +qwen_moe/ggml/linear_q6k_q8_1_x4.loom (K=2048,N=151936), comparing native +projection controls. Link the manifest's library_sources, set token_capacity +1/17 respectively and output_capacity 151936. Verify wave64 selection on +gfx1151 and run endpoint numerical tests. Any performance gap starts with +matched kernel-level investigation, not model-level tuning. + +Original gfx1100 model deltas are in PR196's description: decode ranged +-0.58% to +0.21%; PromptBridge pp66 retained -2.06%. Generic vocabulary +kernel device times were native 143.498 us, row64 before reuse 164.550 us, +retained reuse 145.218 us. Those used HRX82214df and baseline010b2ee; +they are historical comparisons, not results for this rebase or gfx1151. + +Use an idle GPU and a shared advisory lock (runner defaults to +/tmp/lemonade-ws-tek-gpu.lock). Report instability and power/clock drift; +do not discard slow samples. Publish findings to lemonade-ws-aj0j and +lemonade-ws-s9tq, with exact commands and source identities. diff --git a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/native.json b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/native.json new file mode 100644 index 000000000000..4114516e1440 --- /dev/null +++ b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/native.json @@ -0,0 +1,71 @@ +{ + "command_count": 1, + "dispatch_count": 1, + "dispatches": [ + { + "benchmark": "@promptbridge_native_tg32_000_ggml_mul_mat_vector_f32_f32", + "compile_parameters": { + "ggml.matmul.vector.input_size": "1024", + "ggml.matmul.vector.output_accumulation": "0", + "ggml.matmul.vector.output_size": "151936", + "ggml.matmul.vector.output_unary_op": "23", + "ggml.matmul.vector.publish_format": "0", + "ggml.matmul.vector.weight_format": "6", + "ggml.workload.token_capacity": "1" + }, + "corpus_dir": "kernel-corpus/kernels/loom-libs", + "count": 1, + "integer_parameters": { + "token_count": 1 + }, + "kernel": "loom_libs:ggml_mul_mat_vector_f32_f32", + "library_sources": [ + "motifs/matmul/vector_activation.loom", + "motifs/matmul/vector_decode_pair.loom", + "motifs/matmul/vector_publish.loom", + "motifs/dequant.loom", + "motifs/elementwise/unary_apply.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ], + "primary_sources": [ + "ops/matmul/vector_f32_f32.loom" + ], + "sources": [ + "ops/matmul/vector_f32_f32.loom", + "motifs/matmul/vector_activation.loom", + "motifs/matmul/vector_decode_pair.loom", + "motifs/matmul/vector_publish.loom", + "motifs/dequant.loom", + "motifs/elementwise/unary_apply.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ], + "symbol": "ggml_mul_mat_vector_f32_f32", + "workload_parameters": [ + { + "name": "token_count", + "type": "index" + } + ] + } + ], + "generated_count": 1, + "kernel_counts": { + "loom_libs:ggml_mul_mat_vector_f32_f32": 1 + }, + "loom_source": "native.loom", + "model": "promptbridge_native", + "scenario": "tg32", + "schema": "ggml-hrx-model-loom-benchmarks-v2" +} diff --git a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/native.loom b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/native.loom new file mode 100644 index 000000000000..1f0eb477718d --- /dev/null +++ b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/native.loom @@ -0,0 +1,21 @@ +// Generated by tools/benchmarks/generate-model-benchmarks.py for promptbridge_native scenarios: tg32. +// Regenerate from an HRX command program dump rather than editing by hand. + +target.decl @ggml_matmul_vector_gfx11_wave64 + +kernel.decl target(@ggml_matmul_vector_gfx11_wave64) @ggml_mul_mat_vector_f32_f32(%token_count: index) launch(%token_count: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) + + +// Scenario: tg32 + +check.case public @promptbridge_native_tg32_000_ggml_mul_mat_vector_f32_f32_case { + %token_count = check.literal value(1) : index + %input = check.generate.fill value(0) : tensor<4096xi8> + %weight = check.generate.fill value(0) : tensor<127626240xi8> + %output = check.generate.fill value(0) : tensor<607744xi8> + %next_output = check.generate.fill value(0) : tensor<607744xi8> + kernel.launch @ggml_mul_mat_vector_f32_f32[%token_count](%token_count, %input, %weight, %output, %next_output) : [index](index, tensor<4096xi8>, tensor<127626240xi8>, tensor<607744xi8>, tensor<607744xi8>) + check.return +} + +check.benchmark<@promptbridge_native_tg32_000_ggml_mul_mat_vector_f32_f32_case> @promptbridge_native_tg32_000_ggml_mul_mat_vector_f32_f32 diff --git a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/row64.json b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/row64.json new file mode 100644 index 000000000000..d57a2de67c46 --- /dev/null +++ b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/row64.json @@ -0,0 +1,71 @@ +{ + "command_count": 1, + "dispatch_count": 1, + "dispatches": [ + { + "benchmark": "@promptbridge_row64_tg32_000_ggml_mul_mat_vector_f32_f32", + "compile_parameters": { + "ggml.matmul.vector.input_size": "1024", + "ggml.matmul.vector.output_accumulation": "0", + "ggml.matmul.vector.output_size": "151936", + "ggml.matmul.vector.output_unary_op": "23", + "ggml.matmul.vector.publish_format": "0", + "ggml.matmul.vector.weight_format": "46", + "ggml.workload.token_capacity": "1" + }, + "corpus_dir": "kernel-corpus/kernels/loom-libs", + "count": 1, + "integer_parameters": { + "token_count": 1 + }, + "kernel": "loom_libs:ggml_mul_mat_vector_f32_f32", + "library_sources": [ + "motifs/matmul/vector_activation.loom", + "motifs/matmul/vector_decode_pair.loom", + "motifs/matmul/vector_publish.loom", + "motifs/dequant.loom", + "motifs/elementwise/unary_apply.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ], + "primary_sources": [ + "ops/matmul/vector_f32_f32.loom" + ], + "sources": [ + "ops/matmul/vector_f32_f32.loom", + "motifs/matmul/vector_activation.loom", + "motifs/matmul/vector_decode_pair.loom", + "motifs/matmul/vector_publish.loom", + "motifs/dequant.loom", + "motifs/elementwise/unary_apply.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ], + "symbol": "ggml_mul_mat_vector_f32_f32", + "workload_parameters": [ + { + "name": "token_count", + "type": "index" + } + ] + } + ], + "generated_count": 1, + "kernel_counts": { + "loom_libs:ggml_mul_mat_vector_f32_f32": 1 + }, + "loom_source": "row64.loom", + "model": "promptbridge_row64", + "scenario": "tg32", + "schema": "ggml-hrx-model-loom-benchmarks-v2" +} diff --git a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/row64.loom b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/row64.loom new file mode 100644 index 000000000000..54c4e540f065 --- /dev/null +++ b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/row64.loom @@ -0,0 +1,21 @@ +// Generated by tools/benchmarks/generate-model-benchmarks.py for promptbridge_row64 scenarios: tg32. +// Regenerate from an HRX command program dump rather than editing by hand. + +target.decl @ggml_matmul_vector_gfx11_wave64 + +kernel.decl target(@ggml_matmul_vector_gfx11_wave64) @ggml_mul_mat_vector_f32_f32(%token_count: index) launch(%token_count: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) + + +// Scenario: tg32 + +check.case public @promptbridge_row64_tg32_000_ggml_mul_mat_vector_f32_f32_case { + %token_count = check.literal value(1) : index + %input = check.generate.fill value(0) : tensor<4096xi8> + %weight = check.generate.fill value(0) : tensor<127626240xi8> + %output = check.generate.fill value(0) : tensor<607744xi8> + %next_output = check.generate.fill value(0) : tensor<607744xi8> + kernel.launch @ggml_mul_mat_vector_f32_f32[%token_count](%token_count, %input, %weight, %output, %next_output) : [index](index, tensor<4096xi8>, tensor<127626240xi8>, tensor<607744xi8>, tensor<607744xi8>) + check.return +} + +check.benchmark<@promptbridge_row64_tg32_000_ggml_mul_mat_vector_f32_f32_case> @promptbridge_row64_tg32_000_ggml_mul_mat_vector_f32_f32 diff --git a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/run.py b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/run.py new file mode 100644 index 000000000000..929616f080bf --- /dev/null +++ b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/run.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Repeat issue 161 performance comparisons with matched runtime builds. + +Keep raw output and command provenance; synthetic kernel timings have no oracle. +ROCR_VISIBLE_DEVICES and the ROCm runtime library path must be set by the caller. +""" +import argparse +import fcntl +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +MODELS = { + 'promptbridge06': 'PromptBridge-0.6b-Alpha-GGUF/PromptBridge-0.6b-Alpha.Q4_K_M.gguf', + 'qwen3_17': 'Qwen3-1.7B-GGUF/Qwen3-1.7B-Q4_0.gguf', + 'qwen3_4': 'Qwen3-4B-Instruct-2507-GGUF/Qwen3-4B-Instruct-2507-Q4_K_M.gguf', + 'qwen35_08': 'Qwen3.5-0.8B-GGUF/Qwen3.5-0.8B-UD-Q4_K_XL.gguf', + 'qwen35_2': 'Qwen3.5-2B-GGUF/Qwen3.5-2B-UD-Q4_K_XL.gguf', + 'qwen35_9': 'Qwen3.5-9B-GGUF/Qwen3.5-9B-UD-Q4_K_XL.gguf', +} +parser = argparse.ArgumentParser(description=__doc__) +for option in ('baseline-source', 'baseline-build', 'candidate-source', 'candidate-build', 'models', 'output'): + parser.add_argument('--'+option, type=Path, required=True) +parser.add_argument('--phase', choices=('kernel', 'model', 'all'), default='all') +parser.add_argument('--lock', default='/tmp/lemonade-ws-tek-gpu.lock') +a = parser.parse_args() +if not os.environ.get('ROCR_VISIBLE_DEVICES'): + parser.error('set ROCR_VISIBLE_DEVICES to the supported GPU') +for key, value in vars(a).items(): + if isinstance(value, Path): + setattr(a, key, value.resolve()) +a.output.mkdir(parents=True, exist_ok=True) +sources = {'baseline': a.baseline_source, 'candidate': a.candidate_source} +builds = {'baseline': a.baseline_build, 'candidate': a.candidate_build} +provenance = {k: subprocess.check_output(['git', '-C', str(v), 'rev-parse', 'HEAD'], text=True).strip() for k, v in sources.items()} +(a.output/'sources.json').write_text(json.dumps(provenance, indent=2)) +fixture = Path(__file__).resolve().parent + +def run(folder, argv, env): + folder.mkdir(parents=True, exist_ok=False) + argv = list(map(str, argv)) + (folder/'command.json').write_text(json.dumps({'argv': argv, 'env': env}, indent=2)) + with open(a.lock, 'a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + print('START', folder, flush=True) + start = time.monotonic() + with (folder/'stdout.log').open('w') as stdout, (folder/'stderr.log').open('w') as stderr: + try: + code = subprocess.run(argv, env=os.environ | env, stdout=stdout, stderr=stderr, timeout=1800).returncode + except subprocess.TimeoutExpired: + code = 124 + (folder/'exit.json').write_text(json.dumps({'code': code, 'seconds': time.monotonic()-start})) + print('DONE', folder.name, code, flush=True) + if code: + raise SystemExit(f'Failed: {folder}; retain logs before retrying in a new output directory') + +if a.phase in ('kernel', 'all'): + # Four controls identify layout cost separately from the retained reuse optimization. + order = [('baseline','native'), ('baseline','row64'), ('candidate','row64'), ('candidate','native')] + for i, (stage, layout) in enumerate(order + order[::-1]): + hrx = sources[stage]/'ggml/src/ggml-hrx' + d = json.loads((fixture/(layout+'.json')).read_text()) + d['loom_source'] = str(fixture/(layout+'.loom')) + manifest = a.output/f'{stage}-{layout}.json' + manifest.write_text(json.dumps(d, indent=2)) + argv = [sys.executable, hrx/'tools/benchmarks/run-model-benchmarks.py', '--model', d['model'], '--scenario', 'tg32', '--manifest', manifest, '--build-dir', builds[stage], '--output-dir', a.output/f'kernel-{i}-{stage}-{layout}', '--batch-size', '8', '--iterations', '30', '--warmup-iterations', '5', '--input-ring-count', '4', '--profile-final-batch', 'true'] + run(a.output/f'kernel-command-{i}', argv, {'ROCR_VISIBLE_DEVICES': os.environ['ROCR_VISIBLE_DEVICES']}) + +if a.phase in ('model', 'all'): + for name, model in MODELS.items(): + for i, stage in enumerate(('baseline','candidate','candidate','baseline')): + env = {'ROCR_VISIBLE_DEVICES': os.environ['ROCR_VISIBLE_DEVICES'], 'LD_LIBRARY_PATH': str(builds[stage]/'bin')+':'+os.environ.get('LD_LIBRARY_PATH','')} + argv = [builds[stage]/'bin/llama-bench', '-m', a.models/model, '--device', 'HRX0', '-ngl', '99', '-t', '4', '-fa', 'on', '-p', '66,128,512', '-n', '32', '-b', '512', '-ub', '512', '-r', '5', '-o', 'jsonl'] + run(a.output/name/f'{i}-{stage}', argv, env) From 6837ad78c880c96cdf620af6e4a1122c26e7c421 Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Mon, 16 Nov 2026 00:47:40 +0000 Subject: [PATCH 09/18] hrx: resolve current Loom tools in performance runner --- ggml/src/ggml-hrx/tools/benchmarks/row64-161/run.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/run.py b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/run.py index 929616f080bf..0b39ff4295df 100644 --- a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/run.py +++ b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/run.py @@ -67,7 +67,12 @@ def run(folder, argv, env): manifest = a.output/f'{stage}-{layout}.json' manifest.write_text(json.dumps(d, indent=2)) argv = [sys.executable, hrx/'tools/benchmarks/run-model-benchmarks.py', '--model', d['model'], '--scenario', 'tg32', '--manifest', manifest, '--build-dir', builds[stage], '--output-dir', a.output/f'kernel-{i}-{stage}-{layout}', '--batch-size', '8', '--iterations', '30', '--warmup-iterations', '5', '--input-ring-count', '4', '--profile-final-batch', 'true'] - run(a.output/f'kernel-command-{i}', argv, {'ROCR_VISIBLE_DEVICES': os.environ['ROCR_VISIBLE_DEVICES']}) + tools = builds[stage]/'ggml/src/ggml-hrx/hrx/src/ggml-hrx-deps-build/loom/src/loom/tools' + run(a.output/f'kernel-command-{i}', argv, { + 'ROCR_VISIBLE_DEVICES': os.environ['ROCR_VISIBLE_DEVICES'], + 'LOOM_LINK': str(tools/'loom-link/loom-link'), + 'IREE_BENCHMARK_LOOM': str(tools/'loom-benchmark/loom-benchmark'), + }) if a.phase in ('model', 'all'): for name, model in MODELS.items(): From bf368529a15e8715fd6d211692236046bd637b3d Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Mon, 16 Nov 2026 00:52:08 +0000 Subject: [PATCH 10/18] hrx: record rebased gfx1100 performance and projection gap --- .../tools/benchmarks/row64-161/README.md | 5 +- .../row64-161/gfx1100-rebased-results.md | 83 +++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 ggml/src/ggml-hrx/tools/benchmarks/row64-161/gfx1100-rebased-results.md diff --git a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/README.md b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/README.md index eb5ce88f2cd7..eb953186424b 100644 --- a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/README.md +++ b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/README.md @@ -50,7 +50,10 @@ exercise the newly generalized Q6 x Q8 wave64 projection. Compile/run its `ggml_linear_q6k_q8_1_x4_row64_wave64_prefill_17` Loom benchmarks from qwen_moe/ggml/linear_q6k_q8_1_x4.loom (K=2048,N=151936), comparing native projection controls. Link the manifest's library_sources, set token_capacity -1/17 respectively and output_capacity 151936. Verify wave64 selection on +1/17 respectively and output_capacity 151936. Explicitly pass +`--measure=dispatch_complete`; the source benchmark default includes case +setup. Use batch8, warmup5, iterations30, input-ring-count4 and +profile-final-batch=true for separate device timing. Verify wave64 selection on gfx1151 and run endpoint numerical tests. Any performance gap starts with matched kernel-level investigation, not model-level tuning. diff --git a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/gfx1100-rebased-results.md b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/gfx1100-rebased-results.md new file mode 100644 index 000000000000..0390d37f3b14 --- /dev/null +++ b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/gfx1100-rebased-results.md @@ -0,0 +1,83 @@ +# gfx1100 measurements after the rebase + +Baseline 4a5e53048 versus candidate code a27a4bef8; runner 6837ad78c. +Both built RelWithDebInfo using HRX8d78611874afe8b7aaaf0227e28e4c3bd632bc99 +and ROCm10.1.0a20260822-1100. RX7900XTX, GPU-fc2143878bea4591, +automatic power policy. Benchmark-package commits do not alter kernel code. + +All 19 selected nonzero/route tests passed, including native/row64/tied +endpoints, vocabulary embeddings and Q6 residual/short-prefill coverage. +All 24 model benchmark invocations completed. No new perplexity sweep was run. + +## Model throughput + +ABBA, five samples/run, ten samples/build/scenario; batch/ubatch512, +four CPU threads, flash attention, GPU offload, default warmup. Positive +percentages mean faster. All samples are retained, including warmup drift. + +| Model | pp66 | pp128 | pp512 | tg32 | +|---|---:|---:|---:|---:| +| promptbridge06 | -1.38% | +21.07% | -0.72% | +2.23% | +| qwen3_17 | +0.45% | -0.83% | +1.11% | -0.78% | +| qwen3_4 | +5.00% | +13.40% | -0.61% | +0.57% | +| qwen35_08 | +2.99% | +5.17% | +0.98% | -0.55% | +| qwen35_2 | -0.21% | +0.14% | +0.46% | -1.17% | +| qwen35_9 | +0.19% | +0.18% | +0.20% | +0.34% | + +These are screening measurements, not a zero-regression claim. PromptBridge +has substantial run drift: baseline pp128 run medians 18415.7 and15349.5tok/s, +candidate19808.5 and19741.2. Its decode run medians were baseline596.84/612.498 +and candidate647.286/612.436. Do not treat the pooled gains as precise estimates. +Qwen3.5 2B decode measured -1.17%, Qwen3 1.7B -0.78%, and PromptBridge pp66 +-1.38%; these losses remain explicit. This campaign did not tune them. + +## Generic Q6 F32 vocabulary kernel + +K1024,N151936,T1; four binding sets (515383296bytes), batch8, +five warmups, minimum30 iterations/100ms, two runs in forward/reverse order. +All eight runs stopped as stable. Device time is a separate eight-dispatch +profile replay; host time is normalized queue completion. Synthetic zero +buffers have no numerical oracle; use the separate test results above. + +| Variant | Device us, median run mean | Host us, median run p50 | +|---|---:|---:| +| Baseline native |143.530|149.623| +| Baseline row64 |153.845|162.844| +| Candidate row64 |144.855|152.952| +| Candidate native |143.590|151.829| + +Candidate row64 is 5.84% faster than baseline row64 on device, but 0.92% +slower than baseline native. The host comparison shows additional run drift. +This generic kernel is distinct from the Q6xQ8 projection generalized to gfx11. + +Raw logs, commands, JSONL, model/binary hashes and summaries are retained in +the originating workspace under investigation/qwen3-row64-161/rebased. +The tracked runner and fixtures reproduce the procedure on another machine. +The first setup attempt could not resolve the renamed Loom executable and +performed no measurement; the fixed runner completed the campaign. + +## Q6 x Q8 projection + +Candidate source, K2048,N151936, native wave32 versus generic gfx11 row64 +wave64, two runs each in forward/reverse order. Token capacity matches +T1/T17. Explicit dispatch_complete, batch8, warmup5, iterations30, +binding ring4; all eight runs reached stable. Medians of device replay +means and host p50s: + +| Tokens | Native device us | Row64 device us | Native host us | Row64 host us | +|---|---:|---:|---:|---:| +| 1 |289.445|285.790|298.949|300.453| +| 17 |4900.678|5202.768|4886.586|5218.783| + +Decode device time improves1.26%, while host time increases0.50%. +Prefill17 device time increases6.16%, host6.80%. The two prefill device +means were4899.010/4902.345us native and5198.285/5207.250us row64. +This is a repeatable kernel-level gap in this measurement, not proof of +which source change caused it. Preserve it for gfx1151 comparison and +further kernel-first investigation; no tuning was performed here. +Earlier setup probes used case_end_to_end timing (and one used excess +decode token capacity); their logs are retained but excluded from this table. + +Fresh PromptBridge command captures each contain367 commands. Command366 +selects the generic vocabulary kernel at K1024,N151936,T1, changing weight +format6(native) to46(row64), confirming the generic microbenchmark's route. From 13df8559744d98e212406731c048394af345a7ac Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Mon, 16 Nov 2026 13:50:59 +0000 Subject: [PATCH 11/18] hrx: remove row64 investigation scaffolding and cover T17 endpoints --- .../motifs/matmul/q6_q8_1_x4_decode.loom | 39 ------ .../qwen_moe/ggml/linear_q6k_q8_1_x4.loom | 130 ------------------ .../tools/benchmarks/row64-161/README.md | 69 ---------- .../row64-161/gfx1100-rebased-results.md | 83 ----------- .../tools/benchmarks/row64-161/native.json | 71 ---------- .../tools/benchmarks/row64-161/native.loom | 21 --- .../tools/benchmarks/row64-161/row64.json | 71 ---------- .../tools/benchmarks/row64-161/row64.loom | 21 --- .../tools/benchmarks/row64-161/run.py | 82 ----------- tests/hrx-ops-test.cpp | 9 +- 10 files changed, 7 insertions(+), 589 deletions(-) delete mode 100644 ggml/src/ggml-hrx/tools/benchmarks/row64-161/README.md delete mode 100644 ggml/src/ggml-hrx/tools/benchmarks/row64-161/gfx1100-rebased-results.md delete mode 100644 ggml/src/ggml-hrx/tools/benchmarks/row64-161/native.json delete mode 100644 ggml/src/ggml-hrx/tools/benchmarks/row64-161/native.loom delete mode 100644 ggml/src/ggml-hrx/tools/benchmarks/row64-161/row64.json delete mode 100644 ggml/src/ggml-hrx/tools/benchmarks/row64-161/row64.loom delete mode 100644 ggml/src/ggml-hrx/tools/benchmarks/row64-161/run.py diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom index 6be2c8f2b710..c906e25b77bc 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom @@ -7,45 +7,6 @@ func.decl @ggml_q8_1_x4_word(%q8_input: buffer, %row_byte_base: offset, %q8_bloc func.decl @ggml_q6k_f16_vector4(%weight: buffer, %weight_row_byte_base: offset, %q6_block: index, %q6_group: index, %packet: index) -> (vector<4xf16>) -func.decl @ggml_q6k_row64_tile_lane_i8_scale(%weight: buffer, %tile: index, %row_lane: index, %q6_group: index, %packet: index) -> (vector<4xi8>, f32) - -func.def inline @ggml_q6k_q8_1_x4_row64_row_lane(%input_size: index, %row: index, %weight: buffer, %q8_input: buffer, %q8_row_byte_base: offset, %lane: index) -> (f32) { - %bounded_input_size = index.assume %input_size [range(%input_size, 256, 32768), multiple_of(%input_size, 256)] : index - %bounded_lane = index.assume %lane [range(%lane, 0, 31)] : index - %c0 = index.constant 0 : index - %c1 = index.constant 1 : index - %c4 = index.constant 4 : index - %c8 = index.constant 8 : index - %c64 = index.constant 64 : index - %c256 = index.constant 256 : index - %zero = scalar.constant 0.0 : f32 - %zero_i32v = vector.splati 0 : vector<1xi32> - %block_count = index.div %bounded_input_size, %c256 : index - %row_group = index.div %row, %c64 : index - %row_lane = index.rem %row, %c64 : index - %group_low = index.div %bounded_lane, %c8 : index - %packet = index.rem %bounded_lane, %c8 : index - %sum = scf.for %block = [%c0 to %block_count step %c1](%block_acc = %zero : f32) -> (f32) { - %block_q8_base = index.mul %block, %c8 : index - %tile = index.madd %row_group, %block_count, %block : index - %partial = scf.for %half = [%c0 to %c8 step %c4](%half_acc = %block_acc : f32) -> (f32) { - %group = index.add %group_low, %half : index - %weight_values, %weight_scale = func.call @ggml_q6k_row64_tile_lane_i8_scale(%weight, %tile, %row_lane, %group, %packet) : (buffer, index, index, index, index) -> (vector<4xi8>, f32) - %q8_block = index.add %block_q8_base, %group : index - %q8_values, %q8_d = func.call @ggml_q8_1_x4_word(%q8_input, %q8_row_byte_base, %q8_block, %packet) : (buffer, offset, index, index) -> (vector<4xi8>, f32) - %dot_i32v = vector.dot4i %weight_values, %q8_values, %zero_i32v : vector<4xi8>, vector<4xi8>, vector<1xi32> - %dot_i32 = vector.extract %dot_i32v[0] : vector<1xi32> -> i32 - %dot = scalar.sitofp %dot_i32 : i32 to f32 - %scaled_weight = scalar.mulf %dot, %weight_scale : f32 - %scaled = scalar.mulf %scaled_weight, %q8_d : f32 - %next = scalar.addf %half_acc, %scaled : f32 - scf.yield %next : f32 - } - scf.yield %partial : f32 - } - func.return %sum : f32 -} - func.def inline @ggml_q6k_sign_extend_dot4(%code: vector<1xi32>) -> (vector<4xi8>) { %c1_i32v = vector.splati 1 : vector<1xi32> %c2_i32v = vector.splati 2 : vector<1xi32> diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom index c86b2e06d771..9a030722b9f2 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom @@ -27,8 +27,6 @@ func.decl @ggml_q8_1_x4_word(%q8_input: buffer, %row_byte_base: offset, %q8_bloc func.decl @ggml_q6k_q8_1_x4_row_lane_wave64_block4(%input_size: index, %weight: buffer, %weight_row_byte_base: offset, %q8_input: buffer, %q8_row_byte_base: offset, %lane: index) -> (f32) -func.decl @ggml_q6k_q8_1_x4_row64_row_lane(%input_size: index, %row: index, %weight: buffer, %q8_input: buffer, %q8_row_byte_base: offset, %lane: index) -> (f32) - // Dense raw-layout baseline. A 256-thread workgroup carries eight independent // wave32 output rows so decode launches enough waves without duplicating Q6_K // decoding within a row. @@ -69,59 +67,6 @@ kernel.def export("ggml_linear_q6k_q8_1_x4") @ggml_linear_q6k_q8_1_x4(%token_cou kernel.return } -kernel.def export("ggml_linear_q6k_q8_1_x4_row64") @ggml_linear_q6k_q8_1_x4_row64(%token_count: index, %input_size: index, %output_size: index) { - %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index - %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index - %c8 = index.constant 8 : index - %c7 = index.constant 7 : index - %c1 = index.constant 1 : index - %workgroup_size = index.constant 256 : index - %padded_output_size = index.add %output_capacity, %c7 : index - %output_tiles = index.div %padded_output_size, %c8 : index - kernel.launch.config workgroups(%output_tiles, %token_capacity, %c1) workgroup_size(%workgroup_size, %c1, %c1) : index -} launch(%token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) { - %q8_noalias, %weight_noalias, %output_noalias = buffer.assume.noalias %q8_input, %weight, %output : buffer, buffer, buffer - %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index - %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index - %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048), le(%token_count, %token_capacity)] : index - %bounded_input_size = index.assume %input_size [range(%input_size, 256, 32768), multiple_of(%input_size, 256)] : index - %bounded_output_size = index.assume %output_size [range(%output_size, 1, 262144), le(%output_size, %output_capacity), multiple_of(%output_size, 64)] : index - %c0 = index.constant 0 : index - %c8 = index.constant 8 : index - %c128 = index.constant 128 : index - %c144_bytes = index.constant 144 : offset - %token = kernel.workgroup.id : index - %channel_tile = kernel.workgroup.id : index - %subgroup = kernel.subgroup.id : index - %lane = kernel.subgroup.lane.id : index - %channel = index.madd %channel_tile, %c8, %subgroup : index - %valid_token = index.cmp ult, %token, %bounded_token_count : index - %valid_channel = index.cmp ult, %channel, %bounded_output_size : index - %publish = scalar.andi %valid_token, %valid_channel : i1 - %safe_token = scf.select %valid_token, %token, %c0 : index - %safe_channel = scf.select %valid_channel, %channel, %c0 : index - %q8_groups = index.div %bounded_input_size, %c128 : index - %q8_row_bytes = index.scale %q8_groups, %c144_bytes : index, offset -> offset - %q8_row_byte_base = index.scale %safe_token, %q8_row_bytes : index, offset -> offset - %lane_acc = scf.if %publish -> (f32) { - %value = func.call @ggml_q6k_q8_1_x4_row64_row_lane(%bounded_input_size, %safe_channel, %weight_noalias, %q8_noalias, %q8_row_byte_base, %lane) : (index, index, buffer, buffer, offset, index) -> (f32) - scf.yield %value : f32 - } else { - %zero = scalar.constant 0.0 : f32 - scf.yield %zero : f32 - } - %dot = kernel.subgroup.reduce %lane_acc : f32 - %is_lane_zero = index.cmp eq, %lane, %c0 : index - scf.if %publish { - scf.if %is_lane_zero { - %c0_offset = index.constant 0 : offset - %output_view = buffer.view %output_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]x[%bounded_output_size]xf32> - view.store %dot, %output_view[%token, %channel] : f32, view<[%bounded_token_count]x[%bounded_output_size]xf32> - } - } - kernel.return -} - // Wave64 vocabulary schedule matching llama.cpp Vulkan's one-row workgroup and // four-block concurrency on AMD targets with native wave64 execution. kernel.def target(@ggml_q6k_q8_gfx1151_wave64) export("ggml_linear_q6k_q8_1_x4") @ggml_linear_q6k_q8_1_x4_wave64_block4(%token_count: index, %input_size: index, %output_size: index) { @@ -238,19 +183,6 @@ check.case public @ggml_linear_q6k_q8_1_x4_vocabulary_wave32_benchmark_case { check.return } -check.case public @ggml_linear_q6k_q8_1_x4_vocabulary_prefill_benchmark_case { - %token_count = check.literal value(17) : index - %input_size = check.literal value(2048) : index - %output_size = check.literal value(151936) : index - %q8_input = check.generate.fill value(1) : tensor<17x2304xi8> - %weight = check.generate.fill value(0) : tensor<151936x8x210xi8> - %output = check.generate.fill value(1.0) : tensor<17x151936xf32> - %expected = check.generate.fill value(0.0) : tensor<17x151936xf32> - kernel.launch @ggml_linear_q6k_q8_1_x4[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<17x2304xi8>, tensor<151936x8x210xi8>, tensor<17x151936xf32>) - check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<17x151936xf32>, tensor<17x151936xf32> - check.return -} - check.case public @ggml_linear_q6k_q8_1_x4_vocabulary_wave64_block4_benchmark_case { %token_count = check.literal value(1) : index %input_size = check.literal value(2048) : index @@ -264,32 +196,6 @@ check.case public @ggml_linear_q6k_q8_1_x4_vocabulary_wave64_block4_benchmark_ca check.return } -check.case public @ggml_linear_q6k_q8_1_x4_row64_vocabulary_benchmark_case { - %token_count = check.literal value(1) : index - %input_size = check.literal value(2048) : index - %output_size = check.literal value(151936) : index - %q8_input = check.generate.fill value(1) : tensor<1x2304xi8> - %weight = check.generate.fill value(0) : tensor<2374x8x13440xi8> - %output = check.generate.fill value(1.0) : tensor<1x151936xf32> - %expected = check.generate.fill value(0.0) : tensor<1x151936xf32> - kernel.launch @ggml_linear_q6k_q8_1_x4_row64[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<1x2304xi8>, tensor<2374x8x13440xi8>, tensor<1x151936xf32>) - check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<1x151936xf32>, tensor<1x151936xf32> - check.return -} - -check.case public @ggml_linear_q6k_q8_1_x4_row64_vocabulary_prefill_benchmark_case { - %token_count = check.literal value(17) : index - %input_size = check.literal value(2048) : index - %output_size = check.literal value(151936) : index - %q8_input = check.generate.fill value(1) : tensor<17x2304xi8> - %weight = check.generate.fill value(0) : tensor<2374x8x13440xi8> - %output = check.generate.fill value(1.0) : tensor<17x151936xf32> - %expected = check.generate.fill value(0.0) : tensor<17x151936xf32> - kernel.launch @ggml_linear_q6k_q8_1_x4_row64[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<17x2304xi8>, tensor<2374x8x13440xi8>, tensor<17x151936xf32>) - check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<17x151936xf32>, tensor<17x151936xf32> - check.return -} - // Exercises the complete activation-pack and dense Q6_K projection boundary at // the K=2048, M=512 shape used by mixed-format attention V weights. The token // buckets cover decode, awkward prefill tails, llama.cpp's 512-token @@ -315,14 +221,8 @@ check.benchmark<@ggml_linear_q6k_q8_1_x4_benchmark_case> @ggml_linear_q6k_q8_1_x check.benchmark<@ggml_linear_q6k_q8_1_x4_vocabulary_wave32_benchmark_case> @ggml_linear_q6k_q8_1_x4_vocabulary_wave32 -check.benchmark<@ggml_linear_q6k_q8_1_x4_vocabulary_prefill_benchmark_case> @ggml_linear_q6k_q8_1_x4_vocabulary_prefill_17 - check.benchmark<@ggml_linear_q6k_q8_1_x4_vocabulary_wave64_block4_benchmark_case> @ggml_linear_q6k_q8_1_x4_vocabulary_wave64_block4 -check.benchmark<@ggml_linear_q6k_q8_1_x4_row64_vocabulary_benchmark_case> @ggml_linear_q6k_q8_1_x4_row64_vocabulary_decode - -check.benchmark<@ggml_linear_q6k_q8_1_x4_row64_vocabulary_prefill_benchmark_case> @ggml_linear_q6k_q8_1_x4_row64_vocabulary_prefill_17 - check.benchmark<@ggml_linear_q6k_q8_1_x4_benchmark_case> @ggml_linear_q6k_q8_1_x4_prefill_32 {token_count = 32} check.benchmark<@ggml_linear_q6k_q8_1_x4_benchmark_case> @ggml_linear_q6k_q8_1_x4_prefill_128 {token_count = 128} @@ -429,33 +329,3 @@ kernel.def target(@ggml_q6k_row64_gfx11_wave64) export("ggml_linear_q6k_q8_1_x4_ } kernel.return } - -check.case public @ggml_linear_q6k_q8_1_x4_row64_wave64_benchmark_case { - %token_count = check.literal value(1) : index - %input_size = check.literal value(2048) : index - %output_size = check.literal value(151936) : index - %q8_input = check.generate.fill value(1) : tensor<1x2304xi8> - %weight = check.generate.fill value(0) : tensor<2374x8x13440xi8> - %output = check.generate.fill value(1.0) : tensor<1x151936xf32> - %expected = check.generate.fill value(0.0) : tensor<1x151936xf32> - kernel.launch @ggml_linear_q6k_q8_1_x4_row64_wave64[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<1x2304xi8>, tensor<2374x8x13440xi8>, tensor<1x151936xf32>) - check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<1x151936xf32>, tensor<1x151936xf32> - check.return -} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_row64_wave64_benchmark_case> @ggml_linear_q6k_q8_1_x4_row64_wave64_decode - -check.case public @ggml_linear_q6k_q8_1_x4_row64_wave64_prefill_benchmark_case { - %token_count = check.literal value(17) : index - %input_size = check.literal value(2048) : index - %output_size = check.literal value(151936) : index - %q8_input = check.generate.fill value(1) : tensor<17x2304xi8> - %weight = check.generate.fill value(0) : tensor<2374x8x13440xi8> - %output = check.generate.fill value(1.0) : tensor<17x151936xf32> - %expected = check.generate.fill value(0.0) : tensor<17x151936xf32> - kernel.launch @ggml_linear_q6k_q8_1_x4_row64_wave64[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<17x2304xi8>, tensor<2374x8x13440xi8>, tensor<17x151936xf32>) - check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<17x151936xf32>, tensor<17x151936xf32> - check.return -} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_row64_wave64_prefill_benchmark_case> @ggml_linear_q6k_q8_1_x4_row64_wave64_prefill_17 diff --git a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/README.md b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/README.md deleted file mode 100644 index eb953186424b..000000000000 --- a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# Issue 161 performance reproduction - -Compare PR196 (branch `users/astgeorg/qwen3-row64-161`) against upstream -`4a5e53048` after rebasing. The candidate includes the generic gfx11 wave64 -Q6 x Q8 row64 projection. This experiment does not include F32 accumulation -or activation-LDS reuse from the separate accuracy investigation. - -Build both sources as RelWithDebInfo with HRX -`8d78611874afe8b7aaaf0227e28e4c3bd632bc99` and the same ROCm SDK/compiler. -The reference SDK is `10.1.0a20260822-1100`. Use each machine's supported -SDK if needed and report the difference. Enable `GGML_HRX`, set -`HRX_SOURCE_DIR` and `IREE_ROCM_PATH`; build `llama-bench` and `hrx-ops-test`. -The CMake dependency build also produces loom-link and loom-benchmark. -Pin `ROCR_VISIBLE_DEVICES` to the actual gfx1100 or gfx1151 UUID; set the -SDK library directory in `LD_LIBRARY_PATH`. Record GPU, power mode, clocks, -RAM, compiler, exact source commits, model hashes and binary hashes. - -Run from the candidate checkout, supplying absolute paths: - -```sh -python3 ggml/src/ggml-hrx/tools/benchmarks/row64-161/run.py \ - --baseline-source "$BASELINE_SOURCE" --baseline-build "$BASELINE_BUILD" \ - --candidate-source "$CANDIDATE_SOURCE" --candidate-build "$CANDIDATE_BUILD" \ - --models "$MODELS" --output "$RESULTS" -``` - -The runner starts with kernel measurements, then runs the same six models -and pp66/pp128/pp512/tg32 scenarios as the original sweep. Model names are -listed in run.py; use the existing shared model store. ABBA ordering, -five samples per run, ten samples per build/scenario, batch/ubatch 512, -four CPU threads, flash attention, GPU offload and default warmup match -the original measurement. Summarize pooled median throughput changes and -individual run medians; retain all samples and failures. Small differences -are screening results, not statistical significance claims. - -Kernel fixtures capture the original generic vocabulary projection at -K=1024, N=151936, one token. Measure native/row64 on both baseline and -candidate in forward/reverse order. This adds a native candidate control; -it excludes discarded prototype variants. Four binding sets total -515,383,296 bytes, batch eight, five warmups, minimum 30 iterations. -Report device replay timing separately from normalized host completion, -alongside VGPR/LDS and instruction counts. Fixtures contain zero data and -have no numerical oracle. Run relevant nonzero hrx-ops-test coverage -separately before interpreting timings as acceptance evidence. - -Also capture model command programs with `GGML_HRX_DUMP_COMMAND_PROGRAM_DIR` -to verify selection. The generic F32 vocabulary benchmark above does not -exercise the newly generalized Q6 x Q8 wave64 projection. Compile/run its -`ggml_linear_q6k_q8_1_x4_row64_wave64_decode` and -`ggml_linear_q6k_q8_1_x4_row64_wave64_prefill_17` Loom benchmarks from -qwen_moe/ggml/linear_q6k_q8_1_x4.loom (K=2048,N=151936), comparing native -projection controls. Link the manifest's library_sources, set token_capacity -1/17 respectively and output_capacity 151936. Explicitly pass -`--measure=dispatch_complete`; the source benchmark default includes case -setup. Use batch8, warmup5, iterations30, input-ring-count4 and -profile-final-batch=true for separate device timing. Verify wave64 selection on -gfx1151 and run endpoint numerical tests. Any performance gap starts with -matched kernel-level investigation, not model-level tuning. - -Original gfx1100 model deltas are in PR196's description: decode ranged --0.58% to +0.21%; PromptBridge pp66 retained -2.06%. Generic vocabulary -kernel device times were native 143.498 us, row64 before reuse 164.550 us, -retained reuse 145.218 us. Those used HRX82214df and baseline010b2ee; -they are historical comparisons, not results for this rebase or gfx1151. - -Use an idle GPU and a shared advisory lock (runner defaults to -/tmp/lemonade-ws-tek-gpu.lock). Report instability and power/clock drift; -do not discard slow samples. Publish findings to lemonade-ws-aj0j and -lemonade-ws-s9tq, with exact commands and source identities. diff --git a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/gfx1100-rebased-results.md b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/gfx1100-rebased-results.md deleted file mode 100644 index 0390d37f3b14..000000000000 --- a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/gfx1100-rebased-results.md +++ /dev/null @@ -1,83 +0,0 @@ -# gfx1100 measurements after the rebase - -Baseline 4a5e53048 versus candidate code a27a4bef8; runner 6837ad78c. -Both built RelWithDebInfo using HRX8d78611874afe8b7aaaf0227e28e4c3bd632bc99 -and ROCm10.1.0a20260822-1100. RX7900XTX, GPU-fc2143878bea4591, -automatic power policy. Benchmark-package commits do not alter kernel code. - -All 19 selected nonzero/route tests passed, including native/row64/tied -endpoints, vocabulary embeddings and Q6 residual/short-prefill coverage. -All 24 model benchmark invocations completed. No new perplexity sweep was run. - -## Model throughput - -ABBA, five samples/run, ten samples/build/scenario; batch/ubatch512, -four CPU threads, flash attention, GPU offload, default warmup. Positive -percentages mean faster. All samples are retained, including warmup drift. - -| Model | pp66 | pp128 | pp512 | tg32 | -|---|---:|---:|---:|---:| -| promptbridge06 | -1.38% | +21.07% | -0.72% | +2.23% | -| qwen3_17 | +0.45% | -0.83% | +1.11% | -0.78% | -| qwen3_4 | +5.00% | +13.40% | -0.61% | +0.57% | -| qwen35_08 | +2.99% | +5.17% | +0.98% | -0.55% | -| qwen35_2 | -0.21% | +0.14% | +0.46% | -1.17% | -| qwen35_9 | +0.19% | +0.18% | +0.20% | +0.34% | - -These are screening measurements, not a zero-regression claim. PromptBridge -has substantial run drift: baseline pp128 run medians 18415.7 and15349.5tok/s, -candidate19808.5 and19741.2. Its decode run medians were baseline596.84/612.498 -and candidate647.286/612.436. Do not treat the pooled gains as precise estimates. -Qwen3.5 2B decode measured -1.17%, Qwen3 1.7B -0.78%, and PromptBridge pp66 --1.38%; these losses remain explicit. This campaign did not tune them. - -## Generic Q6 F32 vocabulary kernel - -K1024,N151936,T1; four binding sets (515383296bytes), batch8, -five warmups, minimum30 iterations/100ms, two runs in forward/reverse order. -All eight runs stopped as stable. Device time is a separate eight-dispatch -profile replay; host time is normalized queue completion. Synthetic zero -buffers have no numerical oracle; use the separate test results above. - -| Variant | Device us, median run mean | Host us, median run p50 | -|---|---:|---:| -| Baseline native |143.530|149.623| -| Baseline row64 |153.845|162.844| -| Candidate row64 |144.855|152.952| -| Candidate native |143.590|151.829| - -Candidate row64 is 5.84% faster than baseline row64 on device, but 0.92% -slower than baseline native. The host comparison shows additional run drift. -This generic kernel is distinct from the Q6xQ8 projection generalized to gfx11. - -Raw logs, commands, JSONL, model/binary hashes and summaries are retained in -the originating workspace under investigation/qwen3-row64-161/rebased. -The tracked runner and fixtures reproduce the procedure on another machine. -The first setup attempt could not resolve the renamed Loom executable and -performed no measurement; the fixed runner completed the campaign. - -## Q6 x Q8 projection - -Candidate source, K2048,N151936, native wave32 versus generic gfx11 row64 -wave64, two runs each in forward/reverse order. Token capacity matches -T1/T17. Explicit dispatch_complete, batch8, warmup5, iterations30, -binding ring4; all eight runs reached stable. Medians of device replay -means and host p50s: - -| Tokens | Native device us | Row64 device us | Native host us | Row64 host us | -|---|---:|---:|---:|---:| -| 1 |289.445|285.790|298.949|300.453| -| 17 |4900.678|5202.768|4886.586|5218.783| - -Decode device time improves1.26%, while host time increases0.50%. -Prefill17 device time increases6.16%, host6.80%. The two prefill device -means were4899.010/4902.345us native and5198.285/5207.250us row64. -This is a repeatable kernel-level gap in this measurement, not proof of -which source change caused it. Preserve it for gfx1151 comparison and -further kernel-first investigation; no tuning was performed here. -Earlier setup probes used case_end_to_end timing (and one used excess -decode token capacity); their logs are retained but excluded from this table. - -Fresh PromptBridge command captures each contain367 commands. Command366 -selects the generic vocabulary kernel at K1024,N151936,T1, changing weight -format6(native) to46(row64), confirming the generic microbenchmark's route. diff --git a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/native.json b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/native.json deleted file mode 100644 index 4114516e1440..000000000000 --- a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/native.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "command_count": 1, - "dispatch_count": 1, - "dispatches": [ - { - "benchmark": "@promptbridge_native_tg32_000_ggml_mul_mat_vector_f32_f32", - "compile_parameters": { - "ggml.matmul.vector.input_size": "1024", - "ggml.matmul.vector.output_accumulation": "0", - "ggml.matmul.vector.output_size": "151936", - "ggml.matmul.vector.output_unary_op": "23", - "ggml.matmul.vector.publish_format": "0", - "ggml.matmul.vector.weight_format": "6", - "ggml.workload.token_capacity": "1" - }, - "corpus_dir": "kernel-corpus/kernels/loom-libs", - "count": 1, - "integer_parameters": { - "token_count": 1 - }, - "kernel": "loom_libs:ggml_mul_mat_vector_f32_f32", - "library_sources": [ - "motifs/matmul/vector_activation.loom", - "motifs/matmul/vector_decode_pair.loom", - "motifs/matmul/vector_publish.loom", - "motifs/dequant.loom", - "motifs/elementwise/unary_apply.loom", - "motifs/q4_k_f16.loom", - "motifs/q6_k_f16.loom", - "motifs/q8_0_f16.loom", - "motifs/q8_1_f16.loom", - "motifs/f16_f16.loom", - "motifs/bf16_f16.loom", - "motifs/f32_f16.loom" - ], - "primary_sources": [ - "ops/matmul/vector_f32_f32.loom" - ], - "sources": [ - "ops/matmul/vector_f32_f32.loom", - "motifs/matmul/vector_activation.loom", - "motifs/matmul/vector_decode_pair.loom", - "motifs/matmul/vector_publish.loom", - "motifs/dequant.loom", - "motifs/elementwise/unary_apply.loom", - "motifs/q4_k_f16.loom", - "motifs/q6_k_f16.loom", - "motifs/q8_0_f16.loom", - "motifs/q8_1_f16.loom", - "motifs/f16_f16.loom", - "motifs/bf16_f16.loom", - "motifs/f32_f16.loom" - ], - "symbol": "ggml_mul_mat_vector_f32_f32", - "workload_parameters": [ - { - "name": "token_count", - "type": "index" - } - ] - } - ], - "generated_count": 1, - "kernel_counts": { - "loom_libs:ggml_mul_mat_vector_f32_f32": 1 - }, - "loom_source": "native.loom", - "model": "promptbridge_native", - "scenario": "tg32", - "schema": "ggml-hrx-model-loom-benchmarks-v2" -} diff --git a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/native.loom b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/native.loom deleted file mode 100644 index 1f0eb477718d..000000000000 --- a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/native.loom +++ /dev/null @@ -1,21 +0,0 @@ -// Generated by tools/benchmarks/generate-model-benchmarks.py for promptbridge_native scenarios: tg32. -// Regenerate from an HRX command program dump rather than editing by hand. - -target.decl @ggml_matmul_vector_gfx11_wave64 - -kernel.decl target(@ggml_matmul_vector_gfx11_wave64) @ggml_mul_mat_vector_f32_f32(%token_count: index) launch(%token_count: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) - - -// Scenario: tg32 - -check.case public @promptbridge_native_tg32_000_ggml_mul_mat_vector_f32_f32_case { - %token_count = check.literal value(1) : index - %input = check.generate.fill value(0) : tensor<4096xi8> - %weight = check.generate.fill value(0) : tensor<127626240xi8> - %output = check.generate.fill value(0) : tensor<607744xi8> - %next_output = check.generate.fill value(0) : tensor<607744xi8> - kernel.launch @ggml_mul_mat_vector_f32_f32[%token_count](%token_count, %input, %weight, %output, %next_output) : [index](index, tensor<4096xi8>, tensor<127626240xi8>, tensor<607744xi8>, tensor<607744xi8>) - check.return -} - -check.benchmark<@promptbridge_native_tg32_000_ggml_mul_mat_vector_f32_f32_case> @promptbridge_native_tg32_000_ggml_mul_mat_vector_f32_f32 diff --git a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/row64.json b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/row64.json deleted file mode 100644 index d57a2de67c46..000000000000 --- a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/row64.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "command_count": 1, - "dispatch_count": 1, - "dispatches": [ - { - "benchmark": "@promptbridge_row64_tg32_000_ggml_mul_mat_vector_f32_f32", - "compile_parameters": { - "ggml.matmul.vector.input_size": "1024", - "ggml.matmul.vector.output_accumulation": "0", - "ggml.matmul.vector.output_size": "151936", - "ggml.matmul.vector.output_unary_op": "23", - "ggml.matmul.vector.publish_format": "0", - "ggml.matmul.vector.weight_format": "46", - "ggml.workload.token_capacity": "1" - }, - "corpus_dir": "kernel-corpus/kernels/loom-libs", - "count": 1, - "integer_parameters": { - "token_count": 1 - }, - "kernel": "loom_libs:ggml_mul_mat_vector_f32_f32", - "library_sources": [ - "motifs/matmul/vector_activation.loom", - "motifs/matmul/vector_decode_pair.loom", - "motifs/matmul/vector_publish.loom", - "motifs/dequant.loom", - "motifs/elementwise/unary_apply.loom", - "motifs/q4_k_f16.loom", - "motifs/q6_k_f16.loom", - "motifs/q8_0_f16.loom", - "motifs/q8_1_f16.loom", - "motifs/f16_f16.loom", - "motifs/bf16_f16.loom", - "motifs/f32_f16.loom" - ], - "primary_sources": [ - "ops/matmul/vector_f32_f32.loom" - ], - "sources": [ - "ops/matmul/vector_f32_f32.loom", - "motifs/matmul/vector_activation.loom", - "motifs/matmul/vector_decode_pair.loom", - "motifs/matmul/vector_publish.loom", - "motifs/dequant.loom", - "motifs/elementwise/unary_apply.loom", - "motifs/q4_k_f16.loom", - "motifs/q6_k_f16.loom", - "motifs/q8_0_f16.loom", - "motifs/q8_1_f16.loom", - "motifs/f16_f16.loom", - "motifs/bf16_f16.loom", - "motifs/f32_f16.loom" - ], - "symbol": "ggml_mul_mat_vector_f32_f32", - "workload_parameters": [ - { - "name": "token_count", - "type": "index" - } - ] - } - ], - "generated_count": 1, - "kernel_counts": { - "loom_libs:ggml_mul_mat_vector_f32_f32": 1 - }, - "loom_source": "row64.loom", - "model": "promptbridge_row64", - "scenario": "tg32", - "schema": "ggml-hrx-model-loom-benchmarks-v2" -} diff --git a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/row64.loom b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/row64.loom deleted file mode 100644 index 54c4e540f065..000000000000 --- a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/row64.loom +++ /dev/null @@ -1,21 +0,0 @@ -// Generated by tools/benchmarks/generate-model-benchmarks.py for promptbridge_row64 scenarios: tg32. -// Regenerate from an HRX command program dump rather than editing by hand. - -target.decl @ggml_matmul_vector_gfx11_wave64 - -kernel.decl target(@ggml_matmul_vector_gfx11_wave64) @ggml_mul_mat_vector_f32_f32(%token_count: index) launch(%token_count: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) - - -// Scenario: tg32 - -check.case public @promptbridge_row64_tg32_000_ggml_mul_mat_vector_f32_f32_case { - %token_count = check.literal value(1) : index - %input = check.generate.fill value(0) : tensor<4096xi8> - %weight = check.generate.fill value(0) : tensor<127626240xi8> - %output = check.generate.fill value(0) : tensor<607744xi8> - %next_output = check.generate.fill value(0) : tensor<607744xi8> - kernel.launch @ggml_mul_mat_vector_f32_f32[%token_count](%token_count, %input, %weight, %output, %next_output) : [index](index, tensor<4096xi8>, tensor<127626240xi8>, tensor<607744xi8>, tensor<607744xi8>) - check.return -} - -check.benchmark<@promptbridge_row64_tg32_000_ggml_mul_mat_vector_f32_f32_case> @promptbridge_row64_tg32_000_ggml_mul_mat_vector_f32_f32 diff --git a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/run.py b/ggml/src/ggml-hrx/tools/benchmarks/row64-161/run.py deleted file mode 100644 index 0b39ff4295df..000000000000 --- a/ggml/src/ggml-hrx/tools/benchmarks/row64-161/run.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python3 -"""Repeat issue 161 performance comparisons with matched runtime builds. - -Keep raw output and command provenance; synthetic kernel timings have no oracle. -ROCR_VISIBLE_DEVICES and the ROCm runtime library path must be set by the caller. -""" -import argparse -import fcntl -import json -import os -from pathlib import Path -import subprocess -import sys -import time - -MODELS = { - 'promptbridge06': 'PromptBridge-0.6b-Alpha-GGUF/PromptBridge-0.6b-Alpha.Q4_K_M.gguf', - 'qwen3_17': 'Qwen3-1.7B-GGUF/Qwen3-1.7B-Q4_0.gguf', - 'qwen3_4': 'Qwen3-4B-Instruct-2507-GGUF/Qwen3-4B-Instruct-2507-Q4_K_M.gguf', - 'qwen35_08': 'Qwen3.5-0.8B-GGUF/Qwen3.5-0.8B-UD-Q4_K_XL.gguf', - 'qwen35_2': 'Qwen3.5-2B-GGUF/Qwen3.5-2B-UD-Q4_K_XL.gguf', - 'qwen35_9': 'Qwen3.5-9B-GGUF/Qwen3.5-9B-UD-Q4_K_XL.gguf', -} -parser = argparse.ArgumentParser(description=__doc__) -for option in ('baseline-source', 'baseline-build', 'candidate-source', 'candidate-build', 'models', 'output'): - parser.add_argument('--'+option, type=Path, required=True) -parser.add_argument('--phase', choices=('kernel', 'model', 'all'), default='all') -parser.add_argument('--lock', default='/tmp/lemonade-ws-tek-gpu.lock') -a = parser.parse_args() -if not os.environ.get('ROCR_VISIBLE_DEVICES'): - parser.error('set ROCR_VISIBLE_DEVICES to the supported GPU') -for key, value in vars(a).items(): - if isinstance(value, Path): - setattr(a, key, value.resolve()) -a.output.mkdir(parents=True, exist_ok=True) -sources = {'baseline': a.baseline_source, 'candidate': a.candidate_source} -builds = {'baseline': a.baseline_build, 'candidate': a.candidate_build} -provenance = {k: subprocess.check_output(['git', '-C', str(v), 'rev-parse', 'HEAD'], text=True).strip() for k, v in sources.items()} -(a.output/'sources.json').write_text(json.dumps(provenance, indent=2)) -fixture = Path(__file__).resolve().parent - -def run(folder, argv, env): - folder.mkdir(parents=True, exist_ok=False) - argv = list(map(str, argv)) - (folder/'command.json').write_text(json.dumps({'argv': argv, 'env': env}, indent=2)) - with open(a.lock, 'a') as lock: - fcntl.flock(lock, fcntl.LOCK_EX) - print('START', folder, flush=True) - start = time.monotonic() - with (folder/'stdout.log').open('w') as stdout, (folder/'stderr.log').open('w') as stderr: - try: - code = subprocess.run(argv, env=os.environ | env, stdout=stdout, stderr=stderr, timeout=1800).returncode - except subprocess.TimeoutExpired: - code = 124 - (folder/'exit.json').write_text(json.dumps({'code': code, 'seconds': time.monotonic()-start})) - print('DONE', folder.name, code, flush=True) - if code: - raise SystemExit(f'Failed: {folder}; retain logs before retrying in a new output directory') - -if a.phase in ('kernel', 'all'): - # Four controls identify layout cost separately from the retained reuse optimization. - order = [('baseline','native'), ('baseline','row64'), ('candidate','row64'), ('candidate','native')] - for i, (stage, layout) in enumerate(order + order[::-1]): - hrx = sources[stage]/'ggml/src/ggml-hrx' - d = json.loads((fixture/(layout+'.json')).read_text()) - d['loom_source'] = str(fixture/(layout+'.loom')) - manifest = a.output/f'{stage}-{layout}.json' - manifest.write_text(json.dumps(d, indent=2)) - argv = [sys.executable, hrx/'tools/benchmarks/run-model-benchmarks.py', '--model', d['model'], '--scenario', 'tg32', '--manifest', manifest, '--build-dir', builds[stage], '--output-dir', a.output/f'kernel-{i}-{stage}-{layout}', '--batch-size', '8', '--iterations', '30', '--warmup-iterations', '5', '--input-ring-count', '4', '--profile-final-batch', 'true'] - tools = builds[stage]/'ggml/src/ggml-hrx/hrx/src/ggml-hrx-deps-build/loom/src/loom/tools' - run(a.output/f'kernel-command-{i}', argv, { - 'ROCR_VISIBLE_DEVICES': os.environ['ROCR_VISIBLE_DEVICES'], - 'LOOM_LINK': str(tools/'loom-link/loom-link'), - 'IREE_BENCHMARK_LOOM': str(tools/'loom-benchmark/loom-benchmark'), - }) - -if a.phase in ('model', 'all'): - for name, model in MODELS.items(): - for i, stage in enumerate(('baseline','candidate','candidate','baseline')): - env = {'ROCR_VISIBLE_DEVICES': os.environ['ROCR_VISIBLE_DEVICES'], 'LD_LIBRARY_PATH': str(builds[stage]/'bin')+':'+os.environ.get('LD_LIBRARY_PATH','')} - argv = [builds[stage]/'bin/llama-bench', '-m', a.models/model, '--device', 'HRX0', '-ngl', '99', '-t', '4', '-fa', 'on', '-p', '66,128,512', '-n', '32', '-b', '512', '-ub', '512', '-r', '5', '-o', 'jsonl'] - run(a.output/name/f'{i}-{stage}', argv, env) diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index 7704d2848189..c15e43146e16 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -7467,7 +7467,7 @@ static void run_dense_matmul_zero_weight_case(ggml_type weight_type, } static void run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(bool prefer_row64 = false, - bool tied_embedding = false) { + bool tied_embedding = false, int64_t token_count = 1) { ggml_backend_t cpu_backend = init_cpu_backend(); ggml_backend_t hrx_backend = ggml_backend_hrx_init(0); REQUIRE(hrx_backend != nullptr); @@ -7481,7 +7481,6 @@ static void run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(bool prefer_row64 = f REQUIRE(cpu_ctx != nullptr); REQUIRE(hrx_ctx != nullptr); - constexpr int64_t token_count = 1; ggml_tensor * cpu_input = ggml_new_tensor_2d(cpu_ctx, GGML_TYPE_F32, kQwenHiddenSize, token_count); ggml_tensor * cpu_norm_w = ggml_new_tensor_1d(cpu_ctx, GGML_TYPE_F32, kQwenHiddenSize); ggml_tensor * cpu_weight = ggml_new_tensor_2d(cpu_ctx, GGML_TYPE_Q6_K, kQwenHiddenSize, kQwenVocabularyCount); @@ -10137,6 +10136,12 @@ static void register_dense_matmul_cases(Suite & suite) { [] { run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(true); }); suite.device_case("endpoint_rmsnorm_q6k_q8.row64.tied_embedding", [] { run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(true, true); }); + suite.device_case("endpoint_rmsnorm_q6k_q8.cpu_reference.tokens17", + [] { run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(false, false, 17); }); + suite.device_case("endpoint_rmsnorm_q6k_q8.row64.cpu_reference.tokens17", + [] { run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(true, false, 17); }); + suite.device_case("endpoint_rmsnorm_q6k_q8.row64.tied_embedding.tokens17", + [] { run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(true, true, 17); }); suite.device_case("dense_matmul.q6_k.skinny.tokens5.outputs4096.input16384", [] { run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_skinny_input_f32_publish_f32", 5, 4096, 16384); From caf562616c3ad4400029199f4e7e15391481d0fb Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Mon, 16 Nov 2026 16:57:10 +0000 Subject: [PATCH 12/18] hrx: reuse shared row64 matvec for single-token Qwen endpoints --- .../qwen/dispatch-qwen-matmul.cpp | 69 ++++++++------ .../qwen/dispatch-qwen-matmul.h | 4 + .../qwen/dispatch-qwen-rmsnorm.cpp | 33 ++----- .../kernels/loom-libs/motifs/dequant.loom | 15 ++-- .../qwen_moe/ggml/linear_q6k_q8_1_x4.loom | 89 ------------------- .../kernels/qwen_moe/manifest.json | 80 +---------------- tests/hrx-ops-test.cpp | 16 ++-- 7 files changed, 71 insertions(+), 235 deletions(-) diff --git a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp index 3c3dabaaf5da..3be0dd5ecb95 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp @@ -18,8 +18,8 @@ static constexpr KernelCatalogRef kQwenDenseLinearQ4KQ8NextQ8Kernel = GGML_HRX_KERNEL_REF("qwen3_moe", "qwen3_moe_dense_linear_q4k_q8_1_x4_next_q8"); static constexpr KernelCatalogRef kGgmlLinearQ6KQ8_1X4Kernel = GGML_HRX_KERNEL_REF("qwen3_moe", "ggml_linear_q6k_q8_1_x4"); -static constexpr KernelCatalogRef kGgmlLinearQ6KQ8_1X4Row64Kernel = - GGML_HRX_KERNEL_REF("qwen3_moe", "ggml_linear_q6k_q8_1_x4_row64"); +static constexpr KernelCatalogRef kMulMatVectorF32F32Kernel = + GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_vector_f32_f32"); static constexpr int64_t kQwenHiddenSize = kQwen30BMoeDispatchProfile.hidden_size; static constexpr int64_t kQwenVocabularyCount = 151936; @@ -338,33 +338,48 @@ bool qwen_accepts_q6k_q8_input(const DispatchMatchContext & context, output->ne[2] == 1 && output->ne[3] == 1; } -static void build_qwen_matmul_dispatch(const QwenMatmulMatch & match, - DispatchMatch & dispatch_match, - size_t root_index, - bool row64_weight_policy) { +Dispatch make_qwen_q6k_q8_projection_dispatch(const Value & weight, const Value & output, + const DispatchBinding & input, bool row64_weight_policy) { + const int64_t input_size = weight.ne[0]; + const int64_t output_size = weight.ne[1]; + const bool row64_weight = row64_weight_policy && weight.alias_source.value < 0 && + dispatch_weight_layout_eligible(weight); Dispatch dispatch; - const bool row64_weight = row64_weight_policy && match.weight->alias_source.value < 0 && - dispatch_weight_layout_eligible(*match.weight); - dispatch.kernel = make_kernel_specialization(row64_weight ? kGgmlLinearQ6KQ8_1X4Row64Kernel : match.kernel); - dispatch.kernel.integer_parameters.emplace("token_count", match.token_count); - dispatch.kernel.integer_parameters.emplace("input_size", match.input_size); - dispatch.kernel.integer_parameters.emplace("output_size", match.output_size); - dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.token_capacity", - to_config_value(match.token_count)); - dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.output_capacity", - to_config_value(match.output_size)); - dispatch.bindings.push_back({ match.input_value, 0, match.input_bytes }); + dispatch.kernel = + make_kernel_specialization(row64_weight ? kMulMatVectorF32F32Kernel : kGgmlLinearQ6KQ8_1X4Kernel); + dispatch.kernel.integer_parameters.emplace("token_count", 1); if (row64_weight) { - dispatch.bindings.push_back({ match.weight->id, 0, match.weight->byte_count, - kQ6KPackedK256Row64ScaleRowLayout, match.weight->type, match.input_size, - match.output_size, match.weight->byte_count }); + dispatch.kernel.compile_parameters.emplace("ggml.workload.token_capacity", "1"); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.input_size", to_config_value(input_size)); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.output_size", to_config_value(output_size)); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.output_accumulation", "0"); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.output_unary_op", + std::to_string(unary_kind_config_value(UnaryKind::Identity))); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.publish_format", "0"); + dispatch.kernel.compile_parameters.emplace( + "ggml.matmul.vector.weight_format", + to_config_value(common_mul_mat_format_config_value(CommonMulMatWeightFormat::Q6KRow64))); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.activation_format", + std::to_string(GGML_TYPE_Q8_1)); } else { - dispatch.bindings.push_back({ match.weight->id, 0, match.weight->byte_count }); + dispatch.kernel.integer_parameters.emplace("input_size", input_size); + dispatch.kernel.integer_parameters.emplace("output_size", output_size); + dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.token_capacity", "1"); + dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.output_capacity", + to_config_value(output_size)); } - dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count }); - - dispatch_match.covered_nodes.push_back(root_index); - dispatch_match.dispatches.push_back(std::move(dispatch)); + dispatch.bindings.push_back(input); + if (row64_weight) { + dispatch.bindings.push_back({ weight.id, 0, weight.byte_count, kQ6KPackedK256Row64ScaleRowLayout, + weight.type, input_size, output_size, weight.byte_count }); + } else { + dispatch.bindings.push_back({ weight.id, 0, weight.byte_count }); + } + dispatch.bindings.push_back({ output.id, 0, output.byte_count }); + if (row64_weight) { + dispatch.bindings.push_back({ output.id, 0, output.byte_count }); + } + return dispatch; } static bool match_qwen_q6k_q8_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) { @@ -372,7 +387,9 @@ static bool match_qwen_q6k_q8_dispatch(const DispatchMatchContext & context, Dis if (!match.matched()) { return false; } - build_qwen_matmul_dispatch(match, dispatch_match, context.root_index, context.row64_weight_policy); + dispatch_match.covered_nodes.push_back(context.root_index); + dispatch_match.dispatches.push_back(make_qwen_q6k_q8_projection_dispatch( + *match.weight, *match.output, { match.input_value, 0, match.input_bytes }, context.row64_weight_policy)); return true; } diff --git a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.h b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.h index 28f348e9cacb..333279285a24 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.h +++ b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.h @@ -8,6 +8,10 @@ bool qwen_accepts_q6k_q8_input(const DispatchMatchContext & context, const GraphNode & consumer, const Value & input); +// The endpoint Q8 projection is a single-token path; prefill uses common matmul dispatch. +Dispatch make_qwen_q6k_q8_projection_dispatch(const Value & weight, const Value & output, + const DispatchBinding & input, bool row64_weight_policy); + void register_qwen_matmul_dispatches(DispatchRegistryBuilder & registry); } // namespace ggml::hrx diff --git a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-rmsnorm.cpp b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-rmsnorm.cpp index 7e01675f2661..dd3cacb97cff 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-rmsnorm.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-rmsnorm.cpp @@ -1,4 +1,5 @@ #include "dispatch-qwen-rmsnorm.h" +#include "dispatch-qwen-matmul.h" #include "dispatch-llm-profiles.h" #include "../common/dispatch-activation-publication.h" @@ -17,10 +18,6 @@ namespace { static constexpr KernelCatalogRef kQwenRmsNormF32QuantizeQ8_1X4Kernel = GGML_HRX_KERNEL_REF("qwen3_moe", "qwen3_moe_rmsnorm_f32_quantize_q8_1_x4"); -static constexpr KernelCatalogRef kGgmlLinearQ6KQ8_1X4Kernel = - GGML_HRX_KERNEL_REF("qwen3_moe", "ggml_linear_q6k_q8_1_x4"); -static constexpr KernelCatalogRef kGgmlLinearQ6KQ8_1X4Row64Kernel = - GGML_HRX_KERNEL_REF("qwen3_moe", "ggml_linear_q6k_q8_1_x4_row64"); static constexpr float kQwenRmsNormEpsilon = kQwen30BMoeDispatchProfile.rms_norm_epsilon; static constexpr int64_t kQwenHiddenSize = kQwen30BMoeDispatchProfile.hidden_size; static constexpr int64_t kQwenVocabularyCount = 151936; @@ -189,7 +186,7 @@ static QwenEndpointProjectionMatch match_qwen_endpoint_projection(const Graph & } if (input->id != match.output->id || weight->type != GGML_TYPE_Q6_K || output->type != GGML_TYPE_F32 || !weight->contiguous || !input->contiguous || !output->contiguous || match.hidden_size != kQwenHiddenSize || - !is_supported_token_count(match.token_count) || weight->ne[0] != match.hidden_size || + match.token_count != 1 || weight->ne[0] != match.hidden_size || weight->ne[1] != kQwenVocabularyCount || output->ne[0] != kQwenVocabularyCount || output->ne[1] != match.token_count || output->ne[2] != 1 || output->ne[3] != 1) { return {}; @@ -286,29 +283,9 @@ static bool match_qwen_rmsnorm_f32_quantize_q8_1_x4_dispatch(const DispatchMatch rms_dispatch.bindings.push_back({ rms_match.output->id, 0, rms_match.output->byte_count }); rms_dispatch.bindings.push_back({ q8_value, 0, q8_byte_count }); - Dispatch projection_dispatch; - const bool row64_weight = context.row64_weight_policy && projection_match.weight->alias_source.value < 0 && - dispatch_weight_layout_eligible(*projection_match.weight); - projection_dispatch.kernel = make_kernel_specialization( - row64_weight ? kGgmlLinearQ6KQ8_1X4Row64Kernel : kGgmlLinearQ6KQ8_1X4Kernel); - projection_dispatch.kernel.integer_parameters.emplace("token_count", rms_match.token_count); - projection_dispatch.kernel.integer_parameters.emplace("input_size", rms_match.hidden_size); - projection_dispatch.kernel.integer_parameters.emplace("output_size", kQwenVocabularyCount); - projection_dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.token_capacity", - to_config_value(rms_match.token_count)); - projection_dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.output_capacity", - to_config_value(kQwenVocabularyCount)); - projection_dispatch.bindings.push_back({ q8_value, 0, q8_byte_count }); - if (row64_weight) { - projection_dispatch.bindings.push_back({ projection_match.weight->id, 0, - projection_match.weight->byte_count, - kQ6KPackedK256Row64ScaleRowLayout, projection_match.weight->type, - rms_match.hidden_size, kQwenVocabularyCount, - projection_match.weight->byte_count }); - } else { - projection_dispatch.bindings.push_back({ projection_match.weight->id, 0, projection_match.weight->byte_count }); - } - projection_dispatch.bindings.push_back({ projection_match.output->id, 0, projection_match.output->byte_count }); + Dispatch projection_dispatch = make_qwen_q6k_q8_projection_dispatch( + *projection_match.weight, *projection_match.output, { q8_value, 0, q8_byte_count }, + context.row64_weight_policy); match.covered_nodes.push_back(rms_match.rms_node_index); match.covered_nodes.push_back(rms_match.mul_node_index); diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/dequant.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/dequant.loom index 15b9e70d9ee2..3914d5f07574 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/dequant.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/dequant.loom @@ -2710,7 +2710,7 @@ func.def inline @ggml_q6k_f32_vector4(%weight: buffer, %weight_row_byte_base: of func.return %values : vector<4xf32> } -func.def inline @ggml_q6k_row64_tile_lane_i8_scale(%weight: buffer, %tile: index, %row_lane: index, %q6_group: index, %packet: index) -> (vector<4xi8>, f32) { +func.def inline @ggml_q6k_row64_tile_lane_f32_vector4(%weight: buffer, %tile: index, %row_lane: index, %q6_group: index, %packet: index) -> (vector<4xf32>) { %c2 = index.constant 2 : index %c4 = index.constant 4 : index %tile_bytes = index.constant 13440 : offset @@ -2719,7 +2719,7 @@ func.def inline @ggml_q6k_row64_tile_lane_i8_scale(%weight: buffer, %tile: index %c4_i32v = vector.splati 4 : vector<1xi32> %nibble_mask = vector.splati 252645135 : vector<1xi32> %high_mask = vector.splati 50529027 : vector<1xi32> - %c32_i8v = vector.splati 32 : vector<4xi8> + %c32_f32v = vector.splatf 32.0 : vector<4xf32> %bounded_group = index.assume %q6_group [range(%q6_group, 0, 7)] : index %bounded_packet = index.assume %packet [range(%packet, 0, 7)] : index %tile_base = index.scale %tile, %tile_bytes : index, offset -> offset @@ -2748,20 +2748,15 @@ func.def inline @ggml_q6k_row64_tile_lane_i8_scale(%weight: buffer, %tile: index %qh = vector.shli %qh_low, %c4_i32v : vector<1xi32> %code = vector.ori %ql, %qh : vector<1xi32> %code_i8 = vector.bitcast %code : vector<1xi32> to vector<4xi8> - %centered = vector.subi %code_i8, %c32_i8v : vector<4xi8> + %code_f32 = vector.uitofp %code_i8 : vector<4xi8> to vector<4xf32> + %centered = vector.subf %code_f32, %c32_f32v : vector<4xf32> %scale_i8 = view.load %scale_view[%row_lane, %scale_packet_half, %bounded_group] : view<64x2x8xi8> -> i8 %d_f16 = view.load %d_view[%row_lane] : view<64xf16> -> f16 %scale = scalar.sitofp %scale_i8 : i8 to f32 %d = scalar.extf %d_f16 : f16 to f32 %combined_scale = scalar.mulf %scale, %d : f32 - func.return %centered, %combined_scale : vector<4xi8>, f32 -} - -func.def inline @ggml_q6k_row64_tile_lane_f32_vector4(%weight: buffer, %tile: index, %row_lane: index, %q6_group: index, %packet: index) -> (vector<4xf32>) { - %centered, %combined_scale = func.call @ggml_q6k_row64_tile_lane_i8_scale(%weight, %tile, %row_lane, %q6_group, %packet) : (buffer, index, index, index, index) -> (vector<4xi8>, f32) - %centered_f32 = vector.sitofp %centered : vector<4xi8> to vector<4xf32> %combined_scale_vector = vector.splat %combined_scale : vector<4xf32> - %values = vector.mulf %centered_f32, %combined_scale_vector : vector<4xf32> + %values = vector.mulf %centered, %combined_scale_vector : vector<4xf32> func.return %values : vector<4xf32> } diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom index 9a030722b9f2..55da51a6161a 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom @@ -240,92 +240,3 @@ check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6 check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6k_q8_1_x4_dense_v_prefill_1024 {token_count = 1024} check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6k_q8_1_x4_dense_v_prefill_2048 {token_count = 2048} - -amdgpu.target @ggml_q6k_row64_gfx11_wave64 {subgroup_size = 64} - -func.decl @ggml_q6k_row64_tile_lane_i8_scale(%weight: buffer, %tile: index, %row_lane: index, %q6_group: index, %packet: index) -> (vector<4xi8>, f32) - -func.def inline @ggml_q6k_q8_1_x4_row64_row_lane_wave64(%input_size: index, %row: index, %weight: buffer, %q8_input: buffer, %q8_row_byte_base: offset, %lane: index) -> (f32) { - %bounded_input_size = index.assume %input_size [range(%input_size, 256, 32768), multiple_of(%input_size, 256)] : index - %bounded_lane = index.assume %lane [range(%lane, 0, 63)] : index - %c0 = index.constant 0 : index - %c1 = index.constant 1 : index - %c8 = index.constant 8 : index - %c64 = index.constant 64 : index - %c256 = index.constant 256 : index - %zero = scalar.constant 0.0 : f32 - %zero_i32v = vector.splati 0 : vector<1xi32> - %block_count = index.div %bounded_input_size, %c256 : index - %row_group = index.div %row, %c64 : index - %row_lane = index.rem %row, %c64 : index - %group = index.div %bounded_lane, %c8 : index - %packet = index.rem %bounded_lane, %c8 : index - %sum = scf.for %block = [%c0 to %block_count step %c1](%block_acc = %zero : f32) -> (f32) { - %tile = index.madd %row_group, %block_count, %block : index - %block_q8_base = index.mul %block, %c8 : index - %weight_values, %weight_scale = func.call @ggml_q6k_row64_tile_lane_i8_scale(%weight, %tile, %row_lane, %group, %packet) : (buffer, index, index, index, index) -> (vector<4xi8>, f32) - %q8_block = index.add %block_q8_base, %group : index - %q8_values, %q8_d = func.call @ggml_q8_1_x4_word(%q8_input, %q8_row_byte_base, %q8_block, %packet) : (buffer, offset, index, index) -> (vector<4xi8>, f32) - %dot_i32v = vector.dot4i %weight_values, %q8_values, %zero_i32v : vector<4xi8>, vector<4xi8>, vector<1xi32> - %dot_i32 = vector.extract %dot_i32v[0] : vector<1xi32> -> i32 - %dot = scalar.sitofp %dot_i32 : i32 to f32 - %scaled_weight = scalar.mulf %dot, %weight_scale : f32 - %scaled = scalar.mulf %scaled_weight, %q8_d : f32 - %next = scalar.addf %block_acc, %scaled : f32 - scf.yield %next : f32 - } - func.return %sum : f32 -} - -kernel.def target(@ggml_q6k_row64_gfx11_wave64) export("ggml_linear_q6k_q8_1_x4_row64") @ggml_linear_q6k_q8_1_x4_row64_wave64(%token_count: index, %input_size: index, %output_size: index) { - %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index - %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index - %c3 = index.constant 3 : index - %c4 = index.constant 4 : index - %c1 = index.constant 1 : index - %workgroup_size = index.constant 256 : index - %padded_output_size = index.add %output_capacity, %c3 : index - %output_tiles = index.div %padded_output_size, %c4 : index - kernel.launch.config workgroups(%output_tiles, %token_capacity, %c1) workgroup_size(%workgroup_size, %c1, %c1) : index -} launch(%token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) { - %q8_noalias, %weight_noalias, %output_noalias = buffer.assume.noalias %q8_input, %weight, %output : buffer, buffer, buffer - %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index - %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index - %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048), le(%token_count, %token_capacity)] : index - %bounded_input_size = index.assume %input_size [range(%input_size, 256, 32768), multiple_of(%input_size, 256)] : index - %bounded_output_size = index.assume %output_size [range(%output_size, 1, 262144), le(%output_size, %output_capacity), multiple_of(%output_size, 64)] : index - %c0 = index.constant 0 : index - %c4 = index.constant 4 : index - %c128 = index.constant 128 : index - %c144_bytes = index.constant 144 : offset - %token = kernel.workgroup.id : index - %channel_tile = kernel.workgroup.id : index - %subgroup = kernel.subgroup.id : index - %lane = kernel.subgroup.lane.id : index - %channel = index.madd %channel_tile, %c4, %subgroup : index - %valid_token = index.cmp ult, %token, %bounded_token_count : index - %valid_channel = index.cmp ult, %channel, %bounded_output_size : index - %publish = scalar.andi %valid_token, %valid_channel : i1 - %safe_token = scf.select %valid_token, %token, %c0 : index - %safe_channel = scf.select %valid_channel, %channel, %c0 : index - %q8_groups = index.div %bounded_input_size, %c128 : index - %q8_row_bytes = index.scale %q8_groups, %c144_bytes : index, offset -> offset - %q8_row_byte_base = index.scale %safe_token, %q8_row_bytes : index, offset -> offset - %lane_acc = scf.if %publish -> (f32) { - %value = func.call @ggml_q6k_q8_1_x4_row64_row_lane_wave64(%bounded_input_size, %safe_channel, %weight_noalias, %q8_noalias, %q8_row_byte_base, %lane) : (index, index, buffer, buffer, offset, index) -> (f32) - scf.yield %value : f32 - } else { - %zero = scalar.constant 0.0 : f32 - scf.yield %zero : f32 - } - %dot = kernel.subgroup.reduce %lane_acc : f32 - %is_lane_zero = index.cmp eq, %lane, %c0 : index - scf.if %publish { - scf.if %is_lane_zero { - %c0_offset = index.constant 0 : offset - %output_view = buffer.view %output_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]x[%bounded_output_size]xf32> - view.store %dot, %output_view[%token, %channel] : f32, view<[%bounded_token_count]x[%bounded_output_size]xf32> - } - } - kernel.return -} diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json index eb59e8392a31..32fee80b9ee1 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json @@ -2,9 +2,6 @@ "schema": "ggml-hrx-kernel-corpus-v1", "upstream_revision": "local", "files": [ - { - "path": "../loom-libs/motifs/dequant.loom" - }, { "path": "../loom-libs/motifs/q6_k_f16.loom" }, @@ -178,81 +175,14 @@ "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", "../loom-libs/motifs/q6_k_f16.loom", "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom", - "../loom-libs/motifs/dequant.loom" - ] - }, - "compile_dependencies": [ - "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", - "../loom-libs/motifs/q6_k_f16.loom", - "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom", - "../loom-libs/motifs/dequant.loom" - ] - }, - { - "name": "ggml_linear_q6k_q8_1_x4_row64", - "symbol": "ggml_linear_q6k_q8_1_x4_row64_wave64", - "source": "ggml/linear_q6k_q8_1_x4.loom", - "workload_parameters": [ - { - "name": "token_count", - "type": "index" - }, - { - "name": "input_size", - "type": "index" - }, - { - "name": "output_size", - "type": "index" - } - ], - "launch_parameters": [ - { - "name": "token_count", - "type": "index" - }, - { - "name": "input_size", - "type": "index" - }, - { - "name": "output_size", - "type": "index" - } - ], - "bindings": [ - "q8_input", - "weight", - "output" - ], - "binding_access": [ - "read", - "read", - "read_write" - ], - "target_selector": "", - "compile_recipe": { - "mode": "archive", - "link_module": "linear_q6k_q8_1_x4_linked", - "primary_sources": [ - "ggml/linear_q6k_q8_1_x4.loom" - ], - "library_sources": [ - "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", - "../loom-libs/motifs/q6_k_f16.loom", - "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom", - "../loom-libs/motifs/dequant.loom" + "../loom-libs/motifs/quantize_q8_1_x4_group.loom" ] }, "compile_dependencies": [ "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", "../loom-libs/motifs/q6_k_f16.loom", "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom", - "../loom-libs/motifs/dequant.loom" + "../loom-libs/motifs/quantize_q8_1_x4_group.loom" ] }, { @@ -308,16 +238,14 @@ "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", "../loom-libs/motifs/q6_k_f16.loom", "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom", - "../loom-libs/motifs/dequant.loom" + "../loom-libs/motifs/quantize_q8_1_x4_group.loom" ] }, "compile_dependencies": [ "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", "../loom-libs/motifs/q6_k_f16.loom", "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom", - "../loom-libs/motifs/dequant.loom" + "../loom-libs/motifs/quantize_q8_1_x4_group.loom" ] }, { diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index c15e43146e16..9b5008884bcd 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -7519,10 +7519,16 @@ static void run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(bool prefer_row64 = f { "gfx1151", true }, nullptr, tied_embedding ? 2 : 1); } - require_kernel_subsequence( - scheduled_kernel_sequence(hrx_graph, { "gfx1151", prefer_row64 }), - { "qwen3_moe:qwen3_moe_rmsnorm_f32_quantize_q8_1_x4", - prefer_row64 ? "qwen3_moe:ggml_linear_q6k_q8_1_x4_row64" : "qwen3_moe:ggml_linear_q6k_q8_1_x4" }); + if (token_count == 1) { + require_kernel_subsequence( + scheduled_kernel_sequence(hrx_graph, { "gfx1151", prefer_row64 }), + { "qwen3_moe:qwen3_moe_rmsnorm_f32_quantize_q8_1_x4", + prefer_row64 ? "loom_libs:ggml_mul_mat_vector_f32_f32" : "qwen3_moe:ggml_linear_q6k_q8_1_x4" }); + } else { + require_kernel_subsequence( + scheduled_kernel_sequence(hrx_graph, { "gfx1151", prefer_row64 }), + { "loom_libs:ggml_mul_mat_tiled_input_f32_publish_f32" }); + } if (tied_embedding) { require_kernel_subsequence(scheduled_kernel_sequence(hrx_graph, { "gfx1151", prefer_row64 }), { "loom_libs:ggml_get_rows_f32" }); @@ -10140,8 +10146,6 @@ static void register_dense_matmul_cases(Suite & suite) { [] { run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(false, false, 17); }); suite.device_case("endpoint_rmsnorm_q6k_q8.row64.cpu_reference.tokens17", [] { run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(true, false, 17); }); - suite.device_case("endpoint_rmsnorm_q6k_q8.row64.tied_embedding.tokens17", - [] { run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(true, true, 17); }); suite.device_case("dense_matmul.q6_k.skinny.tokens5.outputs4096.input16384", [] { run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_skinny_input_f32_publish_f32", 5, 4096, 16384); From f74241bdbb644868c93ef4d2f53d83ce9b196d9d Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Mon, 16 Nov 2026 17:24:16 +0000 Subject: [PATCH 13/18] hrx: consolidate native and row64 Q6 Q8 decode motif (lemonade-ws-s9tq) --- .../kernels/loom-libs/manifest.json | 20 + .../motifs/matmul/q6_q8_1_x4_decode.loom | 451 ++++++++++++++++++ .../motifs/matmul/vector_activation.loom | 366 +------------- .../qwen_moe/ggml/linear_q6k_q8_1_x4.loom | 72 +-- .../kernels/qwen_moe/manifest.json | 106 +++- 5 files changed, 589 insertions(+), 426 deletions(-) diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json index 15538f76d2d1..67732866c0c8 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json @@ -2737,6 +2737,8 @@ ], "library_sources": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/dequant.loom", @@ -2752,6 +2754,8 @@ }, "compile_dependencies": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/dequant.loom", @@ -2806,6 +2810,8 @@ ], "library_sources": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/matmul/vector_postops.loom", @@ -2821,6 +2827,8 @@ }, "compile_dependencies": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/matmul/vector_postops.loom", @@ -3072,6 +3080,8 @@ ], "library_sources": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/llm_attention_qkv_matmul_postops.loom", @@ -3088,6 +3098,8 @@ }, "compile_dependencies": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/llm_attention_qkv_matmul_postops.loom", @@ -3145,6 +3157,8 @@ ], "library_sources": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/llm_attention_qkv_matmul_postops.loom", @@ -3161,6 +3175,8 @@ }, "compile_dependencies": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/llm_attention_qkv_matmul_postops.loom", @@ -3212,6 +3228,8 @@ ], "library_sources": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/llm_attention_qkv_matmul_postops.loom", @@ -3227,6 +3245,8 @@ }, "compile_dependencies": [ "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", "motifs/llm_attention_qkv_matmul_postops.loom", diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom index c906e25b77bc..8ba290409d14 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom @@ -1,6 +1,16 @@ // Copyright 2026 The HRX Authors // SPDX-License-Identifier: Apache-2.0 +// Q6 x Q8 schedule selection. Layout and native wave mode are compile-time inputs. +template.decl @ggml.matmul.q6_q8_1_x4.core(%weight_format: index, %native_wave64: i1, %token_capacity: index, %output_capacity: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) + +config.decl @ggml.matmul.vector.activation_format : %value: index where [range(%value, 0, 9)] +config.decl @ggml.matmul.vector.publish_format : %value: index where [range(%value, 0, 1)] +config.decl @ggml.workload.token_capacity : %value: index where [range(%value, 1, 1)] +func.decl @ggml_matmul_vector_q8_1_x4_stage_slab(%input: buffer, %stage: buffer, %capacity: index, %slab_begin: index, %blocks: index, %global_row_bytes: offset) +func.decl pure @ggml_matmul_vector_q8_geometry(%token_capacity: index, %input_size: index, %output_size: index, %weight_format: index) -> (index, index) +func.decl @ggml_matmul_vector_q8_publish(%tokens: index, %n: index, %rows_per_group: index, %row_base: index, %local_row: index, %block_lane: index, %channel_group: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %publish_format: index, %total_g0: f32, %total_g1: f32, %total_g2: f32, %total_g3: f32, %total_g4: f32, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) + template.decl @ggml.linear_q6k_q8_1_x4.body(%publish_output: i1, %token_count: index, %token0: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer) -> (f32, index, index, i1, index, index) func.decl @ggml_q8_1_x4_word(%q8_input: buffer, %row_byte_base: offset, %q8_block: index, %word_in_block: index) -> (vector<4xi8>, f32) @@ -202,3 +212,444 @@ template.def<@ggml.linear_q6k_q8_1_x4.body> device @ggml_linear_q6k_q8_1_x4_body %dot = kernel.subgroup.reduce %lane_acc : f32 template.return %dot, %token, %channel, %valid_channel, %launch_token_count, %bounded_output_size : f32, index, index, i1, index, index } + +func.def inline @ggml_matmul_vector_q6_prefetch_native_words(%weight_na: buffer, %row_group_block: index, %block: index, %row_lane: index, %half16: index, %group: index, %half128: index, %ql_side: index, %packet: index) -> (f16, i8, vector<4xi32>, vector<4xi32>) { + %c2 = index.constant 2 : index + %record_bytes = index.constant 13440 : offset + %scale_add = index.constant 128 : offset + %raw_add = index.constant 1152 : offset + %group_block = index.add %row_group_block, %block : index + %block_base = index.scale %group_block, %record_bytes : index, offset -> offset + %scale_base = index.add %block_base, %scale_add : offset + %raw_base = index.add %block_base, %raw_add : offset + %d_view = buffer.view %weight_na[%block_base] : buffer -> view<64xf16> + %s_view = buffer.view %weight_na[%scale_base] : buffer -> view<64x2x8xi8> + %raw_view = buffer.view %weight_na[%raw_base] : buffer -> view<2x3x64x8xi32> + %dh = view.load %d_view[%row_lane] : view<64xf16> -> f16 + %si = view.load %s_view[%row_lane, %half16, %group] : view<64x2x8xi8> -> i8 + %ql = vector.load %raw_view[%half128, %ql_side, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<4xi32> + %qh = vector.load %raw_view[%half128, %c2, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<4xi32> + func.return %dh, %si, %ql, %qh : f16, i8, vector<4xi32>, vector<4xi32> +} + +func.def inline @ggml_matmul_vector_q6_q8_activation_values(%pipelined: i1, %token: index, %q8_row_bytes: offset, %q8_block_add: offset, %capacity: index, %current_slot: index, %q8_block0: index, %q8_input: buffer, %q8_word0: index, %q8_meta: index) -> (vector<16xi8>, f32) { + %code_offset = index.constant 16 : offset + %a, %ad = scf.if %pipelined -> (vector<16xi8>, f32) { + %base = index.constant 0 : offset + %c4 = index.constant 4 : index + %q8_block = index.assume %q8_block0 [range(%q8_block0, 0, 7)] : index + %word = index.add %q8_word0, %c4 : index + %q_view = buffer.view %q8_input[%base] : buffer -> view<2x[%capacity]x8x36xi32> + %ds_view = buffer.view %q8_input[%base] : buffer -> view<2x[%capacity]x8x72xf16> + %words = vector.load %q_view[%current_slot, %token, %q8_block, %word] : view<2x[%capacity]x8x36xi32> -> vector<4xi32> + %a = vector.bitcast %words : vector<4xi32> to vector<16xi8> + %scale = view.load %ds_view[%current_slot, %token, %q8_block, %q8_meta] : view<2x[%capacity]x8x72xf16> -> f16 + %ad = scalar.extf %scale : f16 to f32 + scf.yield %a, %ad : vector<16xi8>, f32 + } else { + %tb = index.scale %token, %q8_row_bytes : index, offset -> offset + %qb = index.add %tb, %q8_block_add : offset + %qc = index.add %qb, %code_offset : offset + %q_view = buffer.view %q8_input[%qc] : buffer -> view<32xi32> + %ds_view = buffer.view %q8_input[%qb] : buffer -> view<8xf16> + %words = vector.load %q_view[%q8_word0] : view<32xi32> -> vector<4xi32> + %a = vector.bitcast %words : vector<4xi32> to vector<16xi8> + %scale = view.load %ds_view[%q8_meta] : view<8xf16> -> f16 + %ad = scalar.extf %scale : f16 to f32 + scf.yield %a, %ad : vector<16xi8>, f32 + } + func.return %a, %ad : vector<16xi8>, f32 +} + +func.def inline @ggml_matmul_vector_q6_q8_dot16(%pipelined: i1, %w: vector<16xi8>, %a: vector<16xi8>) -> (i32) { + %sum = scf.if %pipelined -> (i32) { + %di0 = vector.splati 0 : vector<1xi32> + %dw0 = vector.slice %w[0] : vector<16xi8> -> vector<4xi8> + %da0 = vector.slice %a[0] : vector<16xi8> -> vector<4xi8> + %di1 = vector.dot4i %dw0, %da0, %di0 : vector<4xi8>, vector<4xi8>, vector<1xi32> + %dw1 = vector.slice %w[4] : vector<16xi8> -> vector<4xi8> + %da1 = vector.slice %a[4] : vector<16xi8> -> vector<4xi8> + %di2 = vector.dot4i %dw1, %da1, %di1 : vector<4xi8>, vector<4xi8>, vector<1xi32> + %dw2 = vector.slice %w[8] : vector<16xi8> -> vector<4xi8> + %da2 = vector.slice %a[8] : vector<16xi8> -> vector<4xi8> + %di3 = vector.dot4i %dw2, %da2, %di2 : vector<4xi8>, vector<4xi8>, vector<1xi32> + %dw3 = vector.slice %w[12] : vector<16xi8> -> vector<4xi8> + %da3 = vector.slice %a[12] : vector<16xi8> -> vector<4xi8> + %di4 = vector.dot4i %dw3, %da3, %di3 : vector<4xi8>, vector<4xi8>, vector<1xi32> + %chain_sum = vector.extract %di4[0] : vector<1xi32> -> i32 + scf.yield %chain_sum : i32 + } else { + %init = vector.splati 0 : vector<4xi32> + %dot = vector.dot4i %w, %a, %init : vector<16xi8>, vector<16xi8>, vector<4xi32> + %seed = scalar.constant 0 : i32 + %direct_sum = vector.reduce %dot, %seed : vector<4xi32>, i32 + scf.yield %direct_sum : i32 + } + func.return %sum : i32 +} + +func.def inline @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input: buffer, %capacity0: index, %slot_begin: index, %blocks: index, %global_row_bytes: offset) -> (vector<4xi32>) { + %capacity = index.assume %capacity0 [range(%capacity0, 1, 5)] : index + %c0 = index.constant 0 : index + %c4 = index.constant 4 : index + %c18 = index.constant 18 : index + %c72 = index.constant 72 : index + %packet_bytes = index.constant 16 : offset + %block_bytes = index.constant 288 : offset + %thread = kernel.workitem.id : index + %total_packets = index.mul %capacity, %c72 : index + %copy_packet = index.rem %thread, %total_packets : index + %copy_token = index.div %copy_packet, %c72 : index + %copy_inner = index.rem %copy_packet, %c72 : index + %remaining = index.sub %blocks, %slot_begin : index + %full = index.cmp uge, %remaining, %c4 : index + %active0 = scf.select %full, %c4, %remaining : index + %active = index.assume %active0 [range(%active0, 1, 4)] : index + %active_packets = index.mul %active, %c18 : index + %copy_valid = index.cmp ult, %copy_inner, %active_packets : index + %safe_inner = scf.select %copy_valid, %copy_inner, %c0 : index + %copy_row = index.scale %copy_token, %global_row_bytes : index, offset -> offset + %slot_bytes = index.scale %slot_begin, %block_bytes : index, offset -> offset + %copy_bytes = index.scale %safe_inner, %packet_bytes : index, offset -> offset + %copy_base0 = index.add %copy_row, %slot_bytes : offset + %copy_base = index.add %copy_base0, %copy_bytes : offset + %source = buffer.view %input[%copy_base] : buffer -> view<4xi32> + %values = vector.load %source[%c0] : view<4xi32> -> vector<4xi32> + func.return %values : vector<4xi32> +} +func.def inline @ggml_matmul_vector_q8_1_x4_publish_half_slab(%stage: buffer, %capacity0: index, %slot0: index, %values: vector<4xi32>) { + %capacity = index.assume %capacity0 [range(%capacity0, 1, 5)] : index + %slot = index.assume %slot0 [range(%slot0, 0, 1)] : index + %c4 = index.constant 4 : index + %c72 = index.constant 72 : index + %base = index.constant 0 : offset + %thread = kernel.workitem.id : index + %total_packets = index.mul %capacity, %c72 : index + %thread_valid = index.cmp ult, %thread, %total_packets : index + scf.if %thread_valid { + %copy_token0 = index.div %thread, %c72 : index + %copy_token = index.assume %copy_token0 [range(%copy_token0, 0, 4), lt(%copy_token0, %capacity)] : index + %copy_inner = index.rem %thread, %c72 : index + %origin = index.mul %copy_inner, %c4 : index + %destination = buffer.view %stage[%base] : buffer -> view<2x[%capacity]x288xi32> + vector.store %values, %destination[%slot, %copy_token, %origin] : vector<4xi32>, view<2x[%capacity]x288xi32> + } + func.return +} + +func.def inline @ggml_matmul_vector_q6_q8_accumulate(%pipelined: i1, %tokens: index, %q8_row_bytes: offset, %q8_block_add: offset, %capacity: index, %current_slot: index, %q8_block0: index, %q8_input: buffer, %q8_word0: index, %q8_meta: index, %w: vector<16xi8>, %ws: f32, %acc_g0: f32, %acc_g1: f32, %acc_g2: f32, %acc_g3: f32, %acc_g4: f32) -> (f32, f32, f32, f32, f32) { + %code_offset = index.constant 16 : offset + %t0 = index.constant 0 : index + %valid0 = index.cmp ult, %t0, %tokens : index + %computed_g0 = scf.if %valid0 -> (f32) { + %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t0, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) + %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) + %sumf = scalar.sitofp %sum : i32 to f32 + %scale = scalar.mulf %ws, %ad : f32 + %updated = scalar.fmaf %sumf, %scale, %acc_g0 : f32 + scf.yield %updated : f32 + } else { + scf.yield %acc_g0 : f32 + } + %t1 = index.constant 1 : index + %valid1 = index.cmp ult, %t1, %tokens : index + %computed_g1 = scf.if %valid1 -> (f32) { + %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t1, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) + %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) + %sumf = scalar.sitofp %sum : i32 to f32 + %scale = scalar.mulf %ws, %ad : f32 + %updated = scalar.fmaf %sumf, %scale, %acc_g1 : f32 + scf.yield %updated : f32 + } else { + scf.yield %acc_g1 : f32 + } + %t2 = index.constant 2 : index + %valid2 = index.cmp ult, %t2, %tokens : index + %computed_g2 = scf.if %valid2 -> (f32) { + %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t2, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) + %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) + %sumf = scalar.sitofp %sum : i32 to f32 + %scale = scalar.mulf %ws, %ad : f32 + %updated = scalar.fmaf %sumf, %scale, %acc_g2 : f32 + scf.yield %updated : f32 + } else { + scf.yield %acc_g2 : f32 + } + %t3 = index.constant 3 : index + %valid3 = index.cmp ult, %t3, %tokens : index + %computed_g3 = scf.if %valid3 -> (f32) { + %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t3, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) + %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) + %sumf = scalar.sitofp %sum : i32 to f32 + %scale = scalar.mulf %ws, %ad : f32 + %updated = scalar.fmaf %sumf, %scale, %acc_g3 : f32 + scf.yield %updated : f32 + } else { + scf.yield %acc_g3 : f32 + } + %t4 = index.constant 4 : index + %valid4 = index.cmp ult, %t4, %tokens : index + %computed_g4 = scf.if %valid4 -> (f32) { + %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t4, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) + %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) + %sumf = scalar.sitofp %sum : i32 to f32 + %scale = scalar.mulf %ws, %ad : f32 + %updated = scalar.fmaf %sumf, %scale, %acc_g4 : f32 + scf.yield %updated : f32 + } else { + scf.yield %acc_g4 : f32 + } + func.return %computed_g0, %computed_g1, %computed_g2, %computed_g3, %computed_g4 : f32, f32, f32, f32, f32 +} + +func.def inline @ggml_matmul_vector_q6_q8_1_x4_activation_dot(%capacity: index, %weight_format: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) { + %staged_rows, %k_partitions = func.call pure @ggml_matmul_vector_q8_geometry(%capacity, %input_size0, %output_size0, %weight_format) : (index, index, index, index) -> (index, index) + %no_staged_rows = index.constant 0 : index + %staged = index.cmp ugt, %staged_rows, %no_staged_rows : index + %tokens = index.assume %token_count [range(%token_count, 1, 1), le(%token_count, %capacity)] : index + %k = index.assume %input_size0 [range(%input_size0, 256, 32768), multiple_of(%input_size0, 256)] : index + %n = index.assume %output_size0 [range(%output_size0, 64, 262144), multiple_of(%output_size0, 64)] : index + %c0 = index.constant 0 : index + %c1 = index.constant 1 : index + %c2 = index.constant 2 : index + %c3 = index.constant 3 : index + %c4 = index.constant 4 : index + %c7 = index.constant 7 : index + %c8 = index.constant 8 : index + %c16 = index.constant 16 : index + %c32 = index.constant 32 : index + %c64 = index.constant 64 : index + %c128 = index.constant 128 : index + %c256 = index.constant 256 : index + %base = index.constant 0 : offset + %zero = scalar.constant 0.0 : f32 + %lane0 = kernel.subgroup.lane.id : index + %lane = index.assume %lane0 [range(%lane0, 0, 63)] : index + %wave = kernel.subgroup.id : index + %rows_per_group = scf.select %staged, %c32, %c8 : index + %row_base = index.mul %channel_group, %rows_per_group : index + %wave_row = index.mul %wave, %c4 : index + %cohort = index.div %lane, %c16 : index + %local_row = index.add %wave_row, %cohort : index + %row = index.add %row_base, %local_row : index + %block_lane = index.rem %lane, %c16 : index + %group = index.div %block_lane, %c2 : index + %half16 = index.rem %block_lane, %c2 : index + %half128 = index.div %group, %c4 : index + %group_in_half = index.rem %group, %c4 : index + %ql_side = index.rem %group_in_half, %c2 : index + %nibble = index.div %group_in_half, %c2 : index + %nibble_shift0 = index.mul %nibble, %c4 : index + %nibble_shift1 = index.cast %nibble_shift0 : index to i32 + %nibble_shift = vector.splat %nibble_shift1 : vector<4xi32> + %high_shift0 = index.mul %group_in_half, %c2 : index + %high_shift1 = index.cast %high_shift0 : index to i32 + %high_shift = vector.splat %high_shift1 : vector<4xi32> + %packet = index.mul %half16, %c4 : index + %q8_group_words = index.mul %group_in_half, %c8 : index + %q8_word0 = index.add %q8_group_words, %packet : index + %q8_meta = index.mul %group_in_half, %c2 : index + %blocks = index.div %k, %c256 : index + %q8_groups = index.div %k, %c128 : index + %q8_block_bytes = index.constant 144 : offset + %global_q8_row_bytes = index.scale %q8_groups, %q8_block_bytes : index, offset -> offset + %native_block_bytes = index.constant 210 : index + %cache_bytes = index.constant 33554432 : index + %weight_blocks = index.mul %blocks, %n : index + %weight_bytes = index.mul %weight_blocks, %native_block_bytes : index + %exceeds_cache = index.cmp uge, %weight_bytes, %cache_bytes : index + %pipelined = scalar.andi %staged, %exceeds_cache : i1 + %full_slab_row_bytes = index.constant 2304 : offset + %half_slab_row_bytes = index.constant 1152 : offset + %slab_row_bytes = scf.select %pipelined, %half_slab_row_bytes, %full_slab_row_bytes : offset + %q8_row_bytes = scf.select %staged, %slab_row_bytes, %global_q8_row_bytes : offset + %code_offset = index.constant 16 : offset + %record_bytes = index.constant 13440 : offset + %scale_add = index.constant 128 : offset + %raw_add = index.constant 1152 : offset + %staged_row_group = index.div %channel_group, %c2 : index + %direct_row_group = index.div %row, %c64 : index + %row_group = scf.select %staged, %staged_row_group, %direct_row_group : index + %row_lane = index.rem %row, %c64 : index + %row_group_block = index.mul %row_group, %blocks : index + %input_na, %weight_na = buffer.assume.noalias %input, %weight : buffer, buffer + %slab_blocks = scf.select %pipelined, %c4, %c8 : index + %full_stage_bytes = index.constant 11520 : offset + %q8_stage_bytes = scf.select %staged, %full_stage_bytes, %base : offset + %q8_stage = buffer.alloca align(16) %q8_stage_bytes : buffer + %q8_input = scf.select %staged, %q8_stage, %input_na : buffer + %zero_half = scalar.constant 0.0 : f16 + %zero_code = scalar.constant 0 : i8 + %zero_words = vector.splati 0 : vector<4xi32> + %prime_d, %prime_s, %prime_ql, %prime_qh = scf.if %staged -> (f16, i8, vector<4xi32>, vector<4xi32>) { + %pd, %ps, %pql, %pqh = func.call @ggml_matmul_vector_q6_prefetch_native_words(%weight_na, %row_group_block, %c0, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) + scf.yield %pd, %ps, %pql, %pqh : f16, i8, vector<4xi32>, vector<4xi32> + } else { + scf.yield %zero_half, %zero_code, %zero_words, %zero_words : f16, i8, vector<4xi32>, vector<4xi32> + } + scf.if %pipelined { + %prime_q8 = func.call @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input_na, %capacity, %c0, %blocks, %global_q8_row_bytes) : (buffer, index, index, index, offset) -> (vector<4xi32>) + func.call @ggml_matmul_vector_q8_1_x4_publish_half_slab(%q8_stage, %capacity, %c0, %prime_q8) : (buffer, index, index, vector<4xi32>) + kernel.barrier scope(workgroup) ordering(acq_rel) + } + %total_g0, %total_g1, %total_g2, %total_g3, %total_g4, %last_d, %last_s, %last_ql, %last_qh = scf.for %block = [%c0 to %blocks step %c1](%acc_g0 = %zero : f32, %acc_g1 = %zero : f32, %acc_g2 = %zero : f32, %acc_g3 = %zero : f32, %acc_g4 = %zero : f32, %carry_d = %prime_d : f16, %carry_s = %prime_s : i8, %carry_ql = %prime_ql : vector<4xi32>, %carry_qh = %prime_qh : vector<4xi32>) -> (f32, f32, f32, f32, f32, f16, i8, vector<4xi32>, vector<4xi32>) { + %block_in_slab = index.rem %block, %slab_blocks : index + %slab_boundary = index.cmp eq, %block_in_slab, %c0 : index + %direct_stage = scalar.andi %staged, %slab_boundary : i1 + %true = scalar.constant true : i1 + %not_pipelined = scalar.xori %pipelined, %true : i1 + %stage_current = scalar.andi %direct_stage, %not_pipelined : i1 + scf.if %stage_current { + func.call @ggml_matmul_vector_q8_1_x4_stage_slab(%input_na, %q8_stage, %capacity, %block, %blocks, %global_q8_row_bytes) : (buffer, buffer, index, index, index, offset) + } + %slab_number = index.div %block, %slab_blocks : index + %current_slot = index.rem %slab_number, %c2 : index + %following_slab = index.add %slab_number, %c1 : index + %following_begin = index.mul %following_slab, %slab_blocks : index + %following_exists = index.cmp ult, %following_begin, %blocks : index + %next_slot = index.rem %following_slab, %c2 : index + %staged_future = scalar.andi %pipelined, %following_exists : i1 + %current_d, %current_s, %current_ql, %current_qh = scf.if %staged -> (f16, i8, vector<4xi32>, vector<4xi32>) { + scf.yield %carry_d, %carry_s, %carry_ql, %carry_qh : f16, i8, vector<4xi32>, vector<4xi32> + } else { + %wd, %ws8, %wql, %wqh = func.call @ggml_matmul_vector_q6_prefetch_native_words(%weight_na, %row_group_block, %block, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) + scf.yield %wd, %ws8, %wql, %wqh : f16, i8, vector<4xi32>, vector<4xi32> + } + %d = scalar.extf %current_d : f16 to f32 + %sf = scalar.sitofp %current_s : i8 to f32 + %ws = scalar.mulf %d, %sf : f32 + %nibble_mask = vector.splati 252645135 : vector<4xi32> + %high_mask = vector.splati 50529027 : vector<4xi32> + %four = vector.splati 4 : vector<4xi32> + %ql_shifted = vector.shrui %current_ql, %nibble_shift : vector<4xi32> + %low = vector.andi %ql_shifted, %nibble_mask : vector<4xi32> + %qh_shifted = vector.shrui %current_qh, %high_shift : vector<4xi32> + %high0 = vector.andi %qh_shifted, %high_mask : vector<4xi32> + %high = vector.shli %high0, %four : vector<4xi32> + %code = vector.ori %low, %high : vector<4xi32> + // Guard bits prevent subtraction from borrowing across packed bytes. + %guard_mask = vector.splati -2139062144 : vector<4xi32> + %center = vector.splati 538976288 : vector<4xi32> + %guarded = vector.ori %code, %guard_mask : vector<4xi32> + %centered = vector.subi %guarded, %center : vector<4xi32> + %signed_code = vector.xori %centered, %guard_mask : vector<4xi32> + %w = vector.bitcast %signed_code : vector<4xi32> to vector<16xi8> + %next_block = index.add %block, %c1 : index + %next_exists = index.cmp ult, %next_block, %blocks : index + %has_next = scalar.andi %staged, %next_exists : i1 + %next_d, %next_s, %next_ql, %next_qh = scf.if %has_next -> (f16, i8, vector<4xi32>, vector<4xi32>) { + %wd, %ws8, %wql, %wqh = func.call @ggml_matmul_vector_q6_prefetch_native_words(%weight_na, %row_group_block, %next_block, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) + scf.yield %wd, %ws8, %wql, %wqh : f16, i8, vector<4xi32>, vector<4xi32> + } else { + scf.yield %carry_d, %carry_s, %carry_ql, %carry_qh : f16, i8, vector<4xi32>, vector<4xi32> + } + %input_block = scf.select %staged, %block_in_slab, %block : index + %q8_block0 = index.mul %input_block, %c2 : index + %q8_block = index.add %q8_block0, %half128 : index + %q8_local_block_add = index.scale %q8_block, %q8_block_bytes : index, offset -> offset + %slot_row = index.mul %current_slot, %capacity : index + %slot_byte_add0 = index.scale %slot_row, %slab_row_bytes : index, offset -> offset + %slot_byte_add = scf.select %pipelined, %slot_byte_add0, %base : offset + %q8_block_add = index.add %slot_byte_add, %q8_local_block_add : offset + %slot_complete = index.cmp eq, %block_in_slab, %c3 : index + %publish_next = scalar.andi %staged_future, %slot_complete : i1 + %next_g0, %next_g1, %next_g2, %next_g3, %next_g4 = scf.if %publish_next -> (f32, f32, f32, f32, f32) { + %next_q8 = func.call @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input_na, %capacity, %following_begin, %blocks, %global_q8_row_bytes) : (buffer, index, index, index, offset) -> (vector<4xi32>) + %v0, %v1, %v2, %v3, %v4 = func.call @ggml_matmul_vector_q6_q8_accumulate(%pipelined, %tokens, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block, %q8_input, %q8_word0, %q8_meta, %w, %ws, %acc_g0, %acc_g1, %acc_g2, %acc_g3, %acc_g4) : (i1, index, offset, offset, index, index, index, buffer, index, index, vector<16xi8>, f32, f32, f32, f32, f32, f32) -> (f32, f32, f32, f32, f32) + func.call @ggml_matmul_vector_q8_1_x4_publish_half_slab(%q8_stage, %capacity, %next_slot, %next_q8) : (buffer, index, index, vector<4xi32>) + kernel.barrier scope(workgroup) ordering(acq_rel) + scf.yield %v0, %v1, %v2, %v3, %v4 : f32, f32, f32, f32, f32 + } else { + %v0, %v1, %v2, %v3, %v4 = func.call @ggml_matmul_vector_q6_q8_accumulate(%pipelined, %tokens, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block, %q8_input, %q8_word0, %q8_meta, %w, %ws, %acc_g0, %acc_g1, %acc_g2, %acc_g3, %acc_g4) : (i1, index, offset, offset, index, index, index, buffer, index, index, vector<16xi8>, f32, f32, f32, f32, f32, f32) -> (f32, f32, f32, f32, f32) + scf.yield %v0, %v1, %v2, %v3, %v4 : f32, f32, f32, f32, f32 + } + scf.yield %next_g0, %next_g1, %next_g2, %next_g3, %next_g4, %next_d, %next_s, %next_ql, %next_qh : f32, f32, f32, f32, f32, f16, i8, vector<4xi32>, vector<4xi32> + } + %publish_format = config.get @ggml.matmul.vector.publish_format : index + func.call @ggml_matmul_vector_q8_publish(%tokens, %n, %rows_per_group, %row_base, %local_row, %block_lane, %channel_group, %output_accumulation, %output_unary_op, %epsilon, %publish_format, %total_g0, %total_g1, %total_g2, %total_g3, %total_g4, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, index, index, index, index, index, index, index, index, f32, index, f32, f32, f32, f32, f32, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) + func.return +} + + +func.def inline @ggml_q6_q8_native_wave32(%token_capacity: index, %output_capacity: index, %token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) { + %q8_noalias, %weight_noalias, %output_noalias = buffer.assume.noalias %q8_input, %weight, %output : buffer, buffer, buffer + %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048), le(%token_count, %token_capacity)] : index + %bounded_output_size = index.assume %output_size [range(%output_size, 1, 262144), le(%output_size, %output_capacity)] : index + %c0 = index.constant 0 : index + %token0 = kernel.workgroup.id : index + %valid_token = index.cmp ult, %token0, %bounded_token_count : index + %safe_token = scf.select %valid_token, %token0, %c0 : index + %dot, %token, %channel, %valid_channel, %launch_token_count, %output_bound = template.apply<@ggml.linear_q6k_q8_1_x4.body>(%valid_token, %bounded_token_count, %safe_token, %input_size, %bounded_output_size, %q8_noalias, %weight_noalias) : (i1, index, index, index, index, buffer, buffer) -> (f32, index, index, i1, index, index) + %c0_i32 = scalar.constant 0 : i32 + %c0_offset = index.constant 0 : offset + %lane = kernel.subgroup.lane.id : index + %lane_i32 = index.cast %lane : index to i32 + %is_lane_zero = scalar.cmpi eq, %lane_i32, %c0_i32 : i32 + scf.if %valid_token { + scf.if %valid_channel { + scf.if %is_lane_zero { + %output_view = buffer.view %output_noalias[%c0_offset] : buffer -> view<[%launch_token_count]x[%output_bound]xf32> + view.store %dot, %output_view[%token, %channel] : f32, view<[%launch_token_count]x[%output_bound]xf32> + } + } + } + func.return +} + +func.def inline @ggml_q6_q8_native_wave64_block4(%token_capacity: index, %output_capacity: index, %token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) { + %q8_noalias, %weight_noalias, %output_noalias = buffer.assume.noalias %q8_input, %weight, %output : buffer, buffer, buffer + %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048), le(%token_count, %token_capacity)] : index + %bounded_input_size = index.assume %input_size [range(%input_size, 256, 32768), multiple_of(%input_size, 256)] : index + %bounded_output_size = index.assume %output_size [range(%output_size, 1, 262144), le(%output_size, %output_capacity)] : index + %token = kernel.workgroup.id : index + %channel = kernel.workgroup.id : index + %lane = kernel.subgroup.lane.id : index + %valid_token = index.cmp ult, %token, %bounded_token_count : index + %valid_channel = index.cmp ult, %channel, %bounded_output_size : index + %publish_output = scalar.andi %valid_token, %valid_channel : i1 + %c0 = index.constant 0 : index + %c128 = index.constant 128 : index + %c144_bytes = index.constant 144 : offset + %c210_bytes = index.constant 210 : offset + %c256 = index.constant 256 : index + %safe_token = scf.select %valid_token, %token, %c0 : index + %safe_channel = scf.select %valid_channel, %channel, %c0 : index + %q6_block_count = index.div %bounded_input_size, %c256 : index + %weight_row_bytes = index.scale %q6_block_count, %c210_bytes : index, offset -> offset + %weight_row_byte_base = index.scale %safe_channel, %weight_row_bytes : index, offset -> offset + %q8_group_count = index.div %bounded_input_size, %c128 : index + %q8_row_bytes = index.scale %q8_group_count, %c144_bytes : index, offset -> offset + %q8_row_byte_base = index.scale %safe_token, %q8_row_bytes : index, offset -> offset + %lane_acc = scf.if %publish_output -> (f32) { + %value = func.call @ggml_q6k_q8_1_x4_row_lane_wave64_block4(%bounded_input_size, %weight_noalias, %weight_row_byte_base, %q8_noalias, %q8_row_byte_base, %lane) : (index, buffer, offset, buffer, offset, index) -> (f32) + scf.yield %value : f32 + } else { + %c0_f32 = scalar.constant 0.0 : f32 + scf.yield %c0_f32 : f32 + } + %dot = kernel.subgroup.reduce %lane_acc : f32 + %is_lane_zero = index.cmp eq, %lane, %c0 : index + scf.if %publish_output { + scf.if %is_lane_zero { + %c0_offset = index.constant 0 : offset + %output_view = buffer.view %output_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]x[%bounded_output_size]xf32> + view.store %dot, %output_view[%token, %channel] : f32, view<[%bounded_token_count]x[%bounded_output_size]xf32> + } + } + func.return +} + +template.def<@ggml.matmul.q6_q8_1_x4.core> device @ggml_q6_q8_shared_core(%weight_format: index, %native_wave64: i1, %token_capacity: index, %output_capacity: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) { + %row64_format = index.constant 46 : index + %row64 = index.cmp eq, %weight_format, %row64_format : index + scf.if %row64 { + func.call @ggml_matmul_vector_q6_q8_1_x4_activation_dot(%token_capacity, %weight_format, %token_count, %input_size0, %output_size0, %output_accumulation, %output_unary_op, %epsilon, %channel_group, %token_group, %input, %weight, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) + } else { + scf.if %native_wave64 { + func.call @ggml_q6_q8_native_wave64_block4(%token_capacity, %output_capacity, %token_count, %input_size0, %output_size0, %input, %weight, %output) : (index, index, index, index, index, buffer, buffer, buffer) + } else { + func.call @ggml_q6_q8_native_wave32(%token_capacity, %output_capacity, %token_count, %input_size0, %output_size0, %input, %weight, %output) : (index, index, index, index, index, buffer, buffer, buffer) + } + } + template.return +} diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_activation.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_activation.loom index 30dacc26c240..122c776521b4 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_activation.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_activation.loom @@ -1,3 +1,5 @@ +template.decl @ggml.matmul.q6_q8_1_x4.core(%weight_format: index, %native_wave64: i1, %token_capacity: index, %output_capacity: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) + func.decl @ggml_q4k_scale_min_from_header(%scale0: i32, %scale1: i32, %scale2: i32, %q4_group: index) -> (i32, i32) func.decl @ggml_dot_u8_s8_vector8_f32(%weight: vector<2xi32>, %activation: vector<2xi32>) -> (f32) @@ -1086,366 +1088,6 @@ func.def inline @ggml_matmul_vector_q4_q8_1_x4_activation_dot(%weight_format: in } // Native Q6_K carrier with signed I8 dot products and F32 scaling. -func.def inline @ggml_matmul_vector_q6_prefetch_native_words(%weight_na: buffer, %row_group_block: index, %block: index, %row_lane: index, %half16: index, %group: index, %half128: index, %ql_side: index, %packet: index) -> (f16, i8, vector<4xi32>, vector<4xi32>) { - %c2 = index.constant 2 : index - %record_bytes = index.constant 13440 : offset - %scale_add = index.constant 128 : offset - %raw_add = index.constant 1152 : offset - %group_block = index.add %row_group_block, %block : index - %block_base = index.scale %group_block, %record_bytes : index, offset -> offset - %scale_base = index.add %block_base, %scale_add : offset - %raw_base = index.add %block_base, %raw_add : offset - %d_view = buffer.view %weight_na[%block_base] : buffer -> view<64xf16> - %s_view = buffer.view %weight_na[%scale_base] : buffer -> view<64x2x8xi8> - %raw_view = buffer.view %weight_na[%raw_base] : buffer -> view<2x3x64x8xi32> - %dh = view.load %d_view[%row_lane] : view<64xf16> -> f16 - %si = view.load %s_view[%row_lane, %half16, %group] : view<64x2x8xi8> -> i8 - %ql = vector.load %raw_view[%half128, %ql_side, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<4xi32> - %qh = vector.load %raw_view[%half128, %c2, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<4xi32> - func.return %dh, %si, %ql, %qh : f16, i8, vector<4xi32>, vector<4xi32> -} - -func.def inline @ggml_matmul_vector_q6_q8_activation_values(%pipelined: i1, %token: index, %q8_row_bytes: offset, %q8_block_add: offset, %capacity: index, %current_slot: index, %q8_block0: index, %q8_input: buffer, %q8_word0: index, %q8_meta: index) -> (vector<16xi8>, f32) { - %code_offset = index.constant 16 : offset - %a, %ad = scf.if %pipelined -> (vector<16xi8>, f32) { - %base = index.constant 0 : offset - %c4 = index.constant 4 : index - %q8_block = index.assume %q8_block0 [range(%q8_block0, 0, 7)] : index - %word = index.add %q8_word0, %c4 : index - %q_view = buffer.view %q8_input[%base] : buffer -> view<2x[%capacity]x8x36xi32> - %ds_view = buffer.view %q8_input[%base] : buffer -> view<2x[%capacity]x8x72xf16> - %words = vector.load %q_view[%current_slot, %token, %q8_block, %word] : view<2x[%capacity]x8x36xi32> -> vector<4xi32> - %a = vector.bitcast %words : vector<4xi32> to vector<16xi8> - %scale = view.load %ds_view[%current_slot, %token, %q8_block, %q8_meta] : view<2x[%capacity]x8x72xf16> -> f16 - %ad = scalar.extf %scale : f16 to f32 - scf.yield %a, %ad : vector<16xi8>, f32 - } else { - %tb = index.scale %token, %q8_row_bytes : index, offset -> offset - %qb = index.add %tb, %q8_block_add : offset - %qc = index.add %qb, %code_offset : offset - %q_view = buffer.view %q8_input[%qc] : buffer -> view<32xi32> - %ds_view = buffer.view %q8_input[%qb] : buffer -> view<8xf16> - %words = vector.load %q_view[%q8_word0] : view<32xi32> -> vector<4xi32> - %a = vector.bitcast %words : vector<4xi32> to vector<16xi8> - %scale = view.load %ds_view[%q8_meta] : view<8xf16> -> f16 - %ad = scalar.extf %scale : f16 to f32 - scf.yield %a, %ad : vector<16xi8>, f32 - } - func.return %a, %ad : vector<16xi8>, f32 -} - -func.def inline @ggml_matmul_vector_q6_q8_dot16(%pipelined: i1, %w: vector<16xi8>, %a: vector<16xi8>) -> (i32) { - %sum = scf.if %pipelined -> (i32) { - %di0 = vector.splati 0 : vector<1xi32> - %dw0 = vector.slice %w[0] : vector<16xi8> -> vector<4xi8> - %da0 = vector.slice %a[0] : vector<16xi8> -> vector<4xi8> - %di1 = vector.dot4i %dw0, %da0, %di0 : vector<4xi8>, vector<4xi8>, vector<1xi32> - %dw1 = vector.slice %w[4] : vector<16xi8> -> vector<4xi8> - %da1 = vector.slice %a[4] : vector<16xi8> -> vector<4xi8> - %di2 = vector.dot4i %dw1, %da1, %di1 : vector<4xi8>, vector<4xi8>, vector<1xi32> - %dw2 = vector.slice %w[8] : vector<16xi8> -> vector<4xi8> - %da2 = vector.slice %a[8] : vector<16xi8> -> vector<4xi8> - %di3 = vector.dot4i %dw2, %da2, %di2 : vector<4xi8>, vector<4xi8>, vector<1xi32> - %dw3 = vector.slice %w[12] : vector<16xi8> -> vector<4xi8> - %da3 = vector.slice %a[12] : vector<16xi8> -> vector<4xi8> - %di4 = vector.dot4i %dw3, %da3, %di3 : vector<4xi8>, vector<4xi8>, vector<1xi32> - %chain_sum = vector.extract %di4[0] : vector<1xi32> -> i32 - scf.yield %chain_sum : i32 - } else { - %init = vector.splati 0 : vector<4xi32> - %dot = vector.dot4i %w, %a, %init : vector<16xi8>, vector<16xi8>, vector<4xi32> - %seed = scalar.constant 0 : i32 - %direct_sum = vector.reduce %dot, %seed : vector<4xi32>, i32 - scf.yield %direct_sum : i32 - } - func.return %sum : i32 -} - -func.def inline @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input: buffer, %capacity0: index, %slot_begin: index, %blocks: index, %global_row_bytes: offset) -> (vector<4xi32>) { - %capacity = index.assume %capacity0 [range(%capacity0, 1, 5)] : index - %c0 = index.constant 0 : index - %c4 = index.constant 4 : index - %c18 = index.constant 18 : index - %c72 = index.constant 72 : index - %packet_bytes = index.constant 16 : offset - %block_bytes = index.constant 288 : offset - %thread = kernel.workitem.id : index - %total_packets = index.mul %capacity, %c72 : index - %copy_packet = index.rem %thread, %total_packets : index - %copy_token = index.div %copy_packet, %c72 : index - %copy_inner = index.rem %copy_packet, %c72 : index - %remaining = index.sub %blocks, %slot_begin : index - %full = index.cmp uge, %remaining, %c4 : index - %active0 = scf.select %full, %c4, %remaining : index - %active = index.assume %active0 [range(%active0, 1, 4)] : index - %active_packets = index.mul %active, %c18 : index - %copy_valid = index.cmp ult, %copy_inner, %active_packets : index - %safe_inner = scf.select %copy_valid, %copy_inner, %c0 : index - %copy_row = index.scale %copy_token, %global_row_bytes : index, offset -> offset - %slot_bytes = index.scale %slot_begin, %block_bytes : index, offset -> offset - %copy_bytes = index.scale %safe_inner, %packet_bytes : index, offset -> offset - %copy_base0 = index.add %copy_row, %slot_bytes : offset - %copy_base = index.add %copy_base0, %copy_bytes : offset - %source = buffer.view %input[%copy_base] : buffer -> view<4xi32> - %values = vector.load %source[%c0] : view<4xi32> -> vector<4xi32> - func.return %values : vector<4xi32> -} -func.def inline @ggml_matmul_vector_q8_1_x4_publish_half_slab(%stage: buffer, %capacity0: index, %slot0: index, %values: vector<4xi32>) { - %capacity = index.assume %capacity0 [range(%capacity0, 1, 5)] : index - %slot = index.assume %slot0 [range(%slot0, 0, 1)] : index - %c4 = index.constant 4 : index - %c72 = index.constant 72 : index - %base = index.constant 0 : offset - %thread = kernel.workitem.id : index - %total_packets = index.mul %capacity, %c72 : index - %thread_valid = index.cmp ult, %thread, %total_packets : index - scf.if %thread_valid { - %copy_token0 = index.div %thread, %c72 : index - %copy_token = index.assume %copy_token0 [range(%copy_token0, 0, 4), lt(%copy_token0, %capacity)] : index - %copy_inner = index.rem %thread, %c72 : index - %origin = index.mul %copy_inner, %c4 : index - %destination = buffer.view %stage[%base] : buffer -> view<2x[%capacity]x288xi32> - vector.store %values, %destination[%slot, %copy_token, %origin] : vector<4xi32>, view<2x[%capacity]x288xi32> - } - func.return -} - - -func.def inline @ggml_matmul_vector_q6_q8_accumulate(%pipelined: i1, %tokens: index, %q8_row_bytes: offset, %q8_block_add: offset, %capacity: index, %current_slot: index, %q8_block0: index, %q8_input: buffer, %q8_word0: index, %q8_meta: index, %w: vector<16xi8>, %ws: f32, %acc_g0: f32, %acc_g1: f32, %acc_g2: f32, %acc_g3: f32, %acc_g4: f32) -> (f32, f32, f32, f32, f32) { - %code_offset = index.constant 16 : offset - %t0 = index.constant 0 : index - %valid0 = index.cmp ult, %t0, %tokens : index - %computed_g0 = scf.if %valid0 -> (f32) { - %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t0, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) - %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) - %sumf = scalar.sitofp %sum : i32 to f32 - %scale = scalar.mulf %ws, %ad : f32 - %updated = scalar.fmaf %sumf, %scale, %acc_g0 : f32 - scf.yield %updated : f32 - } else { - scf.yield %acc_g0 : f32 - } - %t1 = index.constant 1 : index - %valid1 = index.cmp ult, %t1, %tokens : index - %computed_g1 = scf.if %valid1 -> (f32) { - %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t1, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) - %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) - %sumf = scalar.sitofp %sum : i32 to f32 - %scale = scalar.mulf %ws, %ad : f32 - %updated = scalar.fmaf %sumf, %scale, %acc_g1 : f32 - scf.yield %updated : f32 - } else { - scf.yield %acc_g1 : f32 - } - %t2 = index.constant 2 : index - %valid2 = index.cmp ult, %t2, %tokens : index - %computed_g2 = scf.if %valid2 -> (f32) { - %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t2, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) - %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) - %sumf = scalar.sitofp %sum : i32 to f32 - %scale = scalar.mulf %ws, %ad : f32 - %updated = scalar.fmaf %sumf, %scale, %acc_g2 : f32 - scf.yield %updated : f32 - } else { - scf.yield %acc_g2 : f32 - } - %t3 = index.constant 3 : index - %valid3 = index.cmp ult, %t3, %tokens : index - %computed_g3 = scf.if %valid3 -> (f32) { - %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t3, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) - %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) - %sumf = scalar.sitofp %sum : i32 to f32 - %scale = scalar.mulf %ws, %ad : f32 - %updated = scalar.fmaf %sumf, %scale, %acc_g3 : f32 - scf.yield %updated : f32 - } else { - scf.yield %acc_g3 : f32 - } - %t4 = index.constant 4 : index - %valid4 = index.cmp ult, %t4, %tokens : index - %computed_g4 = scf.if %valid4 -> (f32) { - %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t4, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) - %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) - %sumf = scalar.sitofp %sum : i32 to f32 - %scale = scalar.mulf %ws, %ad : f32 - %updated = scalar.fmaf %sumf, %scale, %acc_g4 : f32 - scf.yield %updated : f32 - } else { - scf.yield %acc_g4 : f32 - } - func.return %computed_g0, %computed_g1, %computed_g2, %computed_g3, %computed_g4 : f32, f32, f32, f32, f32 -} - -func.def inline @ggml_matmul_vector_q6_q8_1_x4_activation_dot(%weight_format: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) { - %capacity = config.get @ggml.workload.token_capacity : index - %staged_rows, %k_partitions = func.call pure @ggml_matmul_vector_q8_geometry(%capacity, %input_size0, %output_size0, %weight_format) : (index, index, index, index) -> (index, index) - %no_staged_rows = index.constant 0 : index - %staged = index.cmp ugt, %staged_rows, %no_staged_rows : index - %tokens = index.assume %token_count [range(%token_count, 1, 1), le(%token_count, %capacity)] : index - %k = index.assume %input_size0 [range(%input_size0, 256, 32768), multiple_of(%input_size0, 256)] : index - %n = index.assume %output_size0 [range(%output_size0, 64, 262144), multiple_of(%output_size0, 64)] : index - %c0 = index.constant 0 : index - %c1 = index.constant 1 : index - %c2 = index.constant 2 : index - %c3 = index.constant 3 : index - %c4 = index.constant 4 : index - %c7 = index.constant 7 : index - %c8 = index.constant 8 : index - %c16 = index.constant 16 : index - %c32 = index.constant 32 : index - %c64 = index.constant 64 : index - %c128 = index.constant 128 : index - %c256 = index.constant 256 : index - %base = index.constant 0 : offset - %zero = scalar.constant 0.0 : f32 - %lane0 = kernel.subgroup.lane.id : index - %lane = index.assume %lane0 [range(%lane0, 0, 63)] : index - %wave = kernel.subgroup.id : index - %rows_per_group = scf.select %staged, %c32, %c8 : index - %row_base = index.mul %channel_group, %rows_per_group : index - %wave_row = index.mul %wave, %c4 : index - %cohort = index.div %lane, %c16 : index - %local_row = index.add %wave_row, %cohort : index - %row = index.add %row_base, %local_row : index - %block_lane = index.rem %lane, %c16 : index - %group = index.div %block_lane, %c2 : index - %half16 = index.rem %block_lane, %c2 : index - %half128 = index.div %group, %c4 : index - %group_in_half = index.rem %group, %c4 : index - %ql_side = index.rem %group_in_half, %c2 : index - %nibble = index.div %group_in_half, %c2 : index - %nibble_shift0 = index.mul %nibble, %c4 : index - %nibble_shift1 = index.cast %nibble_shift0 : index to i32 - %nibble_shift = vector.splat %nibble_shift1 : vector<4xi32> - %high_shift0 = index.mul %group_in_half, %c2 : index - %high_shift1 = index.cast %high_shift0 : index to i32 - %high_shift = vector.splat %high_shift1 : vector<4xi32> - %packet = index.mul %half16, %c4 : index - %q8_group_words = index.mul %group_in_half, %c8 : index - %q8_word0 = index.add %q8_group_words, %packet : index - %q8_meta = index.mul %group_in_half, %c2 : index - %blocks = index.div %k, %c256 : index - %q8_groups = index.div %k, %c128 : index - %q8_block_bytes = index.constant 144 : offset - %global_q8_row_bytes = index.scale %q8_groups, %q8_block_bytes : index, offset -> offset - %native_block_bytes = index.constant 210 : index - %cache_bytes = index.constant 33554432 : index - %weight_blocks = index.mul %blocks, %n : index - %weight_bytes = index.mul %weight_blocks, %native_block_bytes : index - %exceeds_cache = index.cmp uge, %weight_bytes, %cache_bytes : index - %pipelined = scalar.andi %staged, %exceeds_cache : i1 - %full_slab_row_bytes = index.constant 2304 : offset - %half_slab_row_bytes = index.constant 1152 : offset - %slab_row_bytes = scf.select %pipelined, %half_slab_row_bytes, %full_slab_row_bytes : offset - %q8_row_bytes = scf.select %staged, %slab_row_bytes, %global_q8_row_bytes : offset - %code_offset = index.constant 16 : offset - %record_bytes = index.constant 13440 : offset - %scale_add = index.constant 128 : offset - %raw_add = index.constant 1152 : offset - %staged_row_group = index.div %channel_group, %c2 : index - %direct_row_group = index.div %row, %c64 : index - %row_group = scf.select %staged, %staged_row_group, %direct_row_group : index - %row_lane = index.rem %row, %c64 : index - %row_group_block = index.mul %row_group, %blocks : index - %input_na, %weight_na = buffer.assume.noalias %input, %weight : buffer, buffer - %slab_blocks = scf.select %pipelined, %c4, %c8 : index - %full_stage_bytes = index.constant 11520 : offset - %q8_stage_bytes = scf.select %staged, %full_stage_bytes, %base : offset - %q8_stage = buffer.alloca align(16) %q8_stage_bytes : buffer - %q8_input = scf.select %staged, %q8_stage, %input_na : buffer - %zero_half = scalar.constant 0.0 : f16 - %zero_code = scalar.constant 0 : i8 - %zero_words = vector.splati 0 : vector<4xi32> - %prime_d, %prime_s, %prime_ql, %prime_qh = scf.if %staged -> (f16, i8, vector<4xi32>, vector<4xi32>) { - %pd, %ps, %pql, %pqh = func.call @ggml_matmul_vector_q6_prefetch_native_words(%weight_na, %row_group_block, %c0, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) - scf.yield %pd, %ps, %pql, %pqh : f16, i8, vector<4xi32>, vector<4xi32> - } else { - scf.yield %zero_half, %zero_code, %zero_words, %zero_words : f16, i8, vector<4xi32>, vector<4xi32> - } - scf.if %pipelined { - %prime_q8 = func.call @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input_na, %capacity, %c0, %blocks, %global_q8_row_bytes) : (buffer, index, index, index, offset) -> (vector<4xi32>) - func.call @ggml_matmul_vector_q8_1_x4_publish_half_slab(%q8_stage, %capacity, %c0, %prime_q8) : (buffer, index, index, vector<4xi32>) - kernel.barrier scope(workgroup) ordering(acq_rel) - } - %total_g0, %total_g1, %total_g2, %total_g3, %total_g4, %last_d, %last_s, %last_ql, %last_qh = scf.for %block = [%c0 to %blocks step %c1](%acc_g0 = %zero : f32, %acc_g1 = %zero : f32, %acc_g2 = %zero : f32, %acc_g3 = %zero : f32, %acc_g4 = %zero : f32, %carry_d = %prime_d : f16, %carry_s = %prime_s : i8, %carry_ql = %prime_ql : vector<4xi32>, %carry_qh = %prime_qh : vector<4xi32>) -> (f32, f32, f32, f32, f32, f16, i8, vector<4xi32>, vector<4xi32>) { - %block_in_slab = index.rem %block, %slab_blocks : index - %slab_boundary = index.cmp eq, %block_in_slab, %c0 : index - %direct_stage = scalar.andi %staged, %slab_boundary : i1 - %true = scalar.constant true : i1 - %not_pipelined = scalar.xori %pipelined, %true : i1 - %stage_current = scalar.andi %direct_stage, %not_pipelined : i1 - scf.if %stage_current { - func.call @ggml_matmul_vector_q8_1_x4_stage_slab(%input_na, %q8_stage, %capacity, %block, %blocks, %global_q8_row_bytes) : (buffer, buffer, index, index, index, offset) - } - %slab_number = index.div %block, %slab_blocks : index - %current_slot = index.rem %slab_number, %c2 : index - %following_slab = index.add %slab_number, %c1 : index - %following_begin = index.mul %following_slab, %slab_blocks : index - %following_exists = index.cmp ult, %following_begin, %blocks : index - %next_slot = index.rem %following_slab, %c2 : index - %staged_future = scalar.andi %pipelined, %following_exists : i1 - %current_d, %current_s, %current_ql, %current_qh = scf.if %staged -> (f16, i8, vector<4xi32>, vector<4xi32>) { - scf.yield %carry_d, %carry_s, %carry_ql, %carry_qh : f16, i8, vector<4xi32>, vector<4xi32> - } else { - %wd, %ws8, %wql, %wqh = func.call @ggml_matmul_vector_q6_prefetch_native_words(%weight_na, %row_group_block, %block, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) - scf.yield %wd, %ws8, %wql, %wqh : f16, i8, vector<4xi32>, vector<4xi32> - } - %d = scalar.extf %current_d : f16 to f32 - %sf = scalar.sitofp %current_s : i8 to f32 - %ws = scalar.mulf %d, %sf : f32 - %nibble_mask = vector.splati 252645135 : vector<4xi32> - %high_mask = vector.splati 50529027 : vector<4xi32> - %four = vector.splati 4 : vector<4xi32> - %ql_shifted = vector.shrui %current_ql, %nibble_shift : vector<4xi32> - %low = vector.andi %ql_shifted, %nibble_mask : vector<4xi32> - %qh_shifted = vector.shrui %current_qh, %high_shift : vector<4xi32> - %high0 = vector.andi %qh_shifted, %high_mask : vector<4xi32> - %high = vector.shli %high0, %four : vector<4xi32> - %code = vector.ori %low, %high : vector<4xi32> - // Guard bits prevent subtraction from borrowing across packed bytes. - %guard_mask = vector.splati -2139062144 : vector<4xi32> - %center = vector.splati 538976288 : vector<4xi32> - %guarded = vector.ori %code, %guard_mask : vector<4xi32> - %centered = vector.subi %guarded, %center : vector<4xi32> - %signed_code = vector.xori %centered, %guard_mask : vector<4xi32> - %w = vector.bitcast %signed_code : vector<4xi32> to vector<16xi8> - %next_block = index.add %block, %c1 : index - %next_exists = index.cmp ult, %next_block, %blocks : index - %has_next = scalar.andi %staged, %next_exists : i1 - %next_d, %next_s, %next_ql, %next_qh = scf.if %has_next -> (f16, i8, vector<4xi32>, vector<4xi32>) { - %wd, %ws8, %wql, %wqh = func.call @ggml_matmul_vector_q6_prefetch_native_words(%weight_na, %row_group_block, %next_block, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) - scf.yield %wd, %ws8, %wql, %wqh : f16, i8, vector<4xi32>, vector<4xi32> - } else { - scf.yield %carry_d, %carry_s, %carry_ql, %carry_qh : f16, i8, vector<4xi32>, vector<4xi32> - } - %input_block = scf.select %staged, %block_in_slab, %block : index - %q8_block0 = index.mul %input_block, %c2 : index - %q8_block = index.add %q8_block0, %half128 : index - %q8_local_block_add = index.scale %q8_block, %q8_block_bytes : index, offset -> offset - %slot_row = index.mul %current_slot, %capacity : index - %slot_byte_add0 = index.scale %slot_row, %slab_row_bytes : index, offset -> offset - %slot_byte_add = scf.select %pipelined, %slot_byte_add0, %base : offset - %q8_block_add = index.add %slot_byte_add, %q8_local_block_add : offset - %slot_complete = index.cmp eq, %block_in_slab, %c3 : index - %publish_next = scalar.andi %staged_future, %slot_complete : i1 - %next_g0, %next_g1, %next_g2, %next_g3, %next_g4 = scf.if %publish_next -> (f32, f32, f32, f32, f32) { - %next_q8 = func.call @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input_na, %capacity, %following_begin, %blocks, %global_q8_row_bytes) : (buffer, index, index, index, offset) -> (vector<4xi32>) - %v0, %v1, %v2, %v3, %v4 = func.call @ggml_matmul_vector_q6_q8_accumulate(%pipelined, %tokens, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block, %q8_input, %q8_word0, %q8_meta, %w, %ws, %acc_g0, %acc_g1, %acc_g2, %acc_g3, %acc_g4) : (i1, index, offset, offset, index, index, index, buffer, index, index, vector<16xi8>, f32, f32, f32, f32, f32, f32) -> (f32, f32, f32, f32, f32) - func.call @ggml_matmul_vector_q8_1_x4_publish_half_slab(%q8_stage, %capacity, %next_slot, %next_q8) : (buffer, index, index, vector<4xi32>) - kernel.barrier scope(workgroup) ordering(acq_rel) - scf.yield %v0, %v1, %v2, %v3, %v4 : f32, f32, f32, f32, f32 - } else { - %v0, %v1, %v2, %v3, %v4 = func.call @ggml_matmul_vector_q6_q8_accumulate(%pipelined, %tokens, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block, %q8_input, %q8_word0, %q8_meta, %w, %ws, %acc_g0, %acc_g1, %acc_g2, %acc_g3, %acc_g4) : (i1, index, offset, offset, index, index, index, buffer, index, index, vector<16xi8>, f32, f32, f32, f32, f32, f32) -> (f32, f32, f32, f32, f32) - scf.yield %v0, %v1, %v2, %v3, %v4 : f32, f32, f32, f32, f32 - } - scf.yield %next_g0, %next_g1, %next_g2, %next_g3, %next_g4, %next_d, %next_s, %next_ql, %next_qh : f32, f32, f32, f32, f32, f16, i8, vector<4xi32>, vector<4xi32> - } - %publish_format = config.get @ggml.matmul.vector.publish_format : index - func.call @ggml_matmul_vector_q8_publish(%tokens, %n, %rows_per_group, %row_base, %local_row, %block_lane, %channel_group, %output_accumulation, %output_unary_op, %epsilon, %publish_format, %total_g0, %total_g1, %total_g2, %total_g3, %total_g4, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, index, index, index, index, index, index, index, index, f32, index, f32, f32, f32, f32, f32, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) - func.return -} - - template.def<@ggml.matmul.vector_activation.core> device @ggml_matmul_vector_activation_core(%weight_format: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) { %activation_format = config.get @ggml.matmul.vector.activation_format : index %q8_format = index.constant 9 : index @@ -1454,7 +1096,9 @@ template.def<@ggml.matmul.vector_activation.core> device @ggml_matmul_vector_act %q6_format = index.constant 46 : index %uses_q6 = index.cmp eq, %weight_format, %q6_format : index scf.if %uses_q6 { - func.call @ggml_matmul_vector_q6_q8_1_x4_activation_dot(%weight_format, %token_count, %input_size0, %output_size0, %output_accumulation, %output_unary_op, %epsilon, %channel_group, %token_group, %input, %weight, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) + %token_capacity = config.get @ggml.workload.token_capacity : index + %native_wave64 = scalar.constant false : i1 + template.apply<@ggml.matmul.q6_q8_1_x4.core>(%weight_format, %native_wave64, %token_capacity, %output_size0, %token_count, %input_size0, %output_size0, %output_accumulation, %output_unary_op, %epsilon, %channel_group, %token_group, %input, %weight, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, i1, index, index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) } else { func.call @ggml_matmul_vector_q4_q8_1_x4_activation_dot(%weight_format, %token_count, %input_size0, %output_size0, %output_accumulation, %output_unary_op, %epsilon, %channel_group, %token_group, %input, %weight, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) } diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom index 55da51a6161a..ba98009efbb7 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom @@ -1,3 +1,5 @@ +template.decl @ggml.matmul.q6_q8_1_x4.core(%weight_format: index, %native_wave64: i1, %token_capacity: index, %output_capacity: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) + // Contracts GGML Q6_K weight rows directly with the Q8_1 x4 activation // layout used by llama.cpp's Vulkan matrix path. One 210-byte Q6_K block // represents 256 signed six-bit values: @@ -41,29 +43,13 @@ kernel.def export("ggml_linear_q6k_q8_1_x4") @ggml_linear_q6k_q8_1_x4(%token_cou %output_tiles = index.div %padded_output_size, %c8 : index kernel.launch.config workgroups(%output_tiles, %token_capacity, %c1) workgroup_size(%workgroup_size, %c1, %c1) : index } launch(%token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) { - %q8_noalias, %weight_noalias, %output_noalias = buffer.assume.noalias %q8_input, %weight, %output : buffer, buffer, buffer %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index - %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048), le(%token_count, %token_capacity)] : index - %bounded_output_size = index.assume %output_size [range(%output_size, 1, 262144), le(%output_size, %output_capacity)] : index - %c0 = index.constant 0 : index - %token0 = kernel.workgroup.id : index - %valid_token = index.cmp ult, %token0, %bounded_token_count : index - %safe_token = scf.select %valid_token, %token0, %c0 : index - %dot, %token, %channel, %valid_channel, %launch_token_count, %output_bound = template.apply<@ggml.linear_q6k_q8_1_x4.body>(%valid_token, %bounded_token_count, %safe_token, %input_size, %bounded_output_size, %q8_noalias, %weight_noalias) : (i1, index, index, index, index, buffer, buffer) -> (f32, index, index, i1, index, index) - %c0_i32 = scalar.constant 0 : i32 - %c0_offset = index.constant 0 : offset - %lane = kernel.subgroup.lane.id : index - %lane_i32 = index.cast %lane : index to i32 - %is_lane_zero = scalar.cmpi eq, %lane_i32, %c0_i32 : i32 - scf.if %valid_token { - scf.if %valid_channel { - scf.if %is_lane_zero { - %output_view = buffer.view %output_noalias[%c0_offset] : buffer -> view<[%launch_token_count]x[%output_bound]xf32> - view.store %dot, %output_view[%token, %channel] : f32, view<[%launch_token_count]x[%output_bound]xf32> - } - } - } + %weight_format = index.constant 6 : index + %native_wave64 = scalar.constant false : i1 + %zero = index.constant 0 : index + %zero_f32 = scalar.constant 0.0 : f32 + template.apply<@ggml.matmul.q6_q8_1_x4.core>(%weight_format, %native_wave64, %token_capacity, %output_capacity, %token_count, %input_size, %output_size, %zero, %zero, %zero_f32, %zero, %zero, %q8_input, %weight, %output, %output, %output, %output, %output, %output, %output, %output) : (index, i1, index, index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) kernel.return } @@ -76,47 +62,13 @@ kernel.def target(@ggml_q6k_q8_gfx1151_wave64) export("ggml_linear_q6k_q8_1_x4") %subgroup_size = target.subgroup.size : index kernel.launch.config workgroups(%output_capacity, %token_capacity, %c1) workgroup_size(%subgroup_size, %c1, %c1) : index } launch(%token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) { - %q8_noalias, %weight_noalias, %output_noalias = buffer.assume.noalias %q8_input, %weight, %output : buffer, buffer, buffer %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index - %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048), le(%token_count, %token_capacity)] : index - %bounded_input_size = index.assume %input_size [range(%input_size, 256, 32768), multiple_of(%input_size, 256)] : index - %bounded_output_size = index.assume %output_size [range(%output_size, 1, 262144), le(%output_size, %output_capacity)] : index - %token = kernel.workgroup.id : index - %channel = kernel.workgroup.id : index - %lane = kernel.subgroup.lane.id : index - %valid_token = index.cmp ult, %token, %bounded_token_count : index - %valid_channel = index.cmp ult, %channel, %bounded_output_size : index - %publish_output = scalar.andi %valid_token, %valid_channel : i1 - %c0 = index.constant 0 : index - %c128 = index.constant 128 : index - %c144_bytes = index.constant 144 : offset - %c210_bytes = index.constant 210 : offset - %c256 = index.constant 256 : index - %safe_token = scf.select %valid_token, %token, %c0 : index - %safe_channel = scf.select %valid_channel, %channel, %c0 : index - %q6_block_count = index.div %bounded_input_size, %c256 : index - %weight_row_bytes = index.scale %q6_block_count, %c210_bytes : index, offset -> offset - %weight_row_byte_base = index.scale %safe_channel, %weight_row_bytes : index, offset -> offset - %q8_group_count = index.div %bounded_input_size, %c128 : index - %q8_row_bytes = index.scale %q8_group_count, %c144_bytes : index, offset -> offset - %q8_row_byte_base = index.scale %safe_token, %q8_row_bytes : index, offset -> offset - %lane_acc = scf.if %publish_output -> (f32) { - %value = func.call @ggml_q6k_q8_1_x4_row_lane_wave64_block4(%bounded_input_size, %weight_noalias, %weight_row_byte_base, %q8_noalias, %q8_row_byte_base, %lane) : (index, buffer, offset, buffer, offset, index) -> (f32) - scf.yield %value : f32 - } else { - %c0_f32 = scalar.constant 0.0 : f32 - scf.yield %c0_f32 : f32 - } - %dot = kernel.subgroup.reduce %lane_acc : f32 - %is_lane_zero = index.cmp eq, %lane, %c0 : index - scf.if %publish_output { - scf.if %is_lane_zero { - %c0_offset = index.constant 0 : offset - %output_view = buffer.view %output_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]x[%bounded_output_size]xf32> - view.store %dot, %output_view[%token, %channel] : f32, view<[%bounded_token_count]x[%bounded_output_size]xf32> - } - } + %weight_format = index.constant 6 : index + %native_wave64 = scalar.constant true : i1 + %zero = index.constant 0 : index + %zero_f32 = scalar.constant 0.0 : f32 + template.apply<@ggml.matmul.q6_q8_1_x4.core>(%weight_format, %native_wave64, %token_capacity, %output_capacity, %token_count, %input_size, %output_size, %zero, %zero, %zero_f32, %zero, %zero, %q8_input, %weight, %output, %output, %output, %output, %output, %output, %output, %output) : (index, i1, index, index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) kernel.return } diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json index 32fee80b9ee1..f108d8699fd9 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json @@ -119,6 +119,42 @@ { "path": "../qwen/attention_metadata.loom", "upstream_path": "experimental/qwen/kernels/attention_metadata.loom" + }, + { + "path": "../loom-libs/ops/matmul/vector_f32_f32.loom" + }, + { + "path": "../loom-libs/motifs/matmul/vector_activation.loom" + }, + { + "path": "../loom-libs/motifs/matmul/vector_decode_pair.loom" + }, + { + "path": "../loom-libs/motifs/matmul/vector_publish.loom" + }, + { + "path": "../loom-libs/motifs/dequant.loom" + }, + { + "path": "../loom-libs/motifs/elementwise/unary_apply.loom" + }, + { + "path": "../loom-libs/motifs/q4_k_f16.loom" + }, + { + "path": "../loom-libs/motifs/q8_0_f16.loom" + }, + { + "path": "../loom-libs/motifs/q8_1_f16.loom" + }, + { + "path": "../loom-libs/motifs/f16_f16.loom" + }, + { + "path": "../loom-libs/motifs/bf16_f16.loom" + }, + { + "path": "../loom-libs/motifs/f32_f16.loom" } ], "exports": [ @@ -175,14 +211,38 @@ "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", "../loom-libs/motifs/q6_k_f16.loom", "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom" + "../loom-libs/motifs/quantize_q8_1_x4_group.loom", + "../loom-libs/ops/matmul/vector_f32_f32.loom", + "../loom-libs/motifs/matmul/vector_activation.loom", + "../loom-libs/motifs/matmul/vector_decode_pair.loom", + "../loom-libs/motifs/matmul/vector_publish.loom", + "../loom-libs/motifs/dequant.loom", + "../loom-libs/motifs/elementwise/unary_apply.loom", + "../loom-libs/motifs/q4_k_f16.loom", + "../loom-libs/motifs/q8_0_f16.loom", + "../loom-libs/motifs/q8_1_f16.loom", + "../loom-libs/motifs/f16_f16.loom", + "../loom-libs/motifs/bf16_f16.loom", + "../loom-libs/motifs/f32_f16.loom" ] }, "compile_dependencies": [ "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", "../loom-libs/motifs/q6_k_f16.loom", "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom" + "../loom-libs/motifs/quantize_q8_1_x4_group.loom", + "../loom-libs/ops/matmul/vector_f32_f32.loom", + "../loom-libs/motifs/matmul/vector_activation.loom", + "../loom-libs/motifs/matmul/vector_decode_pair.loom", + "../loom-libs/motifs/matmul/vector_publish.loom", + "../loom-libs/motifs/dequant.loom", + "../loom-libs/motifs/elementwise/unary_apply.loom", + "../loom-libs/motifs/q4_k_f16.loom", + "../loom-libs/motifs/q8_0_f16.loom", + "../loom-libs/motifs/q8_1_f16.loom", + "../loom-libs/motifs/f16_f16.loom", + "../loom-libs/motifs/bf16_f16.loom", + "../loom-libs/motifs/f32_f16.loom" ] }, { @@ -238,14 +298,38 @@ "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", "../loom-libs/motifs/q6_k_f16.loom", "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom" + "../loom-libs/motifs/quantize_q8_1_x4_group.loom", + "../loom-libs/ops/matmul/vector_f32_f32.loom", + "../loom-libs/motifs/matmul/vector_activation.loom", + "../loom-libs/motifs/matmul/vector_decode_pair.loom", + "../loom-libs/motifs/matmul/vector_publish.loom", + "../loom-libs/motifs/dequant.loom", + "../loom-libs/motifs/elementwise/unary_apply.loom", + "../loom-libs/motifs/q4_k_f16.loom", + "../loom-libs/motifs/q8_0_f16.loom", + "../loom-libs/motifs/q8_1_f16.loom", + "../loom-libs/motifs/f16_f16.loom", + "../loom-libs/motifs/bf16_f16.loom", + "../loom-libs/motifs/f32_f16.loom" ] }, "compile_dependencies": [ "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", "../loom-libs/motifs/q6_k_f16.loom", "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom" + "../loom-libs/motifs/quantize_q8_1_x4_group.loom", + "../loom-libs/ops/matmul/vector_f32_f32.loom", + "../loom-libs/motifs/matmul/vector_activation.loom", + "../loom-libs/motifs/matmul/vector_decode_pair.loom", + "../loom-libs/motifs/matmul/vector_publish.loom", + "../loom-libs/motifs/dequant.loom", + "../loom-libs/motifs/elementwise/unary_apply.loom", + "../loom-libs/motifs/q4_k_f16.loom", + "../loom-libs/motifs/q8_0_f16.loom", + "../loom-libs/motifs/q8_1_f16.loom", + "../loom-libs/motifs/f16_f16.loom", + "../loom-libs/motifs/bf16_f16.loom", + "../loom-libs/motifs/f32_f16.loom" ] }, { @@ -1077,7 +1161,19 @@ "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", "../loom-libs/motifs/q6_k_f16.loom", "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom" + "../loom-libs/motifs/quantize_q8_1_x4_group.loom", + "../loom-libs/ops/matmul/vector_f32_f32.loom", + "../loom-libs/motifs/matmul/vector_activation.loom", + "../loom-libs/motifs/matmul/vector_decode_pair.loom", + "../loom-libs/motifs/matmul/vector_publish.loom", + "../loom-libs/motifs/dequant.loom", + "../loom-libs/motifs/elementwise/unary_apply.loom", + "../loom-libs/motifs/q4_k_f16.loom", + "../loom-libs/motifs/q8_0_f16.loom", + "../loom-libs/motifs/q8_1_f16.loom", + "../loom-libs/motifs/f16_f16.loom", + "../loom-libs/motifs/bf16_f16.loom", + "../loom-libs/motifs/f32_f16.loom" ] }, { From e37b2347bf1687aa993a80d887bfa2dd68652df8 Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Mon, 16 Nov 2026 18:13:11 +0000 Subject: [PATCH 14/18] hrx: consolidate Q6 Q8 endpoint ABI and vector entry body --- .../qwen/dispatch-qwen-matmul.cpp | 45 ++- .../kernels/loom-libs/manifest.json | 138 +++++++++ .../motifs/matmul/vector_activation.loom | 9 +- .../loom-libs/ops/matmul/vector_f32_f32.loom | 54 +++- .../qwen_moe/ggml/linear_q6k_q8_1_x4.loom | 194 ------------- .../qwen_moe/ggml/q6_q8_reference_test.loom | 72 +++++ .../kernels/qwen_moe/manifest.json | 274 ++---------------- tests/hrx-backend-test.cpp | 28 +- tests/hrx-loom-jit-test.cpp | 18 +- tests/hrx-ops-test.cpp | 2 +- 10 files changed, 335 insertions(+), 499 deletions(-) delete mode 100644 ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom create mode 100644 ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/q6_q8_reference_test.loom diff --git a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp index 3be0dd5ecb95..70254ec132d1 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp @@ -16,8 +16,8 @@ namespace { static constexpr KernelCatalogRef kQwenDenseLinearQ4KQ8NextQ8Kernel = GGML_HRX_KERNEL_REF("qwen3_moe", "qwen3_moe_dense_linear_q4k_q8_1_x4_next_q8"); -static constexpr KernelCatalogRef kGgmlLinearQ6KQ8_1X4Kernel = - GGML_HRX_KERNEL_REF("qwen3_moe", "ggml_linear_q6k_q8_1_x4"); +static constexpr KernelCatalogRef kMulMatVectorQ6Q8F32Kernel = + GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_vector_q6_q8_f32"); static constexpr KernelCatalogRef kMulMatVectorF32F32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_vector_f32_f32"); @@ -193,7 +193,7 @@ static QwenMatmulMatch match_qwen_q6k_q8_matmul(const Graph & graph, const Graph match.output = output; match.input_value = alternate->alternate_value; match.input_bytes = alternate->byte_count; - match.kernel = kGgmlLinearQ6KQ8_1X4Kernel; + match.kernel = kMulMatVectorQ6Q8F32Kernel; match.input_size = input_size; match.output_size = output_size; match.token_count = token_count; @@ -346,28 +346,21 @@ Dispatch make_qwen_q6k_q8_projection_dispatch(const Value & weight, const Value dispatch_weight_layout_eligible(weight); Dispatch dispatch; dispatch.kernel = - make_kernel_specialization(row64_weight ? kMulMatVectorF32F32Kernel : kGgmlLinearQ6KQ8_1X4Kernel); + make_kernel_specialization(row64_weight ? kMulMatVectorF32F32Kernel : kMulMatVectorQ6Q8F32Kernel); dispatch.kernel.integer_parameters.emplace("token_count", 1); - if (row64_weight) { - dispatch.kernel.compile_parameters.emplace("ggml.workload.token_capacity", "1"); - dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.input_size", to_config_value(input_size)); - dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.output_size", to_config_value(output_size)); - dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.output_accumulation", "0"); - dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.output_unary_op", - std::to_string(unary_kind_config_value(UnaryKind::Identity))); - dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.publish_format", "0"); - dispatch.kernel.compile_parameters.emplace( - "ggml.matmul.vector.weight_format", - to_config_value(common_mul_mat_format_config_value(CommonMulMatWeightFormat::Q6KRow64))); - dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.activation_format", - std::to_string(GGML_TYPE_Q8_1)); - } else { - dispatch.kernel.integer_parameters.emplace("input_size", input_size); - dispatch.kernel.integer_parameters.emplace("output_size", output_size); - dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.token_capacity", "1"); - dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.output_capacity", - to_config_value(output_size)); - } + dispatch.kernel.compile_parameters.emplace("ggml.workload.token_capacity", "1"); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.input_size", to_config_value(input_size)); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.output_size", to_config_value(output_size)); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.output_accumulation", "0"); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.output_unary_op", + std::to_string(unary_kind_config_value(UnaryKind::Identity))); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.publish_format", "0"); + dispatch.kernel.compile_parameters.emplace( + "ggml.matmul.vector.weight_format", + to_config_value(common_mul_mat_format_config_value( + row64_weight ? CommonMulMatWeightFormat::Q6KRow64 : CommonMulMatWeightFormat::Q6K))); + dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.activation_format", + std::to_string(GGML_TYPE_Q8_1)); dispatch.bindings.push_back(input); if (row64_weight) { dispatch.bindings.push_back({ weight.id, 0, weight.byte_count, kQ6KPackedK256Row64ScaleRowLayout, @@ -376,9 +369,7 @@ Dispatch make_qwen_q6k_q8_projection_dispatch(const Value & weight, const Value dispatch.bindings.push_back({ weight.id, 0, weight.byte_count }); } dispatch.bindings.push_back({ output.id, 0, output.byte_count }); - if (row64_weight) { - dispatch.bindings.push_back({ output.id, 0, output.byte_count }); - } + dispatch.bindings.push_back({ output.id, 0, output.byte_count }); return dispatch; } diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json index 67732866c0c8..fe9ef097997d 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json @@ -9988,6 +9988,144 @@ "compile_dependencies": [ "motifs/elementwise/unary_apply.loom" ] + }, + { + "name": "ggml_mul_mat_vector_q6_q8_f32", + "family": "loom_libs", + "symbol": "ggml_mul_mat_vector_q6_q8_f32_wave32", + "source": "ops/matmul/vector_f32_f32.loom", + "workload_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "launch_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "bindings": [ + "input", + "weight", + "output", + "next_output" + ], + "binding_access": [ + "read", + "read", + "read_write", + "write" + ], + "target_selector": "", + "compile_recipe": { + "mode": "direct", + "primary_sources": [ + "ops/matmul/vector_f32_f32.loom" + ], + "library_sources": [ + "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", + "motifs/matmul/vector_decode_pair.loom", + "motifs/matmul/vector_publish.loom", + "motifs/dequant.loom", + "motifs/elementwise/unary_apply.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ] + }, + "compile_dependencies": [ + "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", + "motifs/matmul/vector_decode_pair.loom", + "motifs/matmul/vector_publish.loom", + "motifs/dequant.loom", + "motifs/elementwise/unary_apply.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ] + }, + { + "name": "ggml_mul_mat_vector_q6_q8_f32", + "family": "loom_libs", + "symbol": "ggml_mul_mat_vector_q6_q8_f32_wave64", + "source": "ops/matmul/vector_f32_f32.loom", + "workload_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "launch_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "bindings": [ + "input", + "weight", + "output", + "next_output" + ], + "binding_access": [ + "read", + "read", + "read_write", + "write" + ], + "target_selector": "gfx1151", + "compile_recipe": { + "mode": "direct", + "primary_sources": [ + "ops/matmul/vector_f32_f32.loom" + ], + "library_sources": [ + "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", + "motifs/matmul/vector_decode_pair.loom", + "motifs/matmul/vector_publish.loom", + "motifs/dequant.loom", + "motifs/elementwise/unary_apply.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ] + }, + "compile_dependencies": [ + "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", + "motifs/matmul/vector_decode_pair.loom", + "motifs/matmul/vector_publish.loom", + "motifs/dequant.loom", + "motifs/elementwise/unary_apply.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ] } ], "link_modules": [ diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_activation.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_activation.loom index 122c776521b4..ef58076817a3 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_activation.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_activation.loom @@ -1094,10 +1094,15 @@ template.def<@ggml.matmul.vector_activation.core> device @ggml_matmul_vector_act %uses_q8 = index.cmp eq, %activation_format, %q8_format : index scf.if %uses_q8 { %q6_format = index.constant 46 : index - %uses_q6 = index.cmp eq, %weight_format, %q6_format : index + %q6_native_format = index.constant 6 : index + %uses_row64_q6 = index.cmp eq, %weight_format, %q6_format : index + %uses_native_q6 = index.cmp eq, %weight_format, %q6_native_format : index + %uses_q6 = scalar.ori %uses_row64_q6, %uses_native_q6 : i1 scf.if %uses_q6 { %token_capacity = config.get @ggml.workload.token_capacity : index - %native_wave64 = scalar.constant false : i1 + %wave = target.subgroup.size : index + %c64 = index.constant 64 : index + %native_wave64 = index.cmp eq, %wave, %c64 : index template.apply<@ggml.matmul.q6_q8_1_x4.core>(%weight_format, %native_wave64, %token_capacity, %output_size0, %token_count, %input_size0, %output_size0, %output_accumulation, %output_unary_op, %epsilon, %channel_group, %token_group, %input, %weight, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, i1, index, index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) } else { func.call @ggml_matmul_vector_q4_q8_1_x4_activation_dot(%weight_format, %token_count, %input_size0, %output_size0, %output_accumulation, %output_unary_op, %epsilon, %channel_group, %token_group, %input, %weight, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_f32_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_f32_f32.loom index 0b869108f858..a5b2074a36ff 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_f32_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_f32_f32.loom @@ -1,3 +1,5 @@ +template.decl @ggml.matmul.vector_f32.entry(%token_count: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) + // Vector-shaped GGML matmul reduction for batch-size-1 F32 outputs. template.decl @ggml.matmul.vector_publish.next_vector4(%arg0: i1, %arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index, %arg6: vector<4xf32>, %arg7: buffer) template.decl @ggml.matmul.vector_activation.core(%arg0: index, %arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index, %arg6: f32, %arg7: index, %arg8: index, %arg9: buffer, %arg10: buffer, %arg11: buffer, %arg12: buffer, %arg13: buffer, %arg14: buffer, %arg15: buffer, %arg16: buffer, %arg17: buffer, %arg18: buffer) @@ -102,15 +104,7 @@ template.def<@ggml.matmul.vector_activation.finish_output_group> device @ggml_ma template.return } -kernel.def target(@ggml_matmul_vector_gfx11_wave64) @ggml_mul_mat_vector_f32_f32(%token_count: index) { - %input_size = config.get @ggml.matmul.vector.input_size : index - %output_size = config.get @ggml.matmul.vector.output_size : index - %token_capacity = config.get @ggml.workload.token_capacity : index - %weight_format = config.get @ggml.matmul.vector.weight_format : index - %c1 = index.constant 1 : index - %launch_x, %launch_y, %launch_z, %workgroup_size = template.apply<@ggml.matmul.vector_activation.launch>(%token_capacity, %input_size, %output_size, %weight_format) pure : (index, index, index, index) -> (index, index, index, index) - kernel.launch.config workgroups(%launch_x, %launch_y, %launch_z) workgroup_size(%workgroup_size, %c1, %c1) : index -} launch(%token_count: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) { +template.def<@ggml.matmul.vector_f32.entry> device @ggml_matmul_vector_f32_entry(%token_count: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) { %input_size = config.get @ggml.matmul.vector.input_size : index %output_size = config.get @ggml.matmul.vector.output_size : index %token_capacity = config.get @ggml.workload.token_capacity : index @@ -122,5 +116,47 @@ kernel.def target(@ggml_matmul_vector_gfx11_wave64) @ggml_mul_mat_vector_f32_f32 %channel_group = kernel.workgroup.id : index %token_group = kernel.workgroup.id : index template.apply<@ggml.matmul.vector_activation.core>(%weight_format, %bounded_token_count, %input_size, %output_size, %output_accumulation, %output_unary_op, %c0_f32, %channel_group, %token_group, %input, %weight, %output, %output, %output, %output, %output, %output, %output, %next_output) : (index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) + template.return +} + +kernel.def target(@ggml_matmul_vector_gfx11_wave64) @ggml_mul_mat_vector_f32_f32(%token_count: index) { + %input_size = config.get @ggml.matmul.vector.input_size : index + %output_size = config.get @ggml.matmul.vector.output_size : index + %token_capacity = config.get @ggml.workload.token_capacity : index + %weight_format = config.get @ggml.matmul.vector.weight_format : index + %c1 = index.constant 1 : index + %launch_x, %launch_y, %launch_z, %workgroup_size = template.apply<@ggml.matmul.vector_activation.launch>(%token_capacity, %input_size, %output_size, %weight_format) pure : (index, index, index, index) -> (index, index, index, index) + kernel.launch.config workgroups(%launch_x, %launch_y, %launch_z) workgroup_size(%workgroup_size, %c1, %c1) : index +} launch(%token_count: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) { + template.apply<@ggml.matmul.vector_f32.entry>(%token_count, %input, %weight, %output, %next_output) : (index, buffer, buffer, buffer, buffer) + kernel.return +} + +// Native Q6/Q8 retains eight wave32 rows per workgroup on gfx1100. +amdgpu.target @ggml_matmul_vector_gfx11_wave32 {subgroup_size = 32} +kernel.def target(@ggml_matmul_vector_gfx11_wave32) export("ggml_mul_mat_vector_q6_q8_f32") @ggml_mul_mat_vector_q6_q8_f32_wave32(%token_count: index) { + %output_size = config.get @ggml.matmul.vector.output_size : index + %token_capacity = config.get @ggml.workload.token_capacity : index + %c1 = index.constant 1 : index + %c7 = index.constant 7 : index + %c8 = index.constant 8 : index + %threads = index.constant 256 : index + %rounded = index.add %output_size, %c7 : index + %groups = index.div %rounded, %c8 : index + kernel.launch.config workgroups(%groups, %token_capacity, %c1) workgroup_size(%threads, %c1, %c1) : index +} launch(%token_count: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) { + template.apply<@ggml.matmul.vector_f32.entry>(%token_count, %input, %weight, %output, %next_output) : (index, buffer, buffer, buffer, buffer) + kernel.return +} + +// The catalog export name is shared with wave32; both use the common vector ABI. +kernel.def target(@ggml_matmul_vector_gfx11_wave64) export("ggml_mul_mat_vector_q6_q8_f32") @ggml_mul_mat_vector_q6_q8_f32_wave64(%token_count: index) { + %output_size = config.get @ggml.matmul.vector.output_size : index + %token_capacity = config.get @ggml.workload.token_capacity : index + %c1 = index.constant 1 : index + %wave = target.subgroup.size : index + kernel.launch.config workgroups(%output_size, %token_capacity, %c1) workgroup_size(%wave, %c1, %c1) : index +} launch(%token_count: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) { + template.apply<@ggml.matmul.vector_f32.entry>(%token_count, %input, %weight, %output, %next_output) : (index, buffer, buffer, buffer, buffer) kernel.return } diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom deleted file mode 100644 index ba98009efbb7..000000000000 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/linear_q6k_q8_1_x4.loom +++ /dev/null @@ -1,194 +0,0 @@ -template.decl @ggml.matmul.q6_q8_1_x4.core(%weight_format: index, %native_wave64: i1, %token_capacity: index, %output_capacity: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) - -// Contracts GGML Q6_K weight rows directly with the Q8_1 x4 activation -// layout used by llama.cpp's Vulkan matrix path. One 210-byte Q6_K block -// represents 256 signed six-bit values: -// -// i8 ql[128]; // low four bits -// i8 qh[64]; // high two bits -// i8 scales[16]; // signed scale for each 16-value group -// f16 d; // block-wide scale -// -// A wave owns one output value. Each lane contracts eight values from every -// Q6_K block as two native signed dot4 packets, and the wave reduces those -// partials. The kernel is the raw-layout correctness and decode baseline shared -// by dense and routed projections; prefill schedules can reuse the inline -// packed-row primitive while staging weights across multiple activation rows. -// The shared packer is linked beside this module. -template.decl @ggml.linear_q6k_q8_1_x4.body(%publish_output: i1, %token_count: index, %token0: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer) -> (f32, index, index, i1, index, index) - -amdgpu.target @ggml_q6k_q8_gfx1151_wave64 {subgroup_size = 64} - -config.decl @ggml.linear_q6k_q8_1_x4.token_capacity : %value: index where [range(%value, 1, 2048)] - -config.decl @ggml.linear_q6k_q8_1_x4.output_capacity : %value: index where [range(%value, 1, 262144)] - -kernel.decl @ggml_quantize_q8_1_x4_f32(%token_count$15: index, %input_size$16: index) launch(%token_count$17: index, %input_size$18: index, %input: buffer, %output: buffer) - -func.decl @ggml_q8_1_x4_word(%q8_input: buffer, %row_byte_base: offset, %q8_block: index, %word_in_block: index) -> (vector<4xi8>, f32) - -func.decl @ggml_q6k_q8_1_x4_row_lane_wave64_block4(%input_size: index, %weight: buffer, %weight_row_byte_base: offset, %q8_input: buffer, %q8_row_byte_base: offset, %lane: index) -> (f32) - -// Dense raw-layout baseline. A 256-thread workgroup carries eight independent -// wave32 output rows so decode launches enough waves without duplicating Q6_K -// decoding within a row. -kernel.def export("ggml_linear_q6k_q8_1_x4") @ggml_linear_q6k_q8_1_x4(%token_count: index, %input_size: index, %output_size: index) { - %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index - %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index - %c8 = index.constant 8 : index - %c7 = index.constant 7 : index - %c1 = index.constant 1 : index - %workgroup_size = index.constant 256 : index - %padded_output_size = index.add %output_capacity, %c7 : index - %output_tiles = index.div %padded_output_size, %c8 : index - kernel.launch.config workgroups(%output_tiles, %token_capacity, %c1) workgroup_size(%workgroup_size, %c1, %c1) : index -} launch(%token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) { - %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index - %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index - %weight_format = index.constant 6 : index - %native_wave64 = scalar.constant false : i1 - %zero = index.constant 0 : index - %zero_f32 = scalar.constant 0.0 : f32 - template.apply<@ggml.matmul.q6_q8_1_x4.core>(%weight_format, %native_wave64, %token_capacity, %output_capacity, %token_count, %input_size, %output_size, %zero, %zero, %zero_f32, %zero, %zero, %q8_input, %weight, %output, %output, %output, %output, %output, %output, %output, %output) : (index, i1, index, index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) - kernel.return -} - -// Wave64 vocabulary schedule matching llama.cpp Vulkan's one-row workgroup and -// four-block concurrency on AMD targets with native wave64 execution. -kernel.def target(@ggml_q6k_q8_gfx1151_wave64) export("ggml_linear_q6k_q8_1_x4") @ggml_linear_q6k_q8_1_x4_wave64_block4(%token_count: index, %input_size: index, %output_size: index) { - %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index - %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index - %c1 = index.constant 1 : index - %subgroup_size = target.subgroup.size : index - kernel.launch.config workgroups(%output_capacity, %token_capacity, %c1) workgroup_size(%subgroup_size, %c1, %c1) : index -} launch(%token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) { - %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index - %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index - %weight_format = index.constant 6 : index - %native_wave64 = scalar.constant true : i1 - %zero = index.constant 0 : index - %zero_f32 = scalar.constant 0.0 : f32 - template.apply<@ggml.matmul.q6_q8_1_x4.core>(%weight_format, %native_wave64, %token_capacity, %output_capacity, %token_count, %input_size, %output_size, %zero, %zero, %zero_f32, %zero, %zero, %q8_input, %weight, %output, %output, %output, %output, %output, %output, %output, %output) : (index, i1, index, index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) - kernel.return -} - -// Uniform packed bytes exercise both low/high Q6 fields and signed per-group -// scales. The output width crosses the eight-wave tile boundary, while K=2048 -// covers the exact mixed-format dense V contraction depth. -check.case public @ggml_linear_q6k_q8_1_x4_nonzero_tail_case { - %token_count = check.literal value(2) : index - %input_size = check.literal value(2048) : index - %output_size = check.literal value(9) : index - %input = check.generate.fill value(0.00390625) : tensor<2x2048xf32> - %q8_input = check.generate.fill value(0) : tensor<2x2304xi8> - %weight = check.generate.fill value(-86) : tensor<9x8x210xi8> - %output = check.generate.fill value(0.0) : tensor<2x9xf32> - %expected = check.generate.fill value(358.1715087890625) : tensor<2x9xf32> - kernel.launch @ggml_quantize_q8_1_x4_f32[%token_count, %input_size](%token_count, %input_size, %input, %q8_input) : [index, index](index, index, tensor<2x2048xf32>, tensor<2x2304xi8>) - kernel.launch @ggml_linear_q6k_q8_1_x4[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<2x2304xi8>, tensor<9x8x210xi8>, tensor<2x9xf32>) - check.expect.approximate actual(%output) expected(%expected) atol(0.25) rtol(9.9999999999999995e-07) nan(same) : tensor<2x9xf32>, tensor<2x9xf32> - check.return -} - -// The gfx1151 schedule has an exact target requirement, so its nonzero packed -// data case remains independently selectable from the generic gfx11 case. -check.case public @ggml_linear_q6k_q8_1_x4_wave64_block4_nonzero_tail_case { - %token_count = check.literal value(2) : index - %input_size = check.literal value(2048) : index - %output_size = check.literal value(9) : index - %input = check.generate.fill value(0.00390625) : tensor<2x2048xf32> - %q8_input = check.generate.fill value(0) : tensor<2x2304xi8> - %weight = check.generate.fill value(-86) : tensor<9x8x210xi8> - %output = check.generate.fill value(0.0) : tensor<2x9xf32> - %expected = check.generate.fill value(358.1715087890625) : tensor<2x9xf32> - kernel.launch @ggml_quantize_q8_1_x4_f32[%token_count, %input_size](%token_count, %input_size, %input, %q8_input) : [index, index](index, index, tensor<2x2048xf32>, tensor<2x2304xi8>) - kernel.launch @ggml_linear_q6k_q8_1_x4_wave64_block4[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<2x2304xi8>, tensor<9x8x210xi8>, tensor<2x9xf32>) - check.expect.approximate actual(%output) expected(%expected) atol(0.25) rtol(9.9999999999999995e-07) nan(same) : tensor<2x9xf32>, tensor<2x9xf32> - check.return -} - -check.case public @ggml_linear_q6k_q8_1_x4_benchmark_case { - %token_count = check.param.choice values([1, 17, 32, 63, 128, 129, 512]) name("token_count") : index - %input_size = check.literal value(768) : index - %output_size = check.literal value(2048) : index - %q8_input = check.generate.fill value(0) : tensor<[%token_count]x864xi8> - %weight = check.generate.fill value(0) : tensor<2048x3x210xi8> - %output = check.generate.fill value(1.0) : tensor<[%token_count]x2048xf32> - %expected = check.generate.fill value(0.0) : tensor<[%token_count]x2048xf32> - kernel.launch @ggml_linear_q6k_q8_1_x4[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<[%token_count]x864xi8>, tensor<2048x3x210xi8>, tensor<[%token_count]x2048xf32>) - check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<[%token_count]x2048xf32>, tensor<[%token_count]x2048xf32> - check.return -} - -// Production vocabulary shape used to compare the eight-row wave32 baseline -// with the one-row wave64 block schedule without model-runtime overhead. -check.case public @ggml_linear_q6k_q8_1_x4_vocabulary_wave32_benchmark_case { - %token_count = check.literal value(1) : index - %input_size = check.literal value(2048) : index - %output_size = check.literal value(151936) : index - %q8_input = check.generate.fill value(1) : tensor<1x2304xi8> - %weight = check.generate.fill value(0) : tensor<151936x8x210xi8> - %output = check.generate.fill value(1.0) : tensor<1x151936xf32> - %expected = check.generate.fill value(0.0) : tensor<1x151936xf32> - kernel.launch @ggml_linear_q6k_q8_1_x4[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<1x2304xi8>, tensor<151936x8x210xi8>, tensor<1x151936xf32>) - check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<1x151936xf32>, tensor<1x151936xf32> - check.return -} - -check.case public @ggml_linear_q6k_q8_1_x4_vocabulary_wave64_block4_benchmark_case { - %token_count = check.literal value(1) : index - %input_size = check.literal value(2048) : index - %output_size = check.literal value(151936) : index - %q8_input = check.generate.fill value(1) : tensor<1x2304xi8> - %weight = check.generate.fill value(0) : tensor<151936x8x210xi8> - %output = check.generate.fill value(1.0) : tensor<1x151936xf32> - %expected = check.generate.fill value(0.0) : tensor<1x151936xf32> - kernel.launch @ggml_linear_q6k_q8_1_x4_wave64_block4[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<1x2304xi8>, tensor<151936x8x210xi8>, tensor<1x151936xf32>) - check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<1x151936xf32>, tensor<1x151936xf32> - check.return -} - -// Exercises the complete activation-pack and dense Q6_K projection boundary at -// the K=2048, M=512 shape used by mixed-format attention V weights. The token -// buckets cover decode, awkward prefill tails, llama.cpp's 512-token -// microbatch, and larger schedules available to callers without host routing. -check.case public @ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case { - %token_count = check.param.choice values([1, 17, 32, 63, 128, 129, 512, 1024, 2048]) name("token_count") : index - %input_size = check.literal value(2048) : index - %output_size = check.literal value(512) : index - %input = check.generate.fill value(0.0) : tensor<[%token_count]x2048xf32> - %q8_input = check.generate.fill value(1) : tensor<[%token_count]x2304xi8> - %weight = check.generate.fill value(0) : tensor<512x8x210xi8> - %output = check.generate.fill value(1.0) : tensor<[%token_count]x512xf32> - %expected = check.generate.fill value(0.0) : tensor<[%token_count]x512xf32> - kernel.launch @ggml_quantize_q8_1_x4_f32[%token_count, %input_size](%token_count, %input_size, %input, %q8_input) : [index, index](index, index, tensor<[%token_count]x2048xf32>, tensor<[%token_count]x2304xi8>) - kernel.launch @ggml_linear_q6k_q8_1_x4[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<[%token_count]x2304xi8>, tensor<512x8x210xi8>, tensor<[%token_count]x512xf32>) - check.expect.approximate actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<[%token_count]x512xf32>, tensor<[%token_count]x512xf32> - check.return -} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_nonzero_tail_case> @ggml_linear_q6k_q8_1_x4_small - -check.benchmark<@ggml_linear_q6k_q8_1_x4_benchmark_case> @ggml_linear_q6k_q8_1_x4_decode {token_count = 1} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_vocabulary_wave32_benchmark_case> @ggml_linear_q6k_q8_1_x4_vocabulary_wave32 - -check.benchmark<@ggml_linear_q6k_q8_1_x4_vocabulary_wave64_block4_benchmark_case> @ggml_linear_q6k_q8_1_x4_vocabulary_wave64_block4 - -check.benchmark<@ggml_linear_q6k_q8_1_x4_benchmark_case> @ggml_linear_q6k_q8_1_x4_prefill_32 {token_count = 32} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_benchmark_case> @ggml_linear_q6k_q8_1_x4_prefill_128 {token_count = 128} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_benchmark_case> @ggml_linear_q6k_q8_1_x4_prefill_512 {token_count = 512} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6k_q8_1_x4_dense_v_decode {token_count = 1} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6k_q8_1_x4_dense_v_prefill_32 {token_count = 32} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6k_q8_1_x4_dense_v_prefill_128 {token_count = 128} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6k_q8_1_x4_dense_v_prefill_512 {token_count = 512} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6k_q8_1_x4_dense_v_prefill_1024 {token_count = 1024} - -check.benchmark<@ggml_linear_q6k_q8_1_x4_dense_v_benchmark_case> @ggml_linear_q6k_q8_1_x4_dense_v_prefill_2048 {token_count = 2048} diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/q6_q8_reference_test.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/q6_q8_reference_test.loom new file mode 100644 index 000000000000..945015794ace --- /dev/null +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/q6_q8_reference_test.loom @@ -0,0 +1,72 @@ +// Test-only native reference for packed Q6/Q8 and multi-token QKV checks. +template.decl @ggml.linear_q6k_q8_1_x4.body(%publish_output: i1, %token_count: index, %token0: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer) -> (f32, index, index, i1, index, index) + +config.decl @ggml.linear_q6k_q8_1_x4.token_capacity : %value: index where [range(%value, 1, 2048)] + +config.decl @ggml.linear_q6k_q8_1_x4.output_capacity : %value: index where [range(%value, 1, 262144)] + +kernel.decl @ggml_quantize_q8_1_x4_f32(%token_count$15: index, %input_size$16: index) launch(%token_count$17: index, %input_size$18: index, %input: buffer, %output: buffer) + +func.decl @ggml_q8_1_x4_word(%q8_input: buffer, %row_byte_base: offset, %q8_block: index, %word_in_block: index) -> (vector<4xi8>, f32) + +// Dense raw-layout baseline. A 256-thread workgroup carries eight independent +// wave32 output rows so decode launches enough waves without duplicating Q6_K +// decoding within a row. +kernel.def @ggml_linear_q6k_q8_1_x4(%token_count: index, %input_size: index, %output_size: index) { + %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index + %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index + %c8 = index.constant 8 : index + %c7 = index.constant 7 : index + %c1 = index.constant 1 : index + %workgroup_size = index.constant 256 : index + %padded_output_size = index.add %output_capacity, %c7 : index + %output_tiles = index.div %padded_output_size, %c8 : index + kernel.launch.config workgroups(%output_tiles, %token_capacity, %c1) workgroup_size(%workgroup_size, %c1, %c1) : index +} launch(%token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) { + %q8_noalias, %weight_noalias, %output_noalias = buffer.assume.noalias %q8_input, %weight, %output : buffer, buffer, buffer + %token_capacity = config.get @ggml.linear_q6k_q8_1_x4.token_capacity : index + %output_capacity = config.get @ggml.linear_q6k_q8_1_x4.output_capacity : index + %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048), le(%token_count, %token_capacity)] : index + %bounded_output_size = index.assume %output_size [range(%output_size, 1, 262144), le(%output_size, %output_capacity)] : index + %c0 = index.constant 0 : index + %token0 = kernel.workgroup.id : index + %valid_token = index.cmp ult, %token0, %bounded_token_count : index + %safe_token = scf.select %valid_token, %token0, %c0 : index + %dot, %token, %channel, %valid_channel, %launch_token_count, %output_bound = template.apply<@ggml.linear_q6k_q8_1_x4.body>(%valid_token, %bounded_token_count, %safe_token, %input_size, %bounded_output_size, %q8_noalias, %weight_noalias) : (i1, index, index, index, index, buffer, buffer) -> (f32, index, index, i1, index, index) + %c0_i32 = scalar.constant 0 : i32 + %c0_offset = index.constant 0 : offset + %lane = kernel.subgroup.lane.id : index + %lane_i32 = index.cast %lane : index to i32 + %is_lane_zero = scalar.cmpi eq, %lane_i32, %c0_i32 : i32 + scf.if %valid_token { + scf.if %valid_channel { + scf.if %is_lane_zero { + %output_view = buffer.view %output_noalias[%c0_offset] : buffer -> view<[%launch_token_count]x[%output_bound]xf32> + view.store %dot, %output_view[%token, %channel] : f32, view<[%launch_token_count]x[%output_bound]xf32> + } + } + } + kernel.return +} + +// Uniform packed bytes exercise both low/high Q6 fields and signed per-group +// scales. The output width crosses the eight-wave tile boundary, while K=2048 +// covers the exact mixed-format dense V contraction depth. +check.case public @ggml_linear_q6k_q8_1_x4_nonzero_tail_case { + %token_count = check.literal value(2) : index + %input_size = check.literal value(2048) : index + %output_size = check.literal value(9) : index + %input = check.generate.fill value(0.00390625) : tensor<2x2048xf32> + %q8_input = check.generate.fill value(0) : tensor<2x2304xi8> + %weight = check.generate.fill value(-86) : tensor<9x8x210xi8> + %output = check.generate.fill value(0.0) : tensor<2x9xf32> + %expected = check.generate.fill value(358.1715087890625) : tensor<2x9xf32> + kernel.launch @ggml_quantize_q8_1_x4_f32[%token_count, %input_size](%token_count, %input_size, %input, %q8_input) : [index, index](index, index, tensor<2x2048xf32>, tensor<2x2304xi8>) + kernel.launch @ggml_linear_q6k_q8_1_x4[%token_count, %input_size, %output_size](%token_count, %input_size, %output_size, %q8_input, %weight, %output) : [index, index, index](index, index, index, tensor<2x2304xi8>, tensor<9x8x210xi8>, tensor<2x9xf32>) + check.expect.approximate actual(%output) expected(%expected) atol(0.25) rtol(9.9999999999999995e-07) nan(same) : tensor<2x9xf32>, tensor<2x9xf32> + check.return +} + +// The gfx1151 schedule has an exact target requirement, so its nonzero packed +// data case remains independently selectable from the generic gfx11 case. +check.benchmark<@ggml_linear_q6k_q8_1_x4_nonzero_tail_case> @ggml_linear_q6k_q8_1_x4_small diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json index f108d8699fd9..1469d87f1807 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/manifest.json @@ -47,9 +47,6 @@ { "path": "../loom-libs/motifs/quantize_q8_1_x4_group.loom" }, - { - "path": "ggml/linear_q6k_q8_1_x4.loom" - }, { "path": "ggml/quantize_q8_1_x4.loom" }, @@ -121,217 +118,10 @@ "upstream_path": "experimental/qwen/kernels/attention_metadata.loom" }, { - "path": "../loom-libs/ops/matmul/vector_f32_f32.loom" - }, - { - "path": "../loom-libs/motifs/matmul/vector_activation.loom" - }, - { - "path": "../loom-libs/motifs/matmul/vector_decode_pair.loom" - }, - { - "path": "../loom-libs/motifs/matmul/vector_publish.loom" - }, - { - "path": "../loom-libs/motifs/dequant.loom" - }, - { - "path": "../loom-libs/motifs/elementwise/unary_apply.loom" - }, - { - "path": "../loom-libs/motifs/q4_k_f16.loom" - }, - { - "path": "../loom-libs/motifs/q8_0_f16.loom" - }, - { - "path": "../loom-libs/motifs/q8_1_f16.loom" - }, - { - "path": "../loom-libs/motifs/f16_f16.loom" - }, - { - "path": "../loom-libs/motifs/bf16_f16.loom" - }, - { - "path": "../loom-libs/motifs/f32_f16.loom" + "path": "ggml/q6_q8_reference_test.loom" } ], "exports": [ - { - "name": "ggml_linear_q6k_q8_1_x4", - "symbol": "ggml_linear_q6k_q8_1_x4", - "source": "ggml/linear_q6k_q8_1_x4.loom", - "workload_parameters": [ - { - "name": "token_count", - "type": "index" - }, - { - "name": "input_size", - "type": "index" - }, - { - "name": "output_size", - "type": "index" - } - ], - "launch_parameters": [ - { - "name": "token_count", - "type": "index" - }, - { - "name": "input_size", - "type": "index" - }, - { - "name": "output_size", - "type": "index" - } - ], - "bindings": [ - "q8_input", - "weight", - "output" - ], - "binding_access": [ - "read", - "read", - "read_write" - ], - "target_selector": "", - "compile_recipe": { - "mode": "archive", - "link_module": "linear_q6k_q8_1_x4_linked", - "primary_sources": [ - "ggml/linear_q6k_q8_1_x4.loom" - ], - "library_sources": [ - "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", - "../loom-libs/motifs/q6_k_f16.loom", - "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom", - "../loom-libs/ops/matmul/vector_f32_f32.loom", - "../loom-libs/motifs/matmul/vector_activation.loom", - "../loom-libs/motifs/matmul/vector_decode_pair.loom", - "../loom-libs/motifs/matmul/vector_publish.loom", - "../loom-libs/motifs/dequant.loom", - "../loom-libs/motifs/elementwise/unary_apply.loom", - "../loom-libs/motifs/q4_k_f16.loom", - "../loom-libs/motifs/q8_0_f16.loom", - "../loom-libs/motifs/q8_1_f16.loom", - "../loom-libs/motifs/f16_f16.loom", - "../loom-libs/motifs/bf16_f16.loom", - "../loom-libs/motifs/f32_f16.loom" - ] - }, - "compile_dependencies": [ - "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", - "../loom-libs/motifs/q6_k_f16.loom", - "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom", - "../loom-libs/ops/matmul/vector_f32_f32.loom", - "../loom-libs/motifs/matmul/vector_activation.loom", - "../loom-libs/motifs/matmul/vector_decode_pair.loom", - "../loom-libs/motifs/matmul/vector_publish.loom", - "../loom-libs/motifs/dequant.loom", - "../loom-libs/motifs/elementwise/unary_apply.loom", - "../loom-libs/motifs/q4_k_f16.loom", - "../loom-libs/motifs/q8_0_f16.loom", - "../loom-libs/motifs/q8_1_f16.loom", - "../loom-libs/motifs/f16_f16.loom", - "../loom-libs/motifs/bf16_f16.loom", - "../loom-libs/motifs/f32_f16.loom" - ] - }, - { - "name": "ggml_linear_q6k_q8_1_x4", - "symbol": "ggml_linear_q6k_q8_1_x4_wave64_block4", - "source": "ggml/linear_q6k_q8_1_x4.loom", - "workload_parameters": [ - { - "name": "token_count", - "type": "index" - }, - { - "name": "input_size", - "type": "index" - }, - { - "name": "output_size", - "type": "index" - } - ], - "launch_parameters": [ - { - "name": "token_count", - "type": "index" - }, - { - "name": "input_size", - "type": "index" - }, - { - "name": "output_size", - "type": "index" - } - ], - "bindings": [ - "q8_input", - "weight", - "output" - ], - "binding_access": [ - "read", - "read", - "read_write" - ], - "target_selector": "gfx1151", - "compile_recipe": { - "mode": "archive", - "link_module": "linear_q6k_q8_1_x4_linked", - "primary_sources": [ - "ggml/linear_q6k_q8_1_x4.loom" - ], - "library_sources": [ - "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", - "../loom-libs/motifs/q6_k_f16.loom", - "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom", - "../loom-libs/ops/matmul/vector_f32_f32.loom", - "../loom-libs/motifs/matmul/vector_activation.loom", - "../loom-libs/motifs/matmul/vector_decode_pair.loom", - "../loom-libs/motifs/matmul/vector_publish.loom", - "../loom-libs/motifs/dequant.loom", - "../loom-libs/motifs/elementwise/unary_apply.loom", - "../loom-libs/motifs/q4_k_f16.loom", - "../loom-libs/motifs/q8_0_f16.loom", - "../loom-libs/motifs/q8_1_f16.loom", - "../loom-libs/motifs/f16_f16.loom", - "../loom-libs/motifs/bf16_f16.loom", - "../loom-libs/motifs/f32_f16.loom" - ] - }, - "compile_dependencies": [ - "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", - "../loom-libs/motifs/q6_k_f16.loom", - "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom", - "../loom-libs/ops/matmul/vector_f32_f32.loom", - "../loom-libs/motifs/matmul/vector_activation.loom", - "../loom-libs/motifs/matmul/vector_decode_pair.loom", - "../loom-libs/motifs/matmul/vector_publish.loom", - "../loom-libs/motifs/dequant.loom", - "../loom-libs/motifs/elementwise/unary_apply.loom", - "../loom-libs/motifs/q4_k_f16.loom", - "../loom-libs/motifs/q8_0_f16.loom", - "../loom-libs/motifs/q8_1_f16.loom", - "../loom-libs/motifs/f16_f16.loom", - "../loom-libs/motifs/bf16_f16.loom", - "../loom-libs/motifs/f32_f16.loom" - ] - }, { "name": "ggml_quantize_q8_1_x4_f32", "symbol": "ggml_quantize_q8_1_x4_f32", @@ -1152,30 +942,6 @@ ":attention_qkv_quantized_linked" ] }, - { - "name": "linear_q6k_q8_1_x4_linked", - "srcs": [ - "ggml/linear_q6k_q8_1_x4.loom" - ], - "libraries": [ - "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", - "../loom-libs/motifs/q6_k_f16.loom", - "ggml/quantize_q8_1_x4.loom", - "../loom-libs/motifs/quantize_q8_1_x4_group.loom", - "../loom-libs/ops/matmul/vector_f32_f32.loom", - "../loom-libs/motifs/matmul/vector_activation.loom", - "../loom-libs/motifs/matmul/vector_decode_pair.loom", - "../loom-libs/motifs/matmul/vector_publish.loom", - "../loom-libs/motifs/dequant.loom", - "../loom-libs/motifs/elementwise/unary_apply.loom", - "../loom-libs/motifs/q4_k_f16.loom", - "../loom-libs/motifs/q8_0_f16.loom", - "../loom-libs/motifs/q8_1_f16.loom", - "../loom-libs/motifs/f16_f16.loom", - "../loom-libs/motifs/bf16_f16.loom", - "../loom-libs/motifs/f32_f16.loom" - ] - }, { "name": "routed_down_quantized_f16_wmma_linked", "srcs": [ @@ -1223,7 +989,7 @@ "libraries": [ "../loom-libs/motifs/matmul/q4_q8_1_x4_dense.loom", "../loom-libs/motifs/matmul/q4_q8_1_x4_decode.loom", - "ggml/linear_q6k_q8_1_x4.loom", + "ggml/q6_q8_reference_test.loom", "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", "../loom-libs/motifs/q6_k_f16.loom", "../loom-libs/motifs/normalization/rmsnorm_q8_1_x4.loom", @@ -1236,6 +1002,18 @@ "../loom-libs/motifs/elementwise/binary_apply.loom", "../loom-libs/ops/moe_routing_tables.loom" ] + }, + { + "name": "q6_q8_reference_test_linked", + "srcs": [ + "ggml/q6_q8_reference_test.loom" + ], + "libraries": [ + "../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom", + "../loom-libs/motifs/q6_k_f16.loom", + "ggml/quantize_q8_1_x4.loom", + "../loom-libs/motifs/quantize_q8_1_x4_group.loom" + ] } ], "plan_cases": [ @@ -1403,17 +1181,6 @@ ], "source": "ggml/quantize_q8_1_x4.loom" }, - { - "name": "linear_q6k_q8_1_x4_plan_test", - "args": [ - "$(location :linear_q6k_q8_1_x4_linked)", - "--config=ggml.linear_q6k_q8_1_x4.output_capacity=151936", - "--config=ggml.linear_q6k_q8_1_x4.token_capacity=2048", - "--dry-run", - "--output-format=jsonl" - ], - "link_module": "linear_q6k_q8_1_x4_linked" - }, { "name": "routed_down_weighted_reduce_next_rmsnorm_f32_plan_test", "args": [ @@ -1524,6 +1291,19 @@ "--output-format=jsonl" ], "source": "../qwen/attention_metadata.loom" + }, + { + "name": "q6_q8_reference_tail_plan_test", + "args": [ + "$(location :q6_q8_reference_test_linked)", + "--benchmark=@ggml_linear_q6k_q8_1_x4_small", + "--config=ggml.linear_q6k_q8_1_x4.output_capacity=9", + "--config=ggml.linear_q6k_q8_1_x4.token_capacity=2", + "--config=ggml.quantize_q8_1_x4.group_capacity=32", + "--dry-run", + "--output-format=jsonl" + ], + "link_module": "q6_q8_reference_test_linked" } ] } diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index 862e7d145518..40d1073e163b 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -4004,13 +4004,14 @@ static void schedule_qwen_terminal_q6k_q8_command(int64_t token_count) { REQUIRE(rms_dispatch.bindings[3].length == q8_transient.size); const ggml::hrx::Dispatch & vocab_dispatch = scheduler.plan().dispatches[1]; - REQUIRE(kernel_name_for_id(vocab_dispatch.kernel.kernel_id) == "qwen3_moe:ggml_linear_q6k_q8_1_x4"); + REQUIRE(kernel_name_for_id(vocab_dispatch.kernel.kernel_id) == "loom_libs:ggml_mul_mat_vector_q6_q8_f32"); REQUIRE(vocab_dispatch.kernel.integer_parameters.at("token_count") == token_count); - REQUIRE(vocab_dispatch.kernel.integer_parameters.at("input_size") == 2048); - REQUIRE(vocab_dispatch.kernel.integer_parameters.at("output_size") == 151936); - require_compile_parameter(vocab_dispatch, "ggml.linear_q6k_q8_1_x4.token_capacity", std::to_string(token_count)); - require_compile_parameter(vocab_dispatch, "ggml.linear_q6k_q8_1_x4.output_capacity", "151936"); - REQUIRE(vocab_dispatch.bindings.size() == 3); + require_compile_parameter(vocab_dispatch, "ggml.matmul.vector.input_size", "2048"); + require_compile_parameter(vocab_dispatch, "ggml.matmul.vector.output_size", "151936"); + require_compile_parameter(vocab_dispatch, "ggml.workload.token_capacity", std::to_string(token_count)); + require_compile_parameter(vocab_dispatch, "ggml.matmul.vector.weight_format", "6"); + require_compile_parameter(vocab_dispatch, "ggml.matmul.vector.activation_format", "9"); + REQUIRE(vocab_dispatch.bindings.size() == 4); REQUIRE(vocab_dispatch.bindings[0].value == q8_transient.value); REQUIRE(vocab_dispatch.bindings[0].length == q8_transient.size); @@ -4022,8 +4023,8 @@ static void schedule_qwen_terminal_q6k_q8_command(int64_t token_count) { REQUIRE(commands.commands[0].bindings.size() == 4); REQUIRE(commands.commands[0].bindings[3].name == "q8_output"); REQUIRE(commands.commands[0].bindings[3].origin == ggml::hrx::CommandBindingOrigin::Transient); - REQUIRE(commands.commands[1].bindings.size() == 3); - REQUIRE(commands.commands[1].bindings[0].name == "q8_input"); + REQUIRE(commands.commands[1].bindings.size() == 4); + REQUIRE(commands.commands[1].bindings[0].name == "input"); REQUIRE(commands.commands[1].bindings[0].origin == ggml::hrx::CommandBindingOrigin::Transient); REQUIRE(commands.commands[1].bindings[0].value == q8_transient.value); @@ -4282,8 +4283,9 @@ static void schedule_get_rows_q8_1_alternate_command(ggml_type embedding_weight_ const ggml::hrx::Dispatch & vocab_dispatch = scheduler.plan().dispatches[1]; const std::string vocab_kernel_name = kernel_name_for_id(vocab_dispatch.kernel.kernel_id); - REQUIRE(vocab_dispatch.bindings.size() == 3); - if (vocab_kernel_name == "qwen3_moe:ggml_linear_q6k_q8_1_x4") { + REQUIRE(vocab_dispatch.bindings.size() == + (vocab_kernel_name == "loom_libs:ggml_mul_mat_vector_q6_q8_f32" ? 4 : 3)); + if (vocab_kernel_name == "loom_libs:ggml_mul_mat_vector_q6_q8_f32") { REQUIRE(vocab_dispatch.bindings[0].value == alternate->alternate_value); REQUIRE(vocab_dispatch.bindings[0].length == alternate->byte_count); } else { @@ -4302,9 +4304,9 @@ static void schedule_get_rows_q8_1_alternate_command(ggml_type embedding_weight_ REQUIRE(commands.commands[0].bindings.size() == 4); REQUIRE(commands.commands[0].bindings[3].name == "next_output"); REQUIRE(commands.commands[0].bindings[3].origin == ggml::hrx::CommandBindingOrigin::Transient); - REQUIRE(commands.commands[1].bindings.size() == 3); - REQUIRE(commands.commands[1].bindings[0].name == - (vocab_kernel_name == "qwen3_moe:ggml_linear_q6k_q8_1_x4" ? "q8_input" : "input")); + REQUIRE(commands.commands[1].bindings.size() == + (vocab_kernel_name == "loom_libs:ggml_mul_mat_vector_q6_q8_f32" ? 4 : 3)); + REQUIRE(commands.commands[1].bindings[0].name == "input"); ggml_free(ctx); } diff --git a/tests/hrx-loom-jit-test.cpp b/tests/hrx-loom-jit-test.cpp index df6581f9cae9..4beaf2eb9f6a 100644 --- a/tests/hrx-loom-jit-test.cpp +++ b/tests/hrx-loom-jit-test.cpp @@ -542,7 +542,7 @@ static void run_dynamic_export_reuse_case( static void run_targeted_export_materialize_case(HrxTestDevice & device) { const ggml::hrx::KernelDefinition * definition = - find_targeted_kernel("ggml_linear_q6k_q8_1_x4", device.architecture.c_str()); + find_targeted_kernel("ggml_mul_mat_vector_q6_q8_f32", device.architecture.c_str()); if (definition == nullptr) { return; } @@ -554,11 +554,17 @@ static void run_targeted_export_materialize_case(HrxTestDevice & device) { ggml::hrx::Dispatch dispatch; dispatch.kernel.kernel_id = definition->id; dispatch.kernel.integer_parameters.emplace("token_count", 1); - dispatch.kernel.integer_parameters.emplace("input_size", 2048); - dispatch.kernel.integer_parameters.emplace("output_size", 151936); - dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.token_capacity", "1"); - dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.output_capacity", "151936"); - dispatch.bindings.resize(3); + dispatch.kernel.compile_parameters = { + { "ggml.workload.token_capacity", "1" }, + { "ggml.matmul.vector.input_size", "2048" }, + { "ggml.matmul.vector.output_size", "151936" }, + { "ggml.matmul.vector.weight_format", "6" }, + { "ggml.matmul.vector.activation_format", "9" }, + { "ggml.matmul.vector.output_accumulation", "0" }, + { "ggml.matmul.vector.output_unary_op", "23" }, + { "ggml.matmul.vector.publish_format", "0" }, + }; + dispatch.bindings.resize(4); ggml::hrx::KernelExecutableCache cache(ggml::hrx::LoomJitMode::Sync); std::vector constants; diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index 9b5008884bcd..198c567bd49f 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -7523,7 +7523,7 @@ static void run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(bool prefer_row64 = f require_kernel_subsequence( scheduled_kernel_sequence(hrx_graph, { "gfx1151", prefer_row64 }), { "qwen3_moe:qwen3_moe_rmsnorm_f32_quantize_q8_1_x4", - prefer_row64 ? "loom_libs:ggml_mul_mat_vector_f32_f32" : "qwen3_moe:ggml_linear_q6k_q8_1_x4" }); + prefer_row64 ? "loom_libs:ggml_mul_mat_vector_f32_f32" : "loom_libs:ggml_mul_mat_vector_q6_q8_f32" }); } else { require_kernel_subsequence( scheduled_kernel_sequence(hrx_graph, { "gfx1151", prefer_row64 }), From 02a8bcbe2d1d5b369b5e39ad305c660ccbc8350d Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Mon, 16 Nov 2026 18:33:45 +0000 Subject: [PATCH 15/18] hrx: separate native and row64 Q6 Q8 motifs --- .../kernels/loom-libs/manifest.json | 17 + .../motifs/matmul/q6_q8_1_x4_decode.loom | 384 +----------------- .../matmul/q6_row64_q8_1_x4_decode.loom | 365 +++++++++++++++++ .../motifs/matmul/vector_activation.loom | 20 +- 4 files changed, 398 insertions(+), 388 deletions(-) create mode 100644 ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_row64_q8_1_x4_decode.loom diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json index fe9ef097997d..47628bd230a0 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json @@ -667,6 +667,9 @@ }, { "path": "ops/reduction/amax_scale_f32.loom" + }, + { + "path": "motifs/matmul/q6_row64_q8_1_x4_decode.loom" } ], "exports": [ @@ -2738,6 +2741,7 @@ "library_sources": [ "motifs/matmul/vector_activation.loom", "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", @@ -2755,6 +2759,7 @@ "compile_dependencies": [ "motifs/matmul/vector_activation.loom", "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", @@ -2811,6 +2816,7 @@ "library_sources": [ "motifs/matmul/vector_activation.loom", "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", @@ -2828,6 +2834,7 @@ "compile_dependencies": [ "motifs/matmul/vector_activation.loom", "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", @@ -3081,6 +3088,7 @@ "library_sources": [ "motifs/matmul/vector_activation.loom", "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", @@ -3099,6 +3107,7 @@ "compile_dependencies": [ "motifs/matmul/vector_activation.loom", "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", @@ -3158,6 +3167,7 @@ "library_sources": [ "motifs/matmul/vector_activation.loom", "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", @@ -3176,6 +3186,7 @@ "compile_dependencies": [ "motifs/matmul/vector_activation.loom", "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", @@ -3229,6 +3240,7 @@ "library_sources": [ "motifs/matmul/vector_activation.loom", "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", @@ -3246,6 +3258,7 @@ "compile_dependencies": [ "motifs/matmul/vector_activation.loom", "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", @@ -10027,6 +10040,7 @@ "library_sources": [ "motifs/matmul/vector_activation.loom", "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", @@ -10044,6 +10058,7 @@ "compile_dependencies": [ "motifs/matmul/vector_activation.loom", "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", @@ -10096,6 +10111,7 @@ "library_sources": [ "motifs/matmul/vector_activation.loom", "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", @@ -10113,6 +10129,7 @@ "compile_dependencies": [ "motifs/matmul/vector_activation.loom", "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", "motifs/quantize_q8_1_x4_group.loom", "motifs/matmul/vector_decode_pair.loom", "motifs/matmul/vector_publish.loom", diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom index 8ba290409d14..a6e0f15d0108 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom @@ -1,16 +1,7 @@ // Copyright 2026 The HRX Authors // SPDX-License-Identifier: Apache-2.0 -// Q6 x Q8 schedule selection. Layout and native wave mode are compile-time inputs. -template.decl @ggml.matmul.q6_q8_1_x4.core(%weight_format: index, %native_wave64: i1, %token_capacity: index, %output_capacity: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) - -config.decl @ggml.matmul.vector.activation_format : %value: index where [range(%value, 0, 9)] -config.decl @ggml.matmul.vector.publish_format : %value: index where [range(%value, 0, 1)] -config.decl @ggml.workload.token_capacity : %value: index where [range(%value, 1, 1)] -func.decl @ggml_matmul_vector_q8_1_x4_stage_slab(%input: buffer, %stage: buffer, %capacity: index, %slab_begin: index, %blocks: index, %global_row_bytes: offset) -func.decl pure @ggml_matmul_vector_q8_geometry(%token_capacity: index, %input_size: index, %output_size: index, %weight_format: index) -> (index, index) -func.decl @ggml_matmul_vector_q8_publish(%tokens: index, %n: index, %rows_per_group: index, %row_base: index, %local_row: index, %block_lane: index, %channel_group: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %publish_format: index, %total_g0: f32, %total_g1: f32, %total_g2: f32, %total_g3: f32, %total_g4: f32, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) - +// Native Q6_K/Q8_1 x4 contractions and wave32/wave64 schedules. template.decl @ggml.linear_q6k_q8_1_x4.body(%publish_output: i1, %token_count: index, %token0: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer) -> (f32, index, index, i1, index, index) func.decl @ggml_q8_1_x4_word(%q8_input: buffer, %row_byte_base: offset, %q8_block: index, %word_in_block: index) -> (vector<4xi8>, f32) @@ -213,364 +204,6 @@ template.def<@ggml.linear_q6k_q8_1_x4.body> device @ggml_linear_q6k_q8_1_x4_body template.return %dot, %token, %channel, %valid_channel, %launch_token_count, %bounded_output_size : f32, index, index, i1, index, index } -func.def inline @ggml_matmul_vector_q6_prefetch_native_words(%weight_na: buffer, %row_group_block: index, %block: index, %row_lane: index, %half16: index, %group: index, %half128: index, %ql_side: index, %packet: index) -> (f16, i8, vector<4xi32>, vector<4xi32>) { - %c2 = index.constant 2 : index - %record_bytes = index.constant 13440 : offset - %scale_add = index.constant 128 : offset - %raw_add = index.constant 1152 : offset - %group_block = index.add %row_group_block, %block : index - %block_base = index.scale %group_block, %record_bytes : index, offset -> offset - %scale_base = index.add %block_base, %scale_add : offset - %raw_base = index.add %block_base, %raw_add : offset - %d_view = buffer.view %weight_na[%block_base] : buffer -> view<64xf16> - %s_view = buffer.view %weight_na[%scale_base] : buffer -> view<64x2x8xi8> - %raw_view = buffer.view %weight_na[%raw_base] : buffer -> view<2x3x64x8xi32> - %dh = view.load %d_view[%row_lane] : view<64xf16> -> f16 - %si = view.load %s_view[%row_lane, %half16, %group] : view<64x2x8xi8> -> i8 - %ql = vector.load %raw_view[%half128, %ql_side, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<4xi32> - %qh = vector.load %raw_view[%half128, %c2, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<4xi32> - func.return %dh, %si, %ql, %qh : f16, i8, vector<4xi32>, vector<4xi32> -} - -func.def inline @ggml_matmul_vector_q6_q8_activation_values(%pipelined: i1, %token: index, %q8_row_bytes: offset, %q8_block_add: offset, %capacity: index, %current_slot: index, %q8_block0: index, %q8_input: buffer, %q8_word0: index, %q8_meta: index) -> (vector<16xi8>, f32) { - %code_offset = index.constant 16 : offset - %a, %ad = scf.if %pipelined -> (vector<16xi8>, f32) { - %base = index.constant 0 : offset - %c4 = index.constant 4 : index - %q8_block = index.assume %q8_block0 [range(%q8_block0, 0, 7)] : index - %word = index.add %q8_word0, %c4 : index - %q_view = buffer.view %q8_input[%base] : buffer -> view<2x[%capacity]x8x36xi32> - %ds_view = buffer.view %q8_input[%base] : buffer -> view<2x[%capacity]x8x72xf16> - %words = vector.load %q_view[%current_slot, %token, %q8_block, %word] : view<2x[%capacity]x8x36xi32> -> vector<4xi32> - %a = vector.bitcast %words : vector<4xi32> to vector<16xi8> - %scale = view.load %ds_view[%current_slot, %token, %q8_block, %q8_meta] : view<2x[%capacity]x8x72xf16> -> f16 - %ad = scalar.extf %scale : f16 to f32 - scf.yield %a, %ad : vector<16xi8>, f32 - } else { - %tb = index.scale %token, %q8_row_bytes : index, offset -> offset - %qb = index.add %tb, %q8_block_add : offset - %qc = index.add %qb, %code_offset : offset - %q_view = buffer.view %q8_input[%qc] : buffer -> view<32xi32> - %ds_view = buffer.view %q8_input[%qb] : buffer -> view<8xf16> - %words = vector.load %q_view[%q8_word0] : view<32xi32> -> vector<4xi32> - %a = vector.bitcast %words : vector<4xi32> to vector<16xi8> - %scale = view.load %ds_view[%q8_meta] : view<8xf16> -> f16 - %ad = scalar.extf %scale : f16 to f32 - scf.yield %a, %ad : vector<16xi8>, f32 - } - func.return %a, %ad : vector<16xi8>, f32 -} - -func.def inline @ggml_matmul_vector_q6_q8_dot16(%pipelined: i1, %w: vector<16xi8>, %a: vector<16xi8>) -> (i32) { - %sum = scf.if %pipelined -> (i32) { - %di0 = vector.splati 0 : vector<1xi32> - %dw0 = vector.slice %w[0] : vector<16xi8> -> vector<4xi8> - %da0 = vector.slice %a[0] : vector<16xi8> -> vector<4xi8> - %di1 = vector.dot4i %dw0, %da0, %di0 : vector<4xi8>, vector<4xi8>, vector<1xi32> - %dw1 = vector.slice %w[4] : vector<16xi8> -> vector<4xi8> - %da1 = vector.slice %a[4] : vector<16xi8> -> vector<4xi8> - %di2 = vector.dot4i %dw1, %da1, %di1 : vector<4xi8>, vector<4xi8>, vector<1xi32> - %dw2 = vector.slice %w[8] : vector<16xi8> -> vector<4xi8> - %da2 = vector.slice %a[8] : vector<16xi8> -> vector<4xi8> - %di3 = vector.dot4i %dw2, %da2, %di2 : vector<4xi8>, vector<4xi8>, vector<1xi32> - %dw3 = vector.slice %w[12] : vector<16xi8> -> vector<4xi8> - %da3 = vector.slice %a[12] : vector<16xi8> -> vector<4xi8> - %di4 = vector.dot4i %dw3, %da3, %di3 : vector<4xi8>, vector<4xi8>, vector<1xi32> - %chain_sum = vector.extract %di4[0] : vector<1xi32> -> i32 - scf.yield %chain_sum : i32 - } else { - %init = vector.splati 0 : vector<4xi32> - %dot = vector.dot4i %w, %a, %init : vector<16xi8>, vector<16xi8>, vector<4xi32> - %seed = scalar.constant 0 : i32 - %direct_sum = vector.reduce %dot, %seed : vector<4xi32>, i32 - scf.yield %direct_sum : i32 - } - func.return %sum : i32 -} - -func.def inline @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input: buffer, %capacity0: index, %slot_begin: index, %blocks: index, %global_row_bytes: offset) -> (vector<4xi32>) { - %capacity = index.assume %capacity0 [range(%capacity0, 1, 5)] : index - %c0 = index.constant 0 : index - %c4 = index.constant 4 : index - %c18 = index.constant 18 : index - %c72 = index.constant 72 : index - %packet_bytes = index.constant 16 : offset - %block_bytes = index.constant 288 : offset - %thread = kernel.workitem.id : index - %total_packets = index.mul %capacity, %c72 : index - %copy_packet = index.rem %thread, %total_packets : index - %copy_token = index.div %copy_packet, %c72 : index - %copy_inner = index.rem %copy_packet, %c72 : index - %remaining = index.sub %blocks, %slot_begin : index - %full = index.cmp uge, %remaining, %c4 : index - %active0 = scf.select %full, %c4, %remaining : index - %active = index.assume %active0 [range(%active0, 1, 4)] : index - %active_packets = index.mul %active, %c18 : index - %copy_valid = index.cmp ult, %copy_inner, %active_packets : index - %safe_inner = scf.select %copy_valid, %copy_inner, %c0 : index - %copy_row = index.scale %copy_token, %global_row_bytes : index, offset -> offset - %slot_bytes = index.scale %slot_begin, %block_bytes : index, offset -> offset - %copy_bytes = index.scale %safe_inner, %packet_bytes : index, offset -> offset - %copy_base0 = index.add %copy_row, %slot_bytes : offset - %copy_base = index.add %copy_base0, %copy_bytes : offset - %source = buffer.view %input[%copy_base] : buffer -> view<4xi32> - %values = vector.load %source[%c0] : view<4xi32> -> vector<4xi32> - func.return %values : vector<4xi32> -} -func.def inline @ggml_matmul_vector_q8_1_x4_publish_half_slab(%stage: buffer, %capacity0: index, %slot0: index, %values: vector<4xi32>) { - %capacity = index.assume %capacity0 [range(%capacity0, 1, 5)] : index - %slot = index.assume %slot0 [range(%slot0, 0, 1)] : index - %c4 = index.constant 4 : index - %c72 = index.constant 72 : index - %base = index.constant 0 : offset - %thread = kernel.workitem.id : index - %total_packets = index.mul %capacity, %c72 : index - %thread_valid = index.cmp ult, %thread, %total_packets : index - scf.if %thread_valid { - %copy_token0 = index.div %thread, %c72 : index - %copy_token = index.assume %copy_token0 [range(%copy_token0, 0, 4), lt(%copy_token0, %capacity)] : index - %copy_inner = index.rem %thread, %c72 : index - %origin = index.mul %copy_inner, %c4 : index - %destination = buffer.view %stage[%base] : buffer -> view<2x[%capacity]x288xi32> - vector.store %values, %destination[%slot, %copy_token, %origin] : vector<4xi32>, view<2x[%capacity]x288xi32> - } - func.return -} - -func.def inline @ggml_matmul_vector_q6_q8_accumulate(%pipelined: i1, %tokens: index, %q8_row_bytes: offset, %q8_block_add: offset, %capacity: index, %current_slot: index, %q8_block0: index, %q8_input: buffer, %q8_word0: index, %q8_meta: index, %w: vector<16xi8>, %ws: f32, %acc_g0: f32, %acc_g1: f32, %acc_g2: f32, %acc_g3: f32, %acc_g4: f32) -> (f32, f32, f32, f32, f32) { - %code_offset = index.constant 16 : offset - %t0 = index.constant 0 : index - %valid0 = index.cmp ult, %t0, %tokens : index - %computed_g0 = scf.if %valid0 -> (f32) { - %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t0, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) - %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) - %sumf = scalar.sitofp %sum : i32 to f32 - %scale = scalar.mulf %ws, %ad : f32 - %updated = scalar.fmaf %sumf, %scale, %acc_g0 : f32 - scf.yield %updated : f32 - } else { - scf.yield %acc_g0 : f32 - } - %t1 = index.constant 1 : index - %valid1 = index.cmp ult, %t1, %tokens : index - %computed_g1 = scf.if %valid1 -> (f32) { - %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t1, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) - %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) - %sumf = scalar.sitofp %sum : i32 to f32 - %scale = scalar.mulf %ws, %ad : f32 - %updated = scalar.fmaf %sumf, %scale, %acc_g1 : f32 - scf.yield %updated : f32 - } else { - scf.yield %acc_g1 : f32 - } - %t2 = index.constant 2 : index - %valid2 = index.cmp ult, %t2, %tokens : index - %computed_g2 = scf.if %valid2 -> (f32) { - %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t2, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) - %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) - %sumf = scalar.sitofp %sum : i32 to f32 - %scale = scalar.mulf %ws, %ad : f32 - %updated = scalar.fmaf %sumf, %scale, %acc_g2 : f32 - scf.yield %updated : f32 - } else { - scf.yield %acc_g2 : f32 - } - %t3 = index.constant 3 : index - %valid3 = index.cmp ult, %t3, %tokens : index - %computed_g3 = scf.if %valid3 -> (f32) { - %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t3, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) - %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) - %sumf = scalar.sitofp %sum : i32 to f32 - %scale = scalar.mulf %ws, %ad : f32 - %updated = scalar.fmaf %sumf, %scale, %acc_g3 : f32 - scf.yield %updated : f32 - } else { - scf.yield %acc_g3 : f32 - } - %t4 = index.constant 4 : index - %valid4 = index.cmp ult, %t4, %tokens : index - %computed_g4 = scf.if %valid4 -> (f32) { - %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t4, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) - %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) - %sumf = scalar.sitofp %sum : i32 to f32 - %scale = scalar.mulf %ws, %ad : f32 - %updated = scalar.fmaf %sumf, %scale, %acc_g4 : f32 - scf.yield %updated : f32 - } else { - scf.yield %acc_g4 : f32 - } - func.return %computed_g0, %computed_g1, %computed_g2, %computed_g3, %computed_g4 : f32, f32, f32, f32, f32 -} - -func.def inline @ggml_matmul_vector_q6_q8_1_x4_activation_dot(%capacity: index, %weight_format: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) { - %staged_rows, %k_partitions = func.call pure @ggml_matmul_vector_q8_geometry(%capacity, %input_size0, %output_size0, %weight_format) : (index, index, index, index) -> (index, index) - %no_staged_rows = index.constant 0 : index - %staged = index.cmp ugt, %staged_rows, %no_staged_rows : index - %tokens = index.assume %token_count [range(%token_count, 1, 1), le(%token_count, %capacity)] : index - %k = index.assume %input_size0 [range(%input_size0, 256, 32768), multiple_of(%input_size0, 256)] : index - %n = index.assume %output_size0 [range(%output_size0, 64, 262144), multiple_of(%output_size0, 64)] : index - %c0 = index.constant 0 : index - %c1 = index.constant 1 : index - %c2 = index.constant 2 : index - %c3 = index.constant 3 : index - %c4 = index.constant 4 : index - %c7 = index.constant 7 : index - %c8 = index.constant 8 : index - %c16 = index.constant 16 : index - %c32 = index.constant 32 : index - %c64 = index.constant 64 : index - %c128 = index.constant 128 : index - %c256 = index.constant 256 : index - %base = index.constant 0 : offset - %zero = scalar.constant 0.0 : f32 - %lane0 = kernel.subgroup.lane.id : index - %lane = index.assume %lane0 [range(%lane0, 0, 63)] : index - %wave = kernel.subgroup.id : index - %rows_per_group = scf.select %staged, %c32, %c8 : index - %row_base = index.mul %channel_group, %rows_per_group : index - %wave_row = index.mul %wave, %c4 : index - %cohort = index.div %lane, %c16 : index - %local_row = index.add %wave_row, %cohort : index - %row = index.add %row_base, %local_row : index - %block_lane = index.rem %lane, %c16 : index - %group = index.div %block_lane, %c2 : index - %half16 = index.rem %block_lane, %c2 : index - %half128 = index.div %group, %c4 : index - %group_in_half = index.rem %group, %c4 : index - %ql_side = index.rem %group_in_half, %c2 : index - %nibble = index.div %group_in_half, %c2 : index - %nibble_shift0 = index.mul %nibble, %c4 : index - %nibble_shift1 = index.cast %nibble_shift0 : index to i32 - %nibble_shift = vector.splat %nibble_shift1 : vector<4xi32> - %high_shift0 = index.mul %group_in_half, %c2 : index - %high_shift1 = index.cast %high_shift0 : index to i32 - %high_shift = vector.splat %high_shift1 : vector<4xi32> - %packet = index.mul %half16, %c4 : index - %q8_group_words = index.mul %group_in_half, %c8 : index - %q8_word0 = index.add %q8_group_words, %packet : index - %q8_meta = index.mul %group_in_half, %c2 : index - %blocks = index.div %k, %c256 : index - %q8_groups = index.div %k, %c128 : index - %q8_block_bytes = index.constant 144 : offset - %global_q8_row_bytes = index.scale %q8_groups, %q8_block_bytes : index, offset -> offset - %native_block_bytes = index.constant 210 : index - %cache_bytes = index.constant 33554432 : index - %weight_blocks = index.mul %blocks, %n : index - %weight_bytes = index.mul %weight_blocks, %native_block_bytes : index - %exceeds_cache = index.cmp uge, %weight_bytes, %cache_bytes : index - %pipelined = scalar.andi %staged, %exceeds_cache : i1 - %full_slab_row_bytes = index.constant 2304 : offset - %half_slab_row_bytes = index.constant 1152 : offset - %slab_row_bytes = scf.select %pipelined, %half_slab_row_bytes, %full_slab_row_bytes : offset - %q8_row_bytes = scf.select %staged, %slab_row_bytes, %global_q8_row_bytes : offset - %code_offset = index.constant 16 : offset - %record_bytes = index.constant 13440 : offset - %scale_add = index.constant 128 : offset - %raw_add = index.constant 1152 : offset - %staged_row_group = index.div %channel_group, %c2 : index - %direct_row_group = index.div %row, %c64 : index - %row_group = scf.select %staged, %staged_row_group, %direct_row_group : index - %row_lane = index.rem %row, %c64 : index - %row_group_block = index.mul %row_group, %blocks : index - %input_na, %weight_na = buffer.assume.noalias %input, %weight : buffer, buffer - %slab_blocks = scf.select %pipelined, %c4, %c8 : index - %full_stage_bytes = index.constant 11520 : offset - %q8_stage_bytes = scf.select %staged, %full_stage_bytes, %base : offset - %q8_stage = buffer.alloca align(16) %q8_stage_bytes : buffer - %q8_input = scf.select %staged, %q8_stage, %input_na : buffer - %zero_half = scalar.constant 0.0 : f16 - %zero_code = scalar.constant 0 : i8 - %zero_words = vector.splati 0 : vector<4xi32> - %prime_d, %prime_s, %prime_ql, %prime_qh = scf.if %staged -> (f16, i8, vector<4xi32>, vector<4xi32>) { - %pd, %ps, %pql, %pqh = func.call @ggml_matmul_vector_q6_prefetch_native_words(%weight_na, %row_group_block, %c0, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) - scf.yield %pd, %ps, %pql, %pqh : f16, i8, vector<4xi32>, vector<4xi32> - } else { - scf.yield %zero_half, %zero_code, %zero_words, %zero_words : f16, i8, vector<4xi32>, vector<4xi32> - } - scf.if %pipelined { - %prime_q8 = func.call @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input_na, %capacity, %c0, %blocks, %global_q8_row_bytes) : (buffer, index, index, index, offset) -> (vector<4xi32>) - func.call @ggml_matmul_vector_q8_1_x4_publish_half_slab(%q8_stage, %capacity, %c0, %prime_q8) : (buffer, index, index, vector<4xi32>) - kernel.barrier scope(workgroup) ordering(acq_rel) - } - %total_g0, %total_g1, %total_g2, %total_g3, %total_g4, %last_d, %last_s, %last_ql, %last_qh = scf.for %block = [%c0 to %blocks step %c1](%acc_g0 = %zero : f32, %acc_g1 = %zero : f32, %acc_g2 = %zero : f32, %acc_g3 = %zero : f32, %acc_g4 = %zero : f32, %carry_d = %prime_d : f16, %carry_s = %prime_s : i8, %carry_ql = %prime_ql : vector<4xi32>, %carry_qh = %prime_qh : vector<4xi32>) -> (f32, f32, f32, f32, f32, f16, i8, vector<4xi32>, vector<4xi32>) { - %block_in_slab = index.rem %block, %slab_blocks : index - %slab_boundary = index.cmp eq, %block_in_slab, %c0 : index - %direct_stage = scalar.andi %staged, %slab_boundary : i1 - %true = scalar.constant true : i1 - %not_pipelined = scalar.xori %pipelined, %true : i1 - %stage_current = scalar.andi %direct_stage, %not_pipelined : i1 - scf.if %stage_current { - func.call @ggml_matmul_vector_q8_1_x4_stage_slab(%input_na, %q8_stage, %capacity, %block, %blocks, %global_q8_row_bytes) : (buffer, buffer, index, index, index, offset) - } - %slab_number = index.div %block, %slab_blocks : index - %current_slot = index.rem %slab_number, %c2 : index - %following_slab = index.add %slab_number, %c1 : index - %following_begin = index.mul %following_slab, %slab_blocks : index - %following_exists = index.cmp ult, %following_begin, %blocks : index - %next_slot = index.rem %following_slab, %c2 : index - %staged_future = scalar.andi %pipelined, %following_exists : i1 - %current_d, %current_s, %current_ql, %current_qh = scf.if %staged -> (f16, i8, vector<4xi32>, vector<4xi32>) { - scf.yield %carry_d, %carry_s, %carry_ql, %carry_qh : f16, i8, vector<4xi32>, vector<4xi32> - } else { - %wd, %ws8, %wql, %wqh = func.call @ggml_matmul_vector_q6_prefetch_native_words(%weight_na, %row_group_block, %block, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) - scf.yield %wd, %ws8, %wql, %wqh : f16, i8, vector<4xi32>, vector<4xi32> - } - %d = scalar.extf %current_d : f16 to f32 - %sf = scalar.sitofp %current_s : i8 to f32 - %ws = scalar.mulf %d, %sf : f32 - %nibble_mask = vector.splati 252645135 : vector<4xi32> - %high_mask = vector.splati 50529027 : vector<4xi32> - %four = vector.splati 4 : vector<4xi32> - %ql_shifted = vector.shrui %current_ql, %nibble_shift : vector<4xi32> - %low = vector.andi %ql_shifted, %nibble_mask : vector<4xi32> - %qh_shifted = vector.shrui %current_qh, %high_shift : vector<4xi32> - %high0 = vector.andi %qh_shifted, %high_mask : vector<4xi32> - %high = vector.shli %high0, %four : vector<4xi32> - %code = vector.ori %low, %high : vector<4xi32> - // Guard bits prevent subtraction from borrowing across packed bytes. - %guard_mask = vector.splati -2139062144 : vector<4xi32> - %center = vector.splati 538976288 : vector<4xi32> - %guarded = vector.ori %code, %guard_mask : vector<4xi32> - %centered = vector.subi %guarded, %center : vector<4xi32> - %signed_code = vector.xori %centered, %guard_mask : vector<4xi32> - %w = vector.bitcast %signed_code : vector<4xi32> to vector<16xi8> - %next_block = index.add %block, %c1 : index - %next_exists = index.cmp ult, %next_block, %blocks : index - %has_next = scalar.andi %staged, %next_exists : i1 - %next_d, %next_s, %next_ql, %next_qh = scf.if %has_next -> (f16, i8, vector<4xi32>, vector<4xi32>) { - %wd, %ws8, %wql, %wqh = func.call @ggml_matmul_vector_q6_prefetch_native_words(%weight_na, %row_group_block, %next_block, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) - scf.yield %wd, %ws8, %wql, %wqh : f16, i8, vector<4xi32>, vector<4xi32> - } else { - scf.yield %carry_d, %carry_s, %carry_ql, %carry_qh : f16, i8, vector<4xi32>, vector<4xi32> - } - %input_block = scf.select %staged, %block_in_slab, %block : index - %q8_block0 = index.mul %input_block, %c2 : index - %q8_block = index.add %q8_block0, %half128 : index - %q8_local_block_add = index.scale %q8_block, %q8_block_bytes : index, offset -> offset - %slot_row = index.mul %current_slot, %capacity : index - %slot_byte_add0 = index.scale %slot_row, %slab_row_bytes : index, offset -> offset - %slot_byte_add = scf.select %pipelined, %slot_byte_add0, %base : offset - %q8_block_add = index.add %slot_byte_add, %q8_local_block_add : offset - %slot_complete = index.cmp eq, %block_in_slab, %c3 : index - %publish_next = scalar.andi %staged_future, %slot_complete : i1 - %next_g0, %next_g1, %next_g2, %next_g3, %next_g4 = scf.if %publish_next -> (f32, f32, f32, f32, f32) { - %next_q8 = func.call @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input_na, %capacity, %following_begin, %blocks, %global_q8_row_bytes) : (buffer, index, index, index, offset) -> (vector<4xi32>) - %v0, %v1, %v2, %v3, %v4 = func.call @ggml_matmul_vector_q6_q8_accumulate(%pipelined, %tokens, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block, %q8_input, %q8_word0, %q8_meta, %w, %ws, %acc_g0, %acc_g1, %acc_g2, %acc_g3, %acc_g4) : (i1, index, offset, offset, index, index, index, buffer, index, index, vector<16xi8>, f32, f32, f32, f32, f32, f32) -> (f32, f32, f32, f32, f32) - func.call @ggml_matmul_vector_q8_1_x4_publish_half_slab(%q8_stage, %capacity, %next_slot, %next_q8) : (buffer, index, index, vector<4xi32>) - kernel.barrier scope(workgroup) ordering(acq_rel) - scf.yield %v0, %v1, %v2, %v3, %v4 : f32, f32, f32, f32, f32 - } else { - %v0, %v1, %v2, %v3, %v4 = func.call @ggml_matmul_vector_q6_q8_accumulate(%pipelined, %tokens, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block, %q8_input, %q8_word0, %q8_meta, %w, %ws, %acc_g0, %acc_g1, %acc_g2, %acc_g3, %acc_g4) : (i1, index, offset, offset, index, index, index, buffer, index, index, vector<16xi8>, f32, f32, f32, f32, f32, f32) -> (f32, f32, f32, f32, f32) - scf.yield %v0, %v1, %v2, %v3, %v4 : f32, f32, f32, f32, f32 - } - scf.yield %next_g0, %next_g1, %next_g2, %next_g3, %next_g4, %next_d, %next_s, %next_ql, %next_qh : f32, f32, f32, f32, f32, f16, i8, vector<4xi32>, vector<4xi32> - } - %publish_format = config.get @ggml.matmul.vector.publish_format : index - func.call @ggml_matmul_vector_q8_publish(%tokens, %n, %rows_per_group, %row_base, %local_row, %block_lane, %channel_group, %output_accumulation, %output_unary_op, %epsilon, %publish_format, %total_g0, %total_g1, %total_g2, %total_g3, %total_g4, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, index, index, index, index, index, index, index, index, f32, index, f32, f32, f32, f32, f32, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) - func.return -} - - func.def inline @ggml_q6_q8_native_wave32(%token_capacity: index, %output_capacity: index, %token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) { %q8_noalias, %weight_noalias, %output_noalias = buffer.assume.noalias %q8_input, %weight, %output : buffer, buffer, buffer %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048), le(%token_count, %token_capacity)] : index @@ -638,18 +271,3 @@ func.def inline @ggml_q6_q8_native_wave64_block4(%token_capacity: index, %output } func.return } - -template.def<@ggml.matmul.q6_q8_1_x4.core> device @ggml_q6_q8_shared_core(%weight_format: index, %native_wave64: i1, %token_capacity: index, %output_capacity: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) { - %row64_format = index.constant 46 : index - %row64 = index.cmp eq, %weight_format, %row64_format : index - scf.if %row64 { - func.call @ggml_matmul_vector_q6_q8_1_x4_activation_dot(%token_capacity, %weight_format, %token_count, %input_size0, %output_size0, %output_accumulation, %output_unary_op, %epsilon, %channel_group, %token_group, %input, %weight, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) - } else { - scf.if %native_wave64 { - func.call @ggml_q6_q8_native_wave64_block4(%token_capacity, %output_capacity, %token_count, %input_size0, %output_size0, %input, %weight, %output) : (index, index, index, index, index, buffer, buffer, buffer) - } else { - func.call @ggml_q6_q8_native_wave32(%token_capacity, %output_capacity, %token_count, %input_size0, %output_size0, %input, %weight, %output) : (index, index, index, index, index, buffer, buffer, buffer) - } - } - template.return -} diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_row64_q8_1_x4_decode.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_row64_q8_1_x4_decode.loom new file mode 100644 index 000000000000..024b7ec9f558 --- /dev/null +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/q6_row64_q8_1_x4_decode.loom @@ -0,0 +1,365 @@ +// Copyright 2026 The HRX Authors +// SPDX-License-Identifier: Apache-2.0 + +// Row64 Q6_K weights contracted with packed Q8_1 x4 activations. +config.decl @ggml.matmul.vector.publish_format : %value: index where [range(%value, 0, 1)] +func.decl @ggml_matmul_vector_q8_1_x4_stage_slab(%input: buffer, %stage: buffer, %capacity: index, %slab_begin: index, %blocks: index, %global_row_bytes: offset) +func.decl pure @ggml_matmul_vector_q8_geometry(%token_capacity: index, %input_size: index, %output_size: index, %weight_format: index) -> (index, index) +func.decl @ggml_matmul_vector_q8_publish(%tokens: index, %n: index, %rows_per_group: index, %row_base: index, %local_row: index, %block_lane: index, %channel_group: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %publish_format: index, %total_g0: f32, %total_g1: f32, %total_g2: f32, %total_g3: f32, %total_g4: f32, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) + +func.def inline @ggml_matmul_vector_q6_row64_prefetch_words(%weight_na: buffer, %row_group_block: index, %block: index, %row_lane: index, %half16: index, %group: index, %half128: index, %ql_side: index, %packet: index) -> (f16, i8, vector<4xi32>, vector<4xi32>) { + %c2 = index.constant 2 : index + %record_bytes = index.constant 13440 : offset + %scale_add = index.constant 128 : offset + %raw_add = index.constant 1152 : offset + %group_block = index.add %row_group_block, %block : index + %block_base = index.scale %group_block, %record_bytes : index, offset -> offset + %scale_base = index.add %block_base, %scale_add : offset + %raw_base = index.add %block_base, %raw_add : offset + %d_view = buffer.view %weight_na[%block_base] : buffer -> view<64xf16> + %s_view = buffer.view %weight_na[%scale_base] : buffer -> view<64x2x8xi8> + %raw_view = buffer.view %weight_na[%raw_base] : buffer -> view<2x3x64x8xi32> + %dh = view.load %d_view[%row_lane] : view<64xf16> -> f16 + %si = view.load %s_view[%row_lane, %half16, %group] : view<64x2x8xi8> -> i8 + %ql = vector.load %raw_view[%half128, %ql_side, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<4xi32> + %qh = vector.load %raw_view[%half128, %c2, %row_lane, %packet] : view<2x3x64x8xi32> -> vector<4xi32> + func.return %dh, %si, %ql, %qh : f16, i8, vector<4xi32>, vector<4xi32> +} + +func.def inline @ggml_matmul_vector_q6_q8_activation_values(%pipelined: i1, %token: index, %q8_row_bytes: offset, %q8_block_add: offset, %capacity: index, %current_slot: index, %q8_block0: index, %q8_input: buffer, %q8_word0: index, %q8_meta: index) -> (vector<16xi8>, f32) { + %code_offset = index.constant 16 : offset + %a, %ad = scf.if %pipelined -> (vector<16xi8>, f32) { + %base = index.constant 0 : offset + %c4 = index.constant 4 : index + %q8_block = index.assume %q8_block0 [range(%q8_block0, 0, 7)] : index + %word = index.add %q8_word0, %c4 : index + %q_view = buffer.view %q8_input[%base] : buffer -> view<2x[%capacity]x8x36xi32> + %ds_view = buffer.view %q8_input[%base] : buffer -> view<2x[%capacity]x8x72xf16> + %words = vector.load %q_view[%current_slot, %token, %q8_block, %word] : view<2x[%capacity]x8x36xi32> -> vector<4xi32> + %a = vector.bitcast %words : vector<4xi32> to vector<16xi8> + %scale = view.load %ds_view[%current_slot, %token, %q8_block, %q8_meta] : view<2x[%capacity]x8x72xf16> -> f16 + %ad = scalar.extf %scale : f16 to f32 + scf.yield %a, %ad : vector<16xi8>, f32 + } else { + %tb = index.scale %token, %q8_row_bytes : index, offset -> offset + %qb = index.add %tb, %q8_block_add : offset + %qc = index.add %qb, %code_offset : offset + %q_view = buffer.view %q8_input[%qc] : buffer -> view<32xi32> + %ds_view = buffer.view %q8_input[%qb] : buffer -> view<8xf16> + %words = vector.load %q_view[%q8_word0] : view<32xi32> -> vector<4xi32> + %a = vector.bitcast %words : vector<4xi32> to vector<16xi8> + %scale = view.load %ds_view[%q8_meta] : view<8xf16> -> f16 + %ad = scalar.extf %scale : f16 to f32 + scf.yield %a, %ad : vector<16xi8>, f32 + } + func.return %a, %ad : vector<16xi8>, f32 +} + +func.def inline @ggml_matmul_vector_q6_q8_dot16(%pipelined: i1, %w: vector<16xi8>, %a: vector<16xi8>) -> (i32) { + %sum = scf.if %pipelined -> (i32) { + %di0 = vector.splati 0 : vector<1xi32> + %dw0 = vector.slice %w[0] : vector<16xi8> -> vector<4xi8> + %da0 = vector.slice %a[0] : vector<16xi8> -> vector<4xi8> + %di1 = vector.dot4i %dw0, %da0, %di0 : vector<4xi8>, vector<4xi8>, vector<1xi32> + %dw1 = vector.slice %w[4] : vector<16xi8> -> vector<4xi8> + %da1 = vector.slice %a[4] : vector<16xi8> -> vector<4xi8> + %di2 = vector.dot4i %dw1, %da1, %di1 : vector<4xi8>, vector<4xi8>, vector<1xi32> + %dw2 = vector.slice %w[8] : vector<16xi8> -> vector<4xi8> + %da2 = vector.slice %a[8] : vector<16xi8> -> vector<4xi8> + %di3 = vector.dot4i %dw2, %da2, %di2 : vector<4xi8>, vector<4xi8>, vector<1xi32> + %dw3 = vector.slice %w[12] : vector<16xi8> -> vector<4xi8> + %da3 = vector.slice %a[12] : vector<16xi8> -> vector<4xi8> + %di4 = vector.dot4i %dw3, %da3, %di3 : vector<4xi8>, vector<4xi8>, vector<1xi32> + %chain_sum = vector.extract %di4[0] : vector<1xi32> -> i32 + scf.yield %chain_sum : i32 + } else { + %init = vector.splati 0 : vector<4xi32> + %dot = vector.dot4i %w, %a, %init : vector<16xi8>, vector<16xi8>, vector<4xi32> + %seed = scalar.constant 0 : i32 + %direct_sum = vector.reduce %dot, %seed : vector<4xi32>, i32 + scf.yield %direct_sum : i32 + } + func.return %sum : i32 +} + +func.def inline @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input: buffer, %capacity0: index, %slot_begin: index, %blocks: index, %global_row_bytes: offset) -> (vector<4xi32>) { + %capacity = index.assume %capacity0 [range(%capacity0, 1, 5)] : index + %c0 = index.constant 0 : index + %c4 = index.constant 4 : index + %c18 = index.constant 18 : index + %c72 = index.constant 72 : index + %packet_bytes = index.constant 16 : offset + %block_bytes = index.constant 288 : offset + %thread = kernel.workitem.id : index + %total_packets = index.mul %capacity, %c72 : index + %copy_packet = index.rem %thread, %total_packets : index + %copy_token = index.div %copy_packet, %c72 : index + %copy_inner = index.rem %copy_packet, %c72 : index + %remaining = index.sub %blocks, %slot_begin : index + %full = index.cmp uge, %remaining, %c4 : index + %active0 = scf.select %full, %c4, %remaining : index + %active = index.assume %active0 [range(%active0, 1, 4)] : index + %active_packets = index.mul %active, %c18 : index + %copy_valid = index.cmp ult, %copy_inner, %active_packets : index + %safe_inner = scf.select %copy_valid, %copy_inner, %c0 : index + %copy_row = index.scale %copy_token, %global_row_bytes : index, offset -> offset + %slot_bytes = index.scale %slot_begin, %block_bytes : index, offset -> offset + %copy_bytes = index.scale %safe_inner, %packet_bytes : index, offset -> offset + %copy_base0 = index.add %copy_row, %slot_bytes : offset + %copy_base = index.add %copy_base0, %copy_bytes : offset + %source = buffer.view %input[%copy_base] : buffer -> view<4xi32> + %values = vector.load %source[%c0] : view<4xi32> -> vector<4xi32> + func.return %values : vector<4xi32> +} +func.def inline @ggml_matmul_vector_q8_1_x4_publish_half_slab(%stage: buffer, %capacity0: index, %slot0: index, %values: vector<4xi32>) { + %capacity = index.assume %capacity0 [range(%capacity0, 1, 5)] : index + %slot = index.assume %slot0 [range(%slot0, 0, 1)] : index + %c4 = index.constant 4 : index + %c72 = index.constant 72 : index + %base = index.constant 0 : offset + %thread = kernel.workitem.id : index + %total_packets = index.mul %capacity, %c72 : index + %thread_valid = index.cmp ult, %thread, %total_packets : index + scf.if %thread_valid { + %copy_token0 = index.div %thread, %c72 : index + %copy_token = index.assume %copy_token0 [range(%copy_token0, 0, 4), lt(%copy_token0, %capacity)] : index + %copy_inner = index.rem %thread, %c72 : index + %origin = index.mul %copy_inner, %c4 : index + %destination = buffer.view %stage[%base] : buffer -> view<2x[%capacity]x288xi32> + vector.store %values, %destination[%slot, %copy_token, %origin] : vector<4xi32>, view<2x[%capacity]x288xi32> + } + func.return +} + +func.def inline @ggml_matmul_vector_q6_q8_accumulate(%pipelined: i1, %tokens: index, %q8_row_bytes: offset, %q8_block_add: offset, %capacity: index, %current_slot: index, %q8_block0: index, %q8_input: buffer, %q8_word0: index, %q8_meta: index, %w: vector<16xi8>, %ws: f32, %acc_g0: f32, %acc_g1: f32, %acc_g2: f32, %acc_g3: f32, %acc_g4: f32) -> (f32, f32, f32, f32, f32) { + %code_offset = index.constant 16 : offset + %t0 = index.constant 0 : index + %valid0 = index.cmp ult, %t0, %tokens : index + %computed_g0 = scf.if %valid0 -> (f32) { + %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t0, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) + %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) + %sumf = scalar.sitofp %sum : i32 to f32 + %scale = scalar.mulf %ws, %ad : f32 + %updated = scalar.fmaf %sumf, %scale, %acc_g0 : f32 + scf.yield %updated : f32 + } else { + scf.yield %acc_g0 : f32 + } + %t1 = index.constant 1 : index + %valid1 = index.cmp ult, %t1, %tokens : index + %computed_g1 = scf.if %valid1 -> (f32) { + %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t1, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) + %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) + %sumf = scalar.sitofp %sum : i32 to f32 + %scale = scalar.mulf %ws, %ad : f32 + %updated = scalar.fmaf %sumf, %scale, %acc_g1 : f32 + scf.yield %updated : f32 + } else { + scf.yield %acc_g1 : f32 + } + %t2 = index.constant 2 : index + %valid2 = index.cmp ult, %t2, %tokens : index + %computed_g2 = scf.if %valid2 -> (f32) { + %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t2, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) + %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) + %sumf = scalar.sitofp %sum : i32 to f32 + %scale = scalar.mulf %ws, %ad : f32 + %updated = scalar.fmaf %sumf, %scale, %acc_g2 : f32 + scf.yield %updated : f32 + } else { + scf.yield %acc_g2 : f32 + } + %t3 = index.constant 3 : index + %valid3 = index.cmp ult, %t3, %tokens : index + %computed_g3 = scf.if %valid3 -> (f32) { + %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t3, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) + %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) + %sumf = scalar.sitofp %sum : i32 to f32 + %scale = scalar.mulf %ws, %ad : f32 + %updated = scalar.fmaf %sumf, %scale, %acc_g3 : f32 + scf.yield %updated : f32 + } else { + scf.yield %acc_g3 : f32 + } + %t4 = index.constant 4 : index + %valid4 = index.cmp ult, %t4, %tokens : index + %computed_g4 = scf.if %valid4 -> (f32) { + %a, %ad = func.call @ggml_matmul_vector_q6_q8_activation_values(%pipelined, %t4, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block0, %q8_input, %q8_word0, %q8_meta) : (i1, index, offset, offset, index, index, index, buffer, index, index) -> (vector<16xi8>, f32) + %sum = func.call @ggml_matmul_vector_q6_q8_dot16(%pipelined, %w, %a) : (i1, vector<16xi8>, vector<16xi8>) -> (i32) + %sumf = scalar.sitofp %sum : i32 to f32 + %scale = scalar.mulf %ws, %ad : f32 + %updated = scalar.fmaf %sumf, %scale, %acc_g4 : f32 + scf.yield %updated : f32 + } else { + scf.yield %acc_g4 : f32 + } + func.return %computed_g0, %computed_g1, %computed_g2, %computed_g3, %computed_g4 : f32, f32, f32, f32, f32 +} + +func.def inline @ggml_matmul_vector_q6_row64_q8_1_x4_activation_dot(%capacity: index, %weight_format: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) { + %staged_rows, %k_partitions = func.call pure @ggml_matmul_vector_q8_geometry(%capacity, %input_size0, %output_size0, %weight_format) : (index, index, index, index) -> (index, index) + %no_staged_rows = index.constant 0 : index + %staged = index.cmp ugt, %staged_rows, %no_staged_rows : index + %tokens = index.assume %token_count [range(%token_count, 1, 1), le(%token_count, %capacity)] : index + %k = index.assume %input_size0 [range(%input_size0, 256, 32768), multiple_of(%input_size0, 256)] : index + %n = index.assume %output_size0 [range(%output_size0, 64, 262144), multiple_of(%output_size0, 64)] : index + %c0 = index.constant 0 : index + %c1 = index.constant 1 : index + %c2 = index.constant 2 : index + %c3 = index.constant 3 : index + %c4 = index.constant 4 : index + %c7 = index.constant 7 : index + %c8 = index.constant 8 : index + %c16 = index.constant 16 : index + %c32 = index.constant 32 : index + %c64 = index.constant 64 : index + %c128 = index.constant 128 : index + %c256 = index.constant 256 : index + %base = index.constant 0 : offset + %zero = scalar.constant 0.0 : f32 + %lane0 = kernel.subgroup.lane.id : index + %lane = index.assume %lane0 [range(%lane0, 0, 63)] : index + %wave = kernel.subgroup.id : index + %rows_per_group = scf.select %staged, %c32, %c8 : index + %row_base = index.mul %channel_group, %rows_per_group : index + %wave_row = index.mul %wave, %c4 : index + %cohort = index.div %lane, %c16 : index + %local_row = index.add %wave_row, %cohort : index + %row = index.add %row_base, %local_row : index + %block_lane = index.rem %lane, %c16 : index + %group = index.div %block_lane, %c2 : index + %half16 = index.rem %block_lane, %c2 : index + %half128 = index.div %group, %c4 : index + %group_in_half = index.rem %group, %c4 : index + %ql_side = index.rem %group_in_half, %c2 : index + %nibble = index.div %group_in_half, %c2 : index + %nibble_shift0 = index.mul %nibble, %c4 : index + %nibble_shift1 = index.cast %nibble_shift0 : index to i32 + %nibble_shift = vector.splat %nibble_shift1 : vector<4xi32> + %high_shift0 = index.mul %group_in_half, %c2 : index + %high_shift1 = index.cast %high_shift0 : index to i32 + %high_shift = vector.splat %high_shift1 : vector<4xi32> + %packet = index.mul %half16, %c4 : index + %q8_group_words = index.mul %group_in_half, %c8 : index + %q8_word0 = index.add %q8_group_words, %packet : index + %q8_meta = index.mul %group_in_half, %c2 : index + %blocks = index.div %k, %c256 : index + %q8_groups = index.div %k, %c128 : index + %q8_block_bytes = index.constant 144 : offset + %global_q8_row_bytes = index.scale %q8_groups, %q8_block_bytes : index, offset -> offset + %native_block_bytes = index.constant 210 : index + %cache_bytes = index.constant 33554432 : index + %weight_blocks = index.mul %blocks, %n : index + %weight_bytes = index.mul %weight_blocks, %native_block_bytes : index + %exceeds_cache = index.cmp uge, %weight_bytes, %cache_bytes : index + %pipelined = scalar.andi %staged, %exceeds_cache : i1 + %full_slab_row_bytes = index.constant 2304 : offset + %half_slab_row_bytes = index.constant 1152 : offset + %slab_row_bytes = scf.select %pipelined, %half_slab_row_bytes, %full_slab_row_bytes : offset + %q8_row_bytes = scf.select %staged, %slab_row_bytes, %global_q8_row_bytes : offset + %code_offset = index.constant 16 : offset + %record_bytes = index.constant 13440 : offset + %scale_add = index.constant 128 : offset + %raw_add = index.constant 1152 : offset + %staged_row_group = index.div %channel_group, %c2 : index + %direct_row_group = index.div %row, %c64 : index + %row_group = scf.select %staged, %staged_row_group, %direct_row_group : index + %row_lane = index.rem %row, %c64 : index + %row_group_block = index.mul %row_group, %blocks : index + %input_na, %weight_na = buffer.assume.noalias %input, %weight : buffer, buffer + %slab_blocks = scf.select %pipelined, %c4, %c8 : index + %full_stage_bytes = index.constant 11520 : offset + %q8_stage_bytes = scf.select %staged, %full_stage_bytes, %base : offset + %q8_stage = buffer.alloca align(16) %q8_stage_bytes : buffer + %q8_input = scf.select %staged, %q8_stage, %input_na : buffer + %zero_half = scalar.constant 0.0 : f16 + %zero_code = scalar.constant 0 : i8 + %zero_words = vector.splati 0 : vector<4xi32> + %prime_d, %prime_s, %prime_ql, %prime_qh = scf.if %staged -> (f16, i8, vector<4xi32>, vector<4xi32>) { + %pd, %ps, %pql, %pqh = func.call @ggml_matmul_vector_q6_row64_prefetch_words(%weight_na, %row_group_block, %c0, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) + scf.yield %pd, %ps, %pql, %pqh : f16, i8, vector<4xi32>, vector<4xi32> + } else { + scf.yield %zero_half, %zero_code, %zero_words, %zero_words : f16, i8, vector<4xi32>, vector<4xi32> + } + scf.if %pipelined { + %prime_q8 = func.call @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input_na, %capacity, %c0, %blocks, %global_q8_row_bytes) : (buffer, index, index, index, offset) -> (vector<4xi32>) + func.call @ggml_matmul_vector_q8_1_x4_publish_half_slab(%q8_stage, %capacity, %c0, %prime_q8) : (buffer, index, index, vector<4xi32>) + kernel.barrier scope(workgroup) ordering(acq_rel) + } + %total_g0, %total_g1, %total_g2, %total_g3, %total_g4, %last_d, %last_s, %last_ql, %last_qh = scf.for %block = [%c0 to %blocks step %c1](%acc_g0 = %zero : f32, %acc_g1 = %zero : f32, %acc_g2 = %zero : f32, %acc_g3 = %zero : f32, %acc_g4 = %zero : f32, %carry_d = %prime_d : f16, %carry_s = %prime_s : i8, %carry_ql = %prime_ql : vector<4xi32>, %carry_qh = %prime_qh : vector<4xi32>) -> (f32, f32, f32, f32, f32, f16, i8, vector<4xi32>, vector<4xi32>) { + %block_in_slab = index.rem %block, %slab_blocks : index + %slab_boundary = index.cmp eq, %block_in_slab, %c0 : index + %direct_stage = scalar.andi %staged, %slab_boundary : i1 + %true = scalar.constant true : i1 + %not_pipelined = scalar.xori %pipelined, %true : i1 + %stage_current = scalar.andi %direct_stage, %not_pipelined : i1 + scf.if %stage_current { + func.call @ggml_matmul_vector_q8_1_x4_stage_slab(%input_na, %q8_stage, %capacity, %block, %blocks, %global_q8_row_bytes) : (buffer, buffer, index, index, index, offset) + } + %slab_number = index.div %block, %slab_blocks : index + %current_slot = index.rem %slab_number, %c2 : index + %following_slab = index.add %slab_number, %c1 : index + %following_begin = index.mul %following_slab, %slab_blocks : index + %following_exists = index.cmp ult, %following_begin, %blocks : index + %next_slot = index.rem %following_slab, %c2 : index + %staged_future = scalar.andi %pipelined, %following_exists : i1 + %current_d, %current_s, %current_ql, %current_qh = scf.if %staged -> (f16, i8, vector<4xi32>, vector<4xi32>) { + scf.yield %carry_d, %carry_s, %carry_ql, %carry_qh : f16, i8, vector<4xi32>, vector<4xi32> + } else { + %wd, %ws8, %wql, %wqh = func.call @ggml_matmul_vector_q6_row64_prefetch_words(%weight_na, %row_group_block, %block, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) + scf.yield %wd, %ws8, %wql, %wqh : f16, i8, vector<4xi32>, vector<4xi32> + } + %d = scalar.extf %current_d : f16 to f32 + %sf = scalar.sitofp %current_s : i8 to f32 + %ws = scalar.mulf %d, %sf : f32 + %nibble_mask = vector.splati 252645135 : vector<4xi32> + %high_mask = vector.splati 50529027 : vector<4xi32> + %four = vector.splati 4 : vector<4xi32> + %ql_shifted = vector.shrui %current_ql, %nibble_shift : vector<4xi32> + %low = vector.andi %ql_shifted, %nibble_mask : vector<4xi32> + %qh_shifted = vector.shrui %current_qh, %high_shift : vector<4xi32> + %high0 = vector.andi %qh_shifted, %high_mask : vector<4xi32> + %high = vector.shli %high0, %four : vector<4xi32> + %code = vector.ori %low, %high : vector<4xi32> + // Guard bits prevent subtraction from borrowing across packed bytes. + %guard_mask = vector.splati -2139062144 : vector<4xi32> + %center = vector.splati 538976288 : vector<4xi32> + %guarded = vector.ori %code, %guard_mask : vector<4xi32> + %centered = vector.subi %guarded, %center : vector<4xi32> + %signed_code = vector.xori %centered, %guard_mask : vector<4xi32> + %w = vector.bitcast %signed_code : vector<4xi32> to vector<16xi8> + %next_block = index.add %block, %c1 : index + %next_exists = index.cmp ult, %next_block, %blocks : index + %has_next = scalar.andi %staged, %next_exists : i1 + %next_d, %next_s, %next_ql, %next_qh = scf.if %has_next -> (f16, i8, vector<4xi32>, vector<4xi32>) { + %wd, %ws8, %wql, %wqh = func.call @ggml_matmul_vector_q6_row64_prefetch_words(%weight_na, %row_group_block, %next_block, %row_lane, %half16, %group, %half128, %ql_side, %packet) : (buffer, index, index, index, index, index, index, index, index) -> (f16, i8, vector<4xi32>, vector<4xi32>) + scf.yield %wd, %ws8, %wql, %wqh : f16, i8, vector<4xi32>, vector<4xi32> + } else { + scf.yield %carry_d, %carry_s, %carry_ql, %carry_qh : f16, i8, vector<4xi32>, vector<4xi32> + } + %input_block = scf.select %staged, %block_in_slab, %block : index + %q8_block0 = index.mul %input_block, %c2 : index + %q8_block = index.add %q8_block0, %half128 : index + %q8_local_block_add = index.scale %q8_block, %q8_block_bytes : index, offset -> offset + %slot_row = index.mul %current_slot, %capacity : index + %slot_byte_add0 = index.scale %slot_row, %slab_row_bytes : index, offset -> offset + %slot_byte_add = scf.select %pipelined, %slot_byte_add0, %base : offset + %q8_block_add = index.add %slot_byte_add, %q8_local_block_add : offset + %slot_complete = index.cmp eq, %block_in_slab, %c3 : index + %publish_next = scalar.andi %staged_future, %slot_complete : i1 + %next_g0, %next_g1, %next_g2, %next_g3, %next_g4 = scf.if %publish_next -> (f32, f32, f32, f32, f32) { + %next_q8 = func.call @ggml_matmul_vector_q8_1_x4_prefetch_half_slab(%input_na, %capacity, %following_begin, %blocks, %global_q8_row_bytes) : (buffer, index, index, index, offset) -> (vector<4xi32>) + %v0, %v1, %v2, %v3, %v4 = func.call @ggml_matmul_vector_q6_q8_accumulate(%pipelined, %tokens, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block, %q8_input, %q8_word0, %q8_meta, %w, %ws, %acc_g0, %acc_g1, %acc_g2, %acc_g3, %acc_g4) : (i1, index, offset, offset, index, index, index, buffer, index, index, vector<16xi8>, f32, f32, f32, f32, f32, f32) -> (f32, f32, f32, f32, f32) + func.call @ggml_matmul_vector_q8_1_x4_publish_half_slab(%q8_stage, %capacity, %next_slot, %next_q8) : (buffer, index, index, vector<4xi32>) + kernel.barrier scope(workgroup) ordering(acq_rel) + scf.yield %v0, %v1, %v2, %v3, %v4 : f32, f32, f32, f32, f32 + } else { + %v0, %v1, %v2, %v3, %v4 = func.call @ggml_matmul_vector_q6_q8_accumulate(%pipelined, %tokens, %q8_row_bytes, %q8_block_add, %capacity, %current_slot, %q8_block, %q8_input, %q8_word0, %q8_meta, %w, %ws, %acc_g0, %acc_g1, %acc_g2, %acc_g3, %acc_g4) : (i1, index, offset, offset, index, index, index, buffer, index, index, vector<16xi8>, f32, f32, f32, f32, f32, f32) -> (f32, f32, f32, f32, f32) + scf.yield %v0, %v1, %v2, %v3, %v4 : f32, f32, f32, f32, f32 + } + scf.yield %next_g0, %next_g1, %next_g2, %next_g3, %next_g4, %next_d, %next_s, %next_ql, %next_qh : f32, f32, f32, f32, f32, f16, i8, vector<4xi32>, vector<4xi32> + } + %publish_format = config.get @ggml.matmul.vector.publish_format : index + func.call @ggml_matmul_vector_q8_publish(%tokens, %n, %rows_per_group, %row_base, %local_row, %block_lane, %channel_group, %output_accumulation, %output_unary_op, %epsilon, %publish_format, %total_g0, %total_g1, %total_g2, %total_g3, %total_g4, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, index, index, index, index, index, index, index, index, f32, index, f32, f32, f32, f32, f32, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) + func.return +} diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_activation.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_activation.loom index ef58076817a3..67b8c95f67e8 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_activation.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul/vector_activation.loom @@ -1,4 +1,6 @@ -template.decl @ggml.matmul.q6_q8_1_x4.core(%weight_format: index, %native_wave64: i1, %token_capacity: index, %output_capacity: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) +func.decl @ggml_matmul_vector_q6_row64_q8_1_x4_activation_dot(%capacity: index, %weight_format: index, %token_count: index, %input_size0: index, %output_size0: index, %output_accumulation: index, %output_unary_op: index, %epsilon: f32, %channel_group: index, %token_group: index, %input: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer, %next_output: buffer) +func.decl @ggml_q6_q8_native_wave32(%token_capacity: index, %output_capacity: index, %token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) +func.decl @ggml_q6_q8_native_wave64_block4(%token_capacity: index, %output_capacity: index, %token_count: index, %input_size: index, %output_size: index, %q8_input: buffer, %weight: buffer, %output: buffer) func.decl @ggml_q4k_scale_min_from_header(%scale0: i32, %scale1: i32, %scale2: i32, %q4_group: index) -> (i32, i32) @@ -1100,10 +1102,18 @@ template.def<@ggml.matmul.vector_activation.core> device @ggml_matmul_vector_act %uses_q6 = scalar.ori %uses_row64_q6, %uses_native_q6 : i1 scf.if %uses_q6 { %token_capacity = config.get @ggml.workload.token_capacity : index - %wave = target.subgroup.size : index - %c64 = index.constant 64 : index - %native_wave64 = index.cmp eq, %wave, %c64 : index - template.apply<@ggml.matmul.q6_q8_1_x4.core>(%weight_format, %native_wave64, %token_capacity, %output_size0, %token_count, %input_size0, %output_size0, %output_accumulation, %output_unary_op, %epsilon, %channel_group, %token_group, %input, %weight, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, i1, index, index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) + scf.if %uses_row64_q6 { + func.call @ggml_matmul_vector_q6_row64_q8_1_x4_activation_dot(%token_capacity, %weight_format, %token_count, %input_size0, %output_size0, %output_accumulation, %output_unary_op, %epsilon, %channel_group, %token_group, %input, %weight, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) + } else { + %wave = target.subgroup.size : index + %c64 = index.constant 64 : index + %wave64 = index.cmp eq, %wave, %c64 : index + scf.if %wave64 { + func.call @ggml_q6_q8_native_wave64_block4(%token_capacity, %output_size0, %token_count, %input_size0, %output_size0, %input, %weight, %output) : (index, index, index, index, index, buffer, buffer, buffer) + } else { + func.call @ggml_q6_q8_native_wave32(%token_capacity, %output_size0, %token_count, %input_size0, %output_size0, %input, %weight, %output) : (index, index, index, index, index, buffer, buffer, buffer) + } + } } else { func.call @ggml_matmul_vector_q4_q8_1_x4_activation_dot(%weight_format, %token_count, %input_size0, %output_size0, %output_accumulation, %output_unary_op, %epsilon, %channel_group, %token_group, %input, %weight, %bias, %output, %residual_input, %residual_output, %norm_weight, %normalized_output, %completion_counters, %next_output) : (index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) } From 53dcc975f81395950fb970a5e0909786c10d088b Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Mon, 16 Nov 2026 13:22:30 -0800 Subject: [PATCH 16/18] test: restore native Q6/Q8 entry coverage Assisted-by: Codex --- .../kernels/loom-libs/manifest.json | 99 +++++++++++++++++++ .../ops/matmul/vector_q6_q8_native_test.loom | 64 ++++++++++++ .../qwen_moe/ggml/q6_q8_reference_test.loom | 2 - 3 files changed, 163 insertions(+), 2 deletions(-) create mode 100644 ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_q6_q8_native_test.loom diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json index 47628bd230a0..391f54ddc43f 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json @@ -670,6 +670,9 @@ }, { "path": "motifs/matmul/q6_row64_q8_1_x4_decode.loom" + }, + { + "path": "ops/matmul/vector_q6_q8_native_test.loom" } ], "exports": [ @@ -10160,6 +10163,30 @@ "motifs/flash_attention/completion_counter_reduce.loom", "motifs/quantize_q8_1_x4.loom" ] + }, + { + "name": "vector_q6_q8_native_test_linked", + "srcs": [ + "ops/matmul/vector_q6_q8_native_test.loom" + ], + "libraries": [ + "ops/matmul/vector_f32_f32.loom", + "motifs/matmul/vector_activation.loom", + "motifs/matmul/q6_q8_1_x4_decode.loom", + "motifs/matmul/q6_row64_q8_1_x4_decode.loom", + "motifs/quantize_q8_1_x4_group.loom", + "motifs/matmul/vector_decode_pair.loom", + "motifs/matmul/vector_publish.loom", + "motifs/dequant.loom", + "motifs/elementwise/unary_apply.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ] } ], "plan_cases": [ @@ -10203,6 +10230,78 @@ "--output-format=jsonl" ], "source": "ops/rows/get_rows_scale_f32.loom" + }, + { + "name": "vector_q6_q8_wave32_k768_tail_plan_test", + "args": [ + "$(location :vector_q6_q8_native_test_linked)", + "--benchmark=@ggml_mul_mat_vector_q6_q8_wave32_k768_tail_small", + "--config=ggml.workload.token_capacity=1", + "--config=ggml.matmul.vector.input_size=768", + "--config=ggml.matmul.vector.output_size=9", + "--config=ggml.matmul.vector.weight_format=6", + "--config=ggml.matmul.vector.activation_format=9", + "--config=ggml.matmul.vector.output_accumulation=0", + "--config=ggml.matmul.vector.output_unary_op=23", + "--config=ggml.matmul.vector.publish_format=0", + "--dry-run", + "--output-format=jsonl" + ], + "link_module": "vector_q6_q8_native_test_linked" + }, + { + "name": "vector_q6_q8_wave32_k2048_tail_plan_test", + "args": [ + "$(location :vector_q6_q8_native_test_linked)", + "--benchmark=@ggml_mul_mat_vector_q6_q8_wave32_k2048_tail_small", + "--config=ggml.workload.token_capacity=1", + "--config=ggml.matmul.vector.input_size=2048", + "--config=ggml.matmul.vector.output_size=9", + "--config=ggml.matmul.vector.weight_format=6", + "--config=ggml.matmul.vector.activation_format=9", + "--config=ggml.matmul.vector.output_accumulation=0", + "--config=ggml.matmul.vector.output_unary_op=23", + "--config=ggml.matmul.vector.publish_format=0", + "--dry-run", + "--output-format=jsonl" + ], + "link_module": "vector_q6_q8_native_test_linked" + }, + { + "name": "vector_q6_q8_wave64_k768_tail_plan_test", + "args": [ + "$(location :vector_q6_q8_native_test_linked)", + "--benchmark=@ggml_mul_mat_vector_q6_q8_wave64_k768_tail_small", + "--config=ggml.workload.token_capacity=1", + "--config=ggml.matmul.vector.input_size=768", + "--config=ggml.matmul.vector.output_size=9", + "--config=ggml.matmul.vector.weight_format=6", + "--config=ggml.matmul.vector.activation_format=9", + "--config=ggml.matmul.vector.output_accumulation=0", + "--config=ggml.matmul.vector.output_unary_op=23", + "--config=ggml.matmul.vector.publish_format=0", + "--dry-run", + "--output-format=jsonl" + ], + "link_module": "vector_q6_q8_native_test_linked" + }, + { + "name": "vector_q6_q8_wave64_k2048_tail_plan_test", + "args": [ + "$(location :vector_q6_q8_native_test_linked)", + "--benchmark=@ggml_mul_mat_vector_q6_q8_wave64_k2048_tail_small", + "--config=ggml.workload.token_capacity=1", + "--config=ggml.matmul.vector.input_size=2048", + "--config=ggml.matmul.vector.output_size=9", + "--config=ggml.matmul.vector.weight_format=6", + "--config=ggml.matmul.vector.activation_format=9", + "--config=ggml.matmul.vector.output_accumulation=0", + "--config=ggml.matmul.vector.output_unary_op=23", + "--config=ggml.matmul.vector.publish_format=0", + "--dry-run", + "--output-format=jsonl" + ], + "link_module": "vector_q6_q8_native_test_linked" } ], "sources": [], diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_q6_q8_native_test.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_q6_q8_native_test.loom new file mode 100644 index 000000000000..c8a3d34acd30 --- /dev/null +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_q6_q8_native_test.loom @@ -0,0 +1,64 @@ +// Nonzero native Q6/Q8 checks call both production entries with the shared ABI. +// K=768 covers the wave64 four-block tail; nine rows cross the wave32 row tile. + +target.decl @ggml_matmul_vector_gfx11_wave32 + +kernel.decl target(@ggml_matmul_vector_gfx11_wave32) @ggml_mul_mat_vector_q6_q8_f32_wave32(%token_count$0: index) launch(%token_count$1: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) + +target.decl @ggml_matmul_vector_gfx11_wave64 + +kernel.decl target(@ggml_matmul_vector_gfx11_wave64) @ggml_mul_mat_vector_q6_q8_f32_wave64(%token_count$6: index) launch(%token_count$7: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) + +// Q6 bytes 0xaa decode to (42 - 32) * (-86) * f16(0xaaaa). +// Q8 bytes 0x01 decode to 1 * f16(0x0101); the oracle is K times their product. +check.case public @ggml_mul_mat_vector_q6_q8_wave32_k768_tail { + %token_count = check.literal value(1) : index + %input = check.generate.fill value(1) : tensor<1x864xi8> + %weight = check.generate.fill value(-86) : tensor<9x3x210xi8> + %output = check.generate.fill value(-1.0) : tensor<1x9xf32> + %expected = check.generate.fill value(0.52674690261483192) : tensor<1x9xf32> + kernel.launch @ggml_mul_mat_vector_q6_q8_f32_wave32[%token_count](%token_count, %input, %weight, %output, %output) : [index](index, tensor<1x864xi8>, tensor<9x3x210xi8>, tensor<1x9xf32>, tensor<1x9xf32>) + check.expect.approximate actual(%output) expected(%expected) atol(9.9999999999999995e-07) rtol(9.9999999999999995e-07) nan(same) : tensor<1x9xf32>, tensor<1x9xf32> + check.return +} + +check.benchmark<@ggml_mul_mat_vector_q6_q8_wave32_k768_tail> @ggml_mul_mat_vector_q6_q8_wave32_k768_tail_small + +check.case public @ggml_mul_mat_vector_q6_q8_wave32_k2048_tail { + %token_count = check.literal value(1) : index + %input = check.generate.fill value(1) : tensor<1x2304xi8> + %weight = check.generate.fill value(-86) : tensor<9x8x210xi8> + %output = check.generate.fill value(-1.0) : tensor<1x9xf32> + %expected = check.generate.fill value(1.4046584069728851) : tensor<1x9xf32> + kernel.launch @ggml_mul_mat_vector_q6_q8_f32_wave32[%token_count](%token_count, %input, %weight, %output, %output) : [index](index, tensor<1x2304xi8>, tensor<9x8x210xi8>, tensor<1x9xf32>, tensor<1x9xf32>) + check.expect.approximate actual(%output) expected(%expected) atol(9.9999999999999995e-07) rtol(9.9999999999999995e-07) nan(same) : tensor<1x9xf32>, tensor<1x9xf32> + check.return +} + +check.benchmark<@ggml_mul_mat_vector_q6_q8_wave32_k2048_tail> @ggml_mul_mat_vector_q6_q8_wave32_k2048_tail_small + +check.case public @ggml_mul_mat_vector_q6_q8_wave64_k768_tail { + %token_count = check.literal value(1) : index + %input = check.generate.fill value(1) : tensor<1x864xi8> + %weight = check.generate.fill value(-86) : tensor<9x3x210xi8> + %output = check.generate.fill value(-1.0) : tensor<1x9xf32> + %expected = check.generate.fill value(0.52674690261483192) : tensor<1x9xf32> + kernel.launch @ggml_mul_mat_vector_q6_q8_f32_wave64[%token_count](%token_count, %input, %weight, %output, %output) : [index](index, tensor<1x864xi8>, tensor<9x3x210xi8>, tensor<1x9xf32>, tensor<1x9xf32>) + check.expect.approximate actual(%output) expected(%expected) atol(9.9999999999999995e-07) rtol(9.9999999999999995e-07) nan(same) : tensor<1x9xf32>, tensor<1x9xf32> + check.return +} + +check.benchmark<@ggml_mul_mat_vector_q6_q8_wave64_k768_tail> @ggml_mul_mat_vector_q6_q8_wave64_k768_tail_small + +check.case public @ggml_mul_mat_vector_q6_q8_wave64_k2048_tail { + %token_count = check.literal value(1) : index + %input = check.generate.fill value(1) : tensor<1x2304xi8> + %weight = check.generate.fill value(-86) : tensor<9x8x210xi8> + %output = check.generate.fill value(-1.0) : tensor<1x9xf32> + %expected = check.generate.fill value(1.4046584069728851) : tensor<1x9xf32> + kernel.launch @ggml_mul_mat_vector_q6_q8_f32_wave64[%token_count](%token_count, %input, %weight, %output, %output) : [index](index, tensor<1x2304xi8>, tensor<9x8x210xi8>, tensor<1x9xf32>, tensor<1x9xf32>) + check.expect.approximate actual(%output) expected(%expected) atol(9.9999999999999995e-07) rtol(9.9999999999999995e-07) nan(same) : tensor<1x9xf32>, tensor<1x9xf32> + check.return +} + +check.benchmark<@ggml_mul_mat_vector_q6_q8_wave64_k2048_tail> @ggml_mul_mat_vector_q6_q8_wave64_k2048_tail_small diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/q6_q8_reference_test.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/q6_q8_reference_test.loom index 945015794ace..5b128c3ff948 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/q6_q8_reference_test.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/ggml/q6_q8_reference_test.loom @@ -67,6 +67,4 @@ check.case public @ggml_linear_q6k_q8_1_x4_nonzero_tail_case { check.return } -// The gfx1151 schedule has an exact target requirement, so its nonzero packed -// data case remains independently selectable from the generic gfx11 case. check.benchmark<@ggml_linear_q6k_q8_1_x4_nonzero_tail_case> @ggml_linear_q6k_q8_1_x4_small From 0bad06b7ff3819da1ef482e7b3e84139b6f55ddc Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Mon, 16 Nov 2026 14:59:20 -0800 Subject: [PATCH 17/18] hrx: rename general vector matmul entry --- .../common/dispatch-mul-mat.cpp | 10 ++--- .../qwen/dispatch-qwen-matmul.cpp | 6 +-- .../kernels/loom-libs/manifest.json | 4 +- .../loom-libs/ops/matmul/vector_f32_f32.loom | 2 +- tests/hrx-ops-test.cpp | 44 +++++++++---------- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp index ab09d0cde83c..754d8b057349 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp @@ -47,8 +47,8 @@ static constexpr KernelCatalogRef kMulMatF32F32DecodeWave64Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_f32_f32_decode_wave64"); static constexpr KernelCatalogRef kMulMatAddF32F32DecodeWave64Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_add_f32_f32_decode_wave64"); -static constexpr KernelCatalogRef kMulMatVectorF32F32Kernel = - GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_vector_f32_f32"); +static constexpr KernelCatalogRef kMulMatVectorKernel = + GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_vector"); static constexpr KernelCatalogRef kMulMatVectorQ1_0Q8_1X4F32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_vector_q1_0_q8_1_x4_f32"); static constexpr KernelCatalogRef kMulMatVectorQ1_0Q8_1X4Rows2F32Kernel = @@ -1659,7 +1659,7 @@ static bool build_vector_mul_mat_dispatch(const DispatchMatchContext & context, return false; } dispatch.kernel = make_kernel_specialization( - use_iq_codebook ? vector_iq_codebook_kernel(match.weight_format) : kMulMatVectorF32F32Kernel); + use_iq_codebook ? vector_iq_codebook_kernel(match.weight_format) : kMulMatVectorKernel); dispatch.kernel.integer_parameters.emplace("token_count", match.token_count); dispatch.kernel.compile_parameters.emplace("ggml.workload.token_capacity", common_to_config_value(match.token_count)); @@ -1805,7 +1805,7 @@ static bool build_vector_mul_mat_postops_dispatch(const DispatchMatchContext & c static bool match_vector_mul_mat_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) { CommonMulMatMatch match = - common_match_mul_mat_any_format(context.graph, context.root_node, kMulMatVectorF32F32Kernel, true); + common_match_mul_mat_any_format(context.graph, context.root_node, kMulMatVectorKernel, true); if (!match.matched() || match.token_count != 1) { return false; } @@ -1814,7 +1814,7 @@ static bool match_vector_mul_mat_dispatch(const DispatchMatchContext & context, static bool match_vector_mul_mat_unary_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) { CommonMulMatMatch match = - common_match_mul_mat_any_format(context.graph, context.root_node, kMulMatVectorF32F32Kernel, true); + common_match_mul_mat_any_format(context.graph, context.root_node, kMulMatVectorKernel, true); if (!match.matched() || match.token_count != 1 || !try_match_fused_unary(context, match)) { return false; } diff --git a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp index 70254ec132d1..56b5d73c0534 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp @@ -18,8 +18,8 @@ static constexpr KernelCatalogRef kQwenDenseLinearQ4KQ8NextQ8Kernel = GGML_HRX_KERNEL_REF("qwen3_moe", "qwen3_moe_dense_linear_q4k_q8_1_x4_next_q8"); static constexpr KernelCatalogRef kMulMatVectorQ6Q8F32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_vector_q6_q8_f32"); -static constexpr KernelCatalogRef kMulMatVectorF32F32Kernel = - GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_vector_f32_f32"); +static constexpr KernelCatalogRef kMulMatVectorKernel = + GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_vector"); static constexpr int64_t kQwenHiddenSize = kQwen30BMoeDispatchProfile.hidden_size; static constexpr int64_t kQwenVocabularyCount = 151936; @@ -346,7 +346,7 @@ Dispatch make_qwen_q6k_q8_projection_dispatch(const Value & weight, const Value dispatch_weight_layout_eligible(weight); Dispatch dispatch; dispatch.kernel = - make_kernel_specialization(row64_weight ? kMulMatVectorF32F32Kernel : kMulMatVectorQ6Q8F32Kernel); + make_kernel_specialization(row64_weight ? kMulMatVectorKernel : kMulMatVectorQ6Q8F32Kernel); dispatch.kernel.integer_parameters.emplace("token_count", 1); dispatch.kernel.compile_parameters.emplace("ggml.workload.token_capacity", "1"); dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.input_size", to_config_value(input_size)); diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json index 391f54ddc43f..4699dcc52bd1 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json @@ -2707,9 +2707,9 @@ ] }, { - "name": "ggml_mul_mat_vector_f32_f32", + "name": "ggml_mul_mat_vector", "family": "loom_libs", - "symbol": "ggml_mul_mat_vector_f32_f32", + "symbol": "ggml_mul_mat_vector", "source": "ops/matmul/vector_f32_f32.loom", "workload_parameters": [ { diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_f32_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_f32_f32.loom index a5b2074a36ff..300d38401116 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_f32_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_f32_f32.loom @@ -119,7 +119,7 @@ template.def<@ggml.matmul.vector_f32.entry> device @ggml_matmul_vector_f32_entry template.return } -kernel.def target(@ggml_matmul_vector_gfx11_wave64) @ggml_mul_mat_vector_f32_f32(%token_count: index) { +kernel.def target(@ggml_matmul_vector_gfx11_wave64) @ggml_mul_mat_vector(%token_count: index) { %input_size = config.get @ggml.matmul.vector.input_size : index %output_size = config.get @ggml.matmul.vector.output_size : index %token_capacity = config.get @ggml.workload.token_capacity : index diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index 198c567bd49f..2f60f10ce6fa 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -7189,8 +7189,8 @@ static void run_vector_q8_publish_cpu_reference_case() { ggml_build_forward_expand(hrx_graph, hrx_output); require_kernel_subsequence(scheduled_kernel_sequence(hrx_graph), - { "loom_libs:ggml_mul_mat_vector_f32_f32", "qwen3_moe:ggml_quantize_q8_1_x4_f32", - "loom_libs:ggml_mul_mat_vector_f32_f32" }); + { "loom_libs:ggml_mul_mat_vector", "qwen3_moe:ggml_quantize_q8_1_x4_f32", + "loom_libs:ggml_mul_mat_vector" }); ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*hrx_graph); REQUIRE(imported.valid()); @@ -7210,7 +7210,7 @@ static void run_vector_q8_publish_cpu_reference_case() { const ggml::hrx::Dispatch * second_vector = nullptr; int vector_count = 0; for (const ggml::hrx::Dispatch & dispatch : plan.dispatches) { - if (kernel_name_for_id(dispatch.kernel.kernel_id) == "loom_libs:ggml_mul_mat_vector_f32_f32") { + if (kernel_name_for_id(dispatch.kernel.kernel_id) == "loom_libs:ggml_mul_mat_vector") { ++vector_count; if (vector_count == 2) { second_vector = &dispatch; @@ -7523,7 +7523,7 @@ static void run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(bool prefer_row64 = f require_kernel_subsequence( scheduled_kernel_sequence(hrx_graph, { "gfx1151", prefer_row64 }), { "qwen3_moe:qwen3_moe_rmsnorm_f32_quantize_q8_1_x4", - prefer_row64 ? "loom_libs:ggml_mul_mat_vector_f32_f32" : "loom_libs:ggml_mul_mat_vector_q6_q8_f32" }); + prefer_row64 ? "loom_libs:ggml_mul_mat_vector" : "loom_libs:ggml_mul_mat_vector_q6_q8_f32" }); } else { require_kernel_subsequence( scheduled_kernel_sequence(hrx_graph, { "gfx1151", prefer_row64 }), @@ -9202,7 +9202,7 @@ static void run_arch_decode_native_f32_route_case(llm_arch arch, ggml_type type, } REQUIRE(binding.layout == ggml::hrx::kNativeWeightLayout); REQUIRE(kernel_name_for_id(dispatch.kernel.kernel_id, target) == - "loom_libs:ggml_mul_mat_vector_f32_f32"); + "loom_libs:ggml_mul_mat_vector"); const auto format = dispatch.kernel.compile_parameters.find("ggml.matmul.vector.weight_format"); REQUIRE(format != dispatch.kernel.compile_parameters.end()); REQUIRE(format->second == (type == GGML_TYPE_Q4_K ? "4" : "6")); @@ -9331,7 +9331,7 @@ static std::vector run_dense_matmul_f32_reference_case(ggml_type type, in const bool aligned_prefill = prefill_minimum && prefill_aligned; const bool aligned_q6_prefill = q6_weight && aligned_prefill; const bool short_m64 = short_q6 && n == 1024 && (k == 1024 || k == 3072); - const char * expected_kernel = tokens == 1 ? "loom_libs:ggml_mul_mat_vector_f32_f32" : + const char * expected_kernel = tokens == 1 ? "loom_libs:ggml_mul_mat_vector" : short_m64 ? "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32" : short_q6 ? "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_wave32" : aligned_q6_prefill ? "loom_libs:ggml_mul_mat_q6_k_f16_wmma_prefill_wave32" : @@ -9451,13 +9451,13 @@ static void run_vector_output_capacity_experiment() { constexpr int64_t kAboveOldOutputSize = 262208; constexpr int64_t kLiftedOutputSize = 524288; - run_dense_matmul_cpu_reference_case(GGML_TYPE_F32, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, kAboveOldOutputSize, + run_dense_matmul_cpu_reference_case(GGML_TYPE_F32, "loom_libs:ggml_mul_mat_vector", 1, kAboveOldOutputSize, kSmallVectorInputSize); - run_dense_matmul_unary_cpu_reference_case(GGML_TYPE_F32, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, + run_dense_matmul_unary_cpu_reference_case(GGML_TYPE_F32, "loom_libs:ggml_mul_mat_vector", 1, kAboveOldOutputSize, kSmallVectorInputSize); run_dense_matmul_postops_cpu_reference_case(GGML_TYPE_F16, "loom_libs:ggml_mul_mat_vector_bias_residual_f32_f32", 1, kAboveOldOutputSize, true, false, false, kSmallVectorInputSize); - run_dense_matmul_cpu_reference_case(GGML_TYPE_F16, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, kLiftedOutputSize, + run_dense_matmul_cpu_reference_case(GGML_TYPE_F16, "loom_libs:ggml_mul_mat_vector", 1, kLiftedOutputSize, kSmallVectorInputSize); } @@ -9727,7 +9727,7 @@ static void register_dense_matmul_cases(Suite & suite) { for (const int64_t input_size : { 256, 384, 768 }) { suite.device_case("dense_matmul.q1_0.vector_tail.k" + std::to_string(input_size), [input_size] { run_dense_matmul_cpu_reference_case( - GGML_TYPE_Q1_0, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, 65, input_size); + GGML_TYPE_Q1_0, "loom_libs:ggml_mul_mat_vector", 1, 65, input_size); }); } @@ -9753,7 +9753,7 @@ static void register_dense_matmul_cases(Suite & suite) { [tokens, outputs] { run_dense_matmul_cpu_reference_case(GGML_TYPE_Q4_K, tokens == 1 ? - "loom_libs:ggml_mul_mat_vector_f32_f32" : + "loom_libs:ggml_mul_mat_vector" : "loom_libs:ggml_mul_mat_skinny_input_f32_publish_f32", tokens, outputs); }); @@ -9782,7 +9782,7 @@ static void register_dense_matmul_cases(Suite & suite) { 128); }); suite.device_case("dense_matmul.q4_k.vector.tokens1.outputs128", [] { - run_dense_matmul_cpu_reference_case(GGML_TYPE_Q4_K, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, 128); + run_dense_matmul_cpu_reference_case(GGML_TYPE_Q4_K, "loom_libs:ggml_mul_mat_vector", 1, 128); }); for (const ggml_type type : { GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS }) { suite.device_case("dense_matmul." + type_name(type) + ".codebook.vector.tokens1.outputs128", [type] { @@ -9850,7 +9850,7 @@ static void register_dense_matmul_cases(Suite & suite) { }); } suite.device_case("dense_matmul.iq4_xs.vector.tokens1.outputs128", [] { - run_dense_matmul_cpu_reference_case(GGML_TYPE_IQ4_XS, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, 128); + run_dense_matmul_cpu_reference_case(GGML_TYPE_IQ4_XS, "loom_libs:ggml_mul_mat_vector", 1, 128); }); suite.device_case("dense_matmul.iq4_xs.tiled.tokens64.outputs128", [] { run_dense_matmul_cpu_reference_case(GGML_TYPE_IQ4_XS, @@ -9882,7 +9882,7 @@ static void register_dense_matmul_cases(Suite & suite) { { GGML_TYPE_Q1_0, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_IQ2_S, GGML_TYPE_Q6_K, GGML_TYPE_Q8_0, GGML_TYPE_IQ4_NL, GGML_TYPE_F16, GGML_TYPE_BF16 }) { const char * vector_kernel = type == GGML_TYPE_IQ2_S ? "loom_libs:ggml_mul_mat_vector_iq2_s_f32_f32" : - "loom_libs:ggml_mul_mat_vector_f32_f32"; + "loom_libs:ggml_mul_mat_vector"; const int64_t outputs = type == GGML_TYPE_IQ2_S ? 640 : (type == GGML_TYPE_IQ4_NL ? 1024 : 128); const int64_t input = type == GGML_TYPE_IQ4_NL ? 640 : kQwenHiddenSize; const bool tiled_route = type == GGML_TYPE_IQ2_S || type == GGML_TYPE_IQ4_NL; @@ -9912,20 +9912,20 @@ static void register_dense_matmul_cases(Suite & suite) { 128, 544); }); suite.device_case("dense_matmul.q5_0.vector.tokens1.outputs256.input640", [] { - run_dense_matmul_cpu_reference_case(GGML_TYPE_Q5_0, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, 256, 640); + run_dense_matmul_cpu_reference_case(GGML_TYPE_Q5_0, "loom_libs:ggml_mul_mat_vector", 1, 256, 640); }); suite.device_case("dense_matmul.q8_1.zero_weight.skinny", [] { run_dense_matmul_zero_weight_case(GGML_TYPE_Q8_1, "loom_libs:ggml_mul_mat_skinny_input_f32_publish_f32"); }); suite.device_case("dense_matmul.q8_1.zero_weight.vector", [] { - run_dense_matmul_zero_weight_case(GGML_TYPE_Q8_1, "loom_libs:ggml_mul_mat_vector_f32_f32", 1); + run_dense_matmul_zero_weight_case(GGML_TYPE_Q8_1, "loom_libs:ggml_mul_mat_vector", 1); }); suite.device_case("dense_matmul.f32.skinny.tokens2.outputs256", [] { run_dense_matmul_cpu_reference_case(GGML_TYPE_F32, "loom_libs:ggml_mul_mat_skinny_input_f32_publish_f32", 2, 256); }); suite.device_case("dense_matmul.f32.vector.tokens1.outputs256", [] { - run_dense_matmul_cpu_reference_case(GGML_TYPE_F32, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, 256); + run_dense_matmul_cpu_reference_case(GGML_TYPE_F32, "loom_libs:ggml_mul_mat_vector", 1, 256); }); suite.device_case("dense_matmul_unary.f32.tokens2.outputs128", [] { run_dense_matmul_unary_cpu_reference_case(GGML_TYPE_F32, "loom_libs:ggml_mul_mat_skinny_input_f32_publish_f32", @@ -10221,24 +10221,24 @@ static void register_dense_matmul_cases(Suite & suite) { suite.device_case( "dense_matmul.q4_k.contiguous.tokens" + std::to_string(tokens) + ".outputs256.input5120", [tokens] { run_dense_matmul_cpu_reference_case(GGML_TYPE_Q4_K, - tokens == 1 ? "loom_libs:ggml_mul_mat_vector_f32_f32" : + tokens == 1 ? "loom_libs:ggml_mul_mat_vector" : "loom_libs:ggml_mul_mat_skinny_input_f32_publish_f32", tokens, 256, 5120, true); }); suite.device_case( "dense_matmul.q6_k.contiguous.tokens" + std::to_string(tokens) + ".outputs256.input5120", [tokens] { run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, - tokens == 1 ? "loom_libs:ggml_mul_mat_vector_f32_f32" : + tokens == 1 ? "loom_libs:ggml_mul_mat_vector" : "loom_libs:ggml_mul_mat_skinny_input_f32_publish_f32", tokens, 256, 5120, true, true); }); } suite.device_case("dense_matmul.q6_k.vector_q6.tokens1.outputs256.input5120", [] { - run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, 256, 5120, + run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_vector", 1, 256, 5120, true); }); suite.device_case("dense_matmul.q6_k.vector_q6.tokens1.outputs320.input512", [] { - run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, 320, 512, + run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_vector", 1, 320, 512, true); }); } @@ -10368,7 +10368,7 @@ static void register_rmsnorm_and_scheduling_cases(Suite & suite) { }); suite.device_case("q4_prefill_source_eligibility", [] { run_q4_prefill_source_eligibility_case(); }); suite.device_case("uploaded_native_q6_vector", [] { - run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, 128, + run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_vector", 1, 128, 2048, false, false, GGML_TYPE_COUNT, true); }); suite.device_case("uploaded_native_gated_q4", [] { From 66353815bf4a823f08f679957585242acfe7a96c Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Tue, 17 Nov 2026 08:23:06 -0800 Subject: [PATCH 18/18] hrx: preserve existing vector kernel names Restore the general vector and native Q6/Q8 public identities while retaining the reusable motifs, common endpoint ABI, and launch schedules. Update catalog references and existing tests together. --- .../common/dispatch-mul-mat.cpp | 10 ++--- .../qwen/dispatch-qwen-matmul.cpp | 12 ++--- .../kernels/loom-libs/manifest.json | 16 +++---- .../loom-libs/ops/matmul/vector_f32_f32.loom | 6 +-- .../ops/matmul/vector_q6_q8_native_test.loom | 12 ++--- tests/hrx-backend-test.cpp | 8 ++-- tests/hrx-loom-jit-test.cpp | 2 +- tests/hrx-ops-test.cpp | 44 +++++++++---------- 8 files changed, 55 insertions(+), 55 deletions(-) diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp index 754d8b057349..ab09d0cde83c 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp @@ -47,8 +47,8 @@ static constexpr KernelCatalogRef kMulMatF32F32DecodeWave64Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_f32_f32_decode_wave64"); static constexpr KernelCatalogRef kMulMatAddF32F32DecodeWave64Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_add_f32_f32_decode_wave64"); -static constexpr KernelCatalogRef kMulMatVectorKernel = - GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_vector"); +static constexpr KernelCatalogRef kMulMatVectorF32F32Kernel = + GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_vector_f32_f32"); static constexpr KernelCatalogRef kMulMatVectorQ1_0Q8_1X4F32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_vector_q1_0_q8_1_x4_f32"); static constexpr KernelCatalogRef kMulMatVectorQ1_0Q8_1X4Rows2F32Kernel = @@ -1659,7 +1659,7 @@ static bool build_vector_mul_mat_dispatch(const DispatchMatchContext & context, return false; } dispatch.kernel = make_kernel_specialization( - use_iq_codebook ? vector_iq_codebook_kernel(match.weight_format) : kMulMatVectorKernel); + use_iq_codebook ? vector_iq_codebook_kernel(match.weight_format) : kMulMatVectorF32F32Kernel); dispatch.kernel.integer_parameters.emplace("token_count", match.token_count); dispatch.kernel.compile_parameters.emplace("ggml.workload.token_capacity", common_to_config_value(match.token_count)); @@ -1805,7 +1805,7 @@ static bool build_vector_mul_mat_postops_dispatch(const DispatchMatchContext & c static bool match_vector_mul_mat_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) { CommonMulMatMatch match = - common_match_mul_mat_any_format(context.graph, context.root_node, kMulMatVectorKernel, true); + common_match_mul_mat_any_format(context.graph, context.root_node, kMulMatVectorF32F32Kernel, true); if (!match.matched() || match.token_count != 1) { return false; } @@ -1814,7 +1814,7 @@ static bool match_vector_mul_mat_dispatch(const DispatchMatchContext & context, static bool match_vector_mul_mat_unary_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) { CommonMulMatMatch match = - common_match_mul_mat_any_format(context.graph, context.root_node, kMulMatVectorKernel, true); + common_match_mul_mat_any_format(context.graph, context.root_node, kMulMatVectorF32F32Kernel, true); if (!match.matched() || match.token_count != 1 || !try_match_fused_unary(context, match)) { return false; } diff --git a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp index 56b5d73c0534..bba2f27a5648 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-matmul.cpp @@ -16,10 +16,10 @@ namespace { static constexpr KernelCatalogRef kQwenDenseLinearQ4KQ8NextQ8Kernel = GGML_HRX_KERNEL_REF("qwen3_moe", "qwen3_moe_dense_linear_q4k_q8_1_x4_next_q8"); -static constexpr KernelCatalogRef kMulMatVectorQ6Q8F32Kernel = - GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_vector_q6_q8_f32"); -static constexpr KernelCatalogRef kMulMatVectorKernel = - GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_vector"); +static constexpr KernelCatalogRef kGgmlLinearQ6KQ8_1X4Kernel = + GGML_HRX_KERNEL_REF("qwen3_moe", "ggml_linear_q6k_q8_1_x4"); +static constexpr KernelCatalogRef kMulMatVectorF32F32Kernel = + GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_vector_f32_f32"); static constexpr int64_t kQwenHiddenSize = kQwen30BMoeDispatchProfile.hidden_size; static constexpr int64_t kQwenVocabularyCount = 151936; @@ -193,7 +193,7 @@ static QwenMatmulMatch match_qwen_q6k_q8_matmul(const Graph & graph, const Graph match.output = output; match.input_value = alternate->alternate_value; match.input_bytes = alternate->byte_count; - match.kernel = kMulMatVectorQ6Q8F32Kernel; + match.kernel = kGgmlLinearQ6KQ8_1X4Kernel; match.input_size = input_size; match.output_size = output_size; match.token_count = token_count; @@ -346,7 +346,7 @@ Dispatch make_qwen_q6k_q8_projection_dispatch(const Value & weight, const Value dispatch_weight_layout_eligible(weight); Dispatch dispatch; dispatch.kernel = - make_kernel_specialization(row64_weight ? kMulMatVectorKernel : kMulMatVectorQ6Q8F32Kernel); + make_kernel_specialization(row64_weight ? kMulMatVectorF32F32Kernel : kGgmlLinearQ6KQ8_1X4Kernel); dispatch.kernel.integer_parameters.emplace("token_count", 1); dispatch.kernel.compile_parameters.emplace("ggml.workload.token_capacity", "1"); dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.input_size", to_config_value(input_size)); diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json index 4699dcc52bd1..4ff469eeeebb 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json @@ -2707,9 +2707,9 @@ ] }, { - "name": "ggml_mul_mat_vector", + "name": "ggml_mul_mat_vector_f32_f32", "family": "loom_libs", - "symbol": "ggml_mul_mat_vector", + "symbol": "ggml_mul_mat_vector_f32_f32", "source": "ops/matmul/vector_f32_f32.loom", "workload_parameters": [ { @@ -10006,9 +10006,9 @@ ] }, { - "name": "ggml_mul_mat_vector_q6_q8_f32", - "family": "loom_libs", - "symbol": "ggml_mul_mat_vector_q6_q8_f32_wave32", + "name": "ggml_linear_q6k_q8_1_x4", + "family": "qwen3_moe", + "symbol": "ggml_linear_q6k_q8_1_x4", "source": "ops/matmul/vector_f32_f32.loom", "workload_parameters": [ { @@ -10077,9 +10077,9 @@ ] }, { - "name": "ggml_mul_mat_vector_q6_q8_f32", - "family": "loom_libs", - "symbol": "ggml_mul_mat_vector_q6_q8_f32_wave64", + "name": "ggml_linear_q6k_q8_1_x4", + "family": "qwen3_moe", + "symbol": "ggml_linear_q6k_q8_1_x4_wave64_block4", "source": "ops/matmul/vector_f32_f32.loom", "workload_parameters": [ { diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_f32_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_f32_f32.loom index 300d38401116..77f0f80fa291 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_f32_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_f32_f32.loom @@ -119,7 +119,7 @@ template.def<@ggml.matmul.vector_f32.entry> device @ggml_matmul_vector_f32_entry template.return } -kernel.def target(@ggml_matmul_vector_gfx11_wave64) @ggml_mul_mat_vector(%token_count: index) { +kernel.def target(@ggml_matmul_vector_gfx11_wave64) @ggml_mul_mat_vector_f32_f32(%token_count: index) { %input_size = config.get @ggml.matmul.vector.input_size : index %output_size = config.get @ggml.matmul.vector.output_size : index %token_capacity = config.get @ggml.workload.token_capacity : index @@ -134,7 +134,7 @@ kernel.def target(@ggml_matmul_vector_gfx11_wave64) @ggml_mul_mat_vector(%token_ // Native Q6/Q8 retains eight wave32 rows per workgroup on gfx1100. amdgpu.target @ggml_matmul_vector_gfx11_wave32 {subgroup_size = 32} -kernel.def target(@ggml_matmul_vector_gfx11_wave32) export("ggml_mul_mat_vector_q6_q8_f32") @ggml_mul_mat_vector_q6_q8_f32_wave32(%token_count: index) { +kernel.def target(@ggml_matmul_vector_gfx11_wave32) export("ggml_linear_q6k_q8_1_x4") @ggml_linear_q6k_q8_1_x4(%token_count: index) { %output_size = config.get @ggml.matmul.vector.output_size : index %token_capacity = config.get @ggml.workload.token_capacity : index %c1 = index.constant 1 : index @@ -150,7 +150,7 @@ kernel.def target(@ggml_matmul_vector_gfx11_wave32) export("ggml_mul_mat_vector_ } // The catalog export name is shared with wave32; both use the common vector ABI. -kernel.def target(@ggml_matmul_vector_gfx11_wave64) export("ggml_mul_mat_vector_q6_q8_f32") @ggml_mul_mat_vector_q6_q8_f32_wave64(%token_count: index) { +kernel.def target(@ggml_matmul_vector_gfx11_wave64) export("ggml_linear_q6k_q8_1_x4") @ggml_linear_q6k_q8_1_x4_wave64_block4(%token_count: index) { %output_size = config.get @ggml.matmul.vector.output_size : index %token_capacity = config.get @ggml.workload.token_capacity : index %c1 = index.constant 1 : index diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_q6_q8_native_test.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_q6_q8_native_test.loom index c8a3d34acd30..b8ca3a7b0937 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_q6_q8_native_test.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul/vector_q6_q8_native_test.loom @@ -3,11 +3,11 @@ target.decl @ggml_matmul_vector_gfx11_wave32 -kernel.decl target(@ggml_matmul_vector_gfx11_wave32) @ggml_mul_mat_vector_q6_q8_f32_wave32(%token_count$0: index) launch(%token_count$1: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) +kernel.decl target(@ggml_matmul_vector_gfx11_wave32) @ggml_linear_q6k_q8_1_x4(%token_count$0: index) launch(%token_count$1: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) target.decl @ggml_matmul_vector_gfx11_wave64 -kernel.decl target(@ggml_matmul_vector_gfx11_wave64) @ggml_mul_mat_vector_q6_q8_f32_wave64(%token_count$6: index) launch(%token_count$7: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) +kernel.decl target(@ggml_matmul_vector_gfx11_wave64) @ggml_linear_q6k_q8_1_x4_wave64_block4(%token_count$6: index) launch(%token_count$7: index, %input: buffer, %weight: buffer, %output: buffer, %next_output: buffer) // Q6 bytes 0xaa decode to (42 - 32) * (-86) * f16(0xaaaa). // Q8 bytes 0x01 decode to 1 * f16(0x0101); the oracle is K times their product. @@ -17,7 +17,7 @@ check.case public @ggml_mul_mat_vector_q6_q8_wave32_k768_tail { %weight = check.generate.fill value(-86) : tensor<9x3x210xi8> %output = check.generate.fill value(-1.0) : tensor<1x9xf32> %expected = check.generate.fill value(0.52674690261483192) : tensor<1x9xf32> - kernel.launch @ggml_mul_mat_vector_q6_q8_f32_wave32[%token_count](%token_count, %input, %weight, %output, %output) : [index](index, tensor<1x864xi8>, tensor<9x3x210xi8>, tensor<1x9xf32>, tensor<1x9xf32>) + kernel.launch @ggml_linear_q6k_q8_1_x4[%token_count](%token_count, %input, %weight, %output, %output) : [index](index, tensor<1x864xi8>, tensor<9x3x210xi8>, tensor<1x9xf32>, tensor<1x9xf32>) check.expect.approximate actual(%output) expected(%expected) atol(9.9999999999999995e-07) rtol(9.9999999999999995e-07) nan(same) : tensor<1x9xf32>, tensor<1x9xf32> check.return } @@ -30,7 +30,7 @@ check.case public @ggml_mul_mat_vector_q6_q8_wave32_k2048_tail { %weight = check.generate.fill value(-86) : tensor<9x8x210xi8> %output = check.generate.fill value(-1.0) : tensor<1x9xf32> %expected = check.generate.fill value(1.4046584069728851) : tensor<1x9xf32> - kernel.launch @ggml_mul_mat_vector_q6_q8_f32_wave32[%token_count](%token_count, %input, %weight, %output, %output) : [index](index, tensor<1x2304xi8>, tensor<9x8x210xi8>, tensor<1x9xf32>, tensor<1x9xf32>) + kernel.launch @ggml_linear_q6k_q8_1_x4[%token_count](%token_count, %input, %weight, %output, %output) : [index](index, tensor<1x2304xi8>, tensor<9x8x210xi8>, tensor<1x9xf32>, tensor<1x9xf32>) check.expect.approximate actual(%output) expected(%expected) atol(9.9999999999999995e-07) rtol(9.9999999999999995e-07) nan(same) : tensor<1x9xf32>, tensor<1x9xf32> check.return } @@ -43,7 +43,7 @@ check.case public @ggml_mul_mat_vector_q6_q8_wave64_k768_tail { %weight = check.generate.fill value(-86) : tensor<9x3x210xi8> %output = check.generate.fill value(-1.0) : tensor<1x9xf32> %expected = check.generate.fill value(0.52674690261483192) : tensor<1x9xf32> - kernel.launch @ggml_mul_mat_vector_q6_q8_f32_wave64[%token_count](%token_count, %input, %weight, %output, %output) : [index](index, tensor<1x864xi8>, tensor<9x3x210xi8>, tensor<1x9xf32>, tensor<1x9xf32>) + kernel.launch @ggml_linear_q6k_q8_1_x4_wave64_block4[%token_count](%token_count, %input, %weight, %output, %output) : [index](index, tensor<1x864xi8>, tensor<9x3x210xi8>, tensor<1x9xf32>, tensor<1x9xf32>) check.expect.approximate actual(%output) expected(%expected) atol(9.9999999999999995e-07) rtol(9.9999999999999995e-07) nan(same) : tensor<1x9xf32>, tensor<1x9xf32> check.return } @@ -56,7 +56,7 @@ check.case public @ggml_mul_mat_vector_q6_q8_wave64_k2048_tail { %weight = check.generate.fill value(-86) : tensor<9x8x210xi8> %output = check.generate.fill value(-1.0) : tensor<1x9xf32> %expected = check.generate.fill value(1.4046584069728851) : tensor<1x9xf32> - kernel.launch @ggml_mul_mat_vector_q6_q8_f32_wave64[%token_count](%token_count, %input, %weight, %output, %output) : [index](index, tensor<1x2304xi8>, tensor<9x8x210xi8>, tensor<1x9xf32>, tensor<1x9xf32>) + kernel.launch @ggml_linear_q6k_q8_1_x4_wave64_block4[%token_count](%token_count, %input, %weight, %output, %output) : [index](index, tensor<1x2304xi8>, tensor<9x8x210xi8>, tensor<1x9xf32>, tensor<1x9xf32>) check.expect.approximate actual(%output) expected(%expected) atol(9.9999999999999995e-07) rtol(9.9999999999999995e-07) nan(same) : tensor<1x9xf32>, tensor<1x9xf32> check.return } diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index 40d1073e163b..72b4c17a05d6 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -4004,7 +4004,7 @@ static void schedule_qwen_terminal_q6k_q8_command(int64_t token_count) { REQUIRE(rms_dispatch.bindings[3].length == q8_transient.size); const ggml::hrx::Dispatch & vocab_dispatch = scheduler.plan().dispatches[1]; - REQUIRE(kernel_name_for_id(vocab_dispatch.kernel.kernel_id) == "loom_libs:ggml_mul_mat_vector_q6_q8_f32"); + REQUIRE(kernel_name_for_id(vocab_dispatch.kernel.kernel_id) == "qwen3_moe:ggml_linear_q6k_q8_1_x4"); REQUIRE(vocab_dispatch.kernel.integer_parameters.at("token_count") == token_count); require_compile_parameter(vocab_dispatch, "ggml.matmul.vector.input_size", "2048"); require_compile_parameter(vocab_dispatch, "ggml.matmul.vector.output_size", "151936"); @@ -4284,8 +4284,8 @@ static void schedule_get_rows_q8_1_alternate_command(ggml_type embedding_weight_ const ggml::hrx::Dispatch & vocab_dispatch = scheduler.plan().dispatches[1]; const std::string vocab_kernel_name = kernel_name_for_id(vocab_dispatch.kernel.kernel_id); REQUIRE(vocab_dispatch.bindings.size() == - (vocab_kernel_name == "loom_libs:ggml_mul_mat_vector_q6_q8_f32" ? 4 : 3)); - if (vocab_kernel_name == "loom_libs:ggml_mul_mat_vector_q6_q8_f32") { + (vocab_kernel_name == "qwen3_moe:ggml_linear_q6k_q8_1_x4" ? 4 : 3)); + if (vocab_kernel_name == "qwen3_moe:ggml_linear_q6k_q8_1_x4") { REQUIRE(vocab_dispatch.bindings[0].value == alternate->alternate_value); REQUIRE(vocab_dispatch.bindings[0].length == alternate->byte_count); } else { @@ -4305,7 +4305,7 @@ static void schedule_get_rows_q8_1_alternate_command(ggml_type embedding_weight_ REQUIRE(commands.commands[0].bindings[3].name == "next_output"); REQUIRE(commands.commands[0].bindings[3].origin == ggml::hrx::CommandBindingOrigin::Transient); REQUIRE(commands.commands[1].bindings.size() == - (vocab_kernel_name == "loom_libs:ggml_mul_mat_vector_q6_q8_f32" ? 4 : 3)); + (vocab_kernel_name == "qwen3_moe:ggml_linear_q6k_q8_1_x4" ? 4 : 3)); REQUIRE(commands.commands[1].bindings[0].name == "input"); ggml_free(ctx); diff --git a/tests/hrx-loom-jit-test.cpp b/tests/hrx-loom-jit-test.cpp index 4beaf2eb9f6a..e815c5a126dd 100644 --- a/tests/hrx-loom-jit-test.cpp +++ b/tests/hrx-loom-jit-test.cpp @@ -542,7 +542,7 @@ static void run_dynamic_export_reuse_case( static void run_targeted_export_materialize_case(HrxTestDevice & device) { const ggml::hrx::KernelDefinition * definition = - find_targeted_kernel("ggml_mul_mat_vector_q6_q8_f32", device.architecture.c_str()); + find_targeted_kernel("ggml_linear_q6k_q8_1_x4", device.architecture.c_str()); if (definition == nullptr) { return; } diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index 2f60f10ce6fa..9b5008884bcd 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -7189,8 +7189,8 @@ static void run_vector_q8_publish_cpu_reference_case() { ggml_build_forward_expand(hrx_graph, hrx_output); require_kernel_subsequence(scheduled_kernel_sequence(hrx_graph), - { "loom_libs:ggml_mul_mat_vector", "qwen3_moe:ggml_quantize_q8_1_x4_f32", - "loom_libs:ggml_mul_mat_vector" }); + { "loom_libs:ggml_mul_mat_vector_f32_f32", "qwen3_moe:ggml_quantize_q8_1_x4_f32", + "loom_libs:ggml_mul_mat_vector_f32_f32" }); ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*hrx_graph); REQUIRE(imported.valid()); @@ -7210,7 +7210,7 @@ static void run_vector_q8_publish_cpu_reference_case() { const ggml::hrx::Dispatch * second_vector = nullptr; int vector_count = 0; for (const ggml::hrx::Dispatch & dispatch : plan.dispatches) { - if (kernel_name_for_id(dispatch.kernel.kernel_id) == "loom_libs:ggml_mul_mat_vector") { + if (kernel_name_for_id(dispatch.kernel.kernel_id) == "loom_libs:ggml_mul_mat_vector_f32_f32") { ++vector_count; if (vector_count == 2) { second_vector = &dispatch; @@ -7523,7 +7523,7 @@ static void run_endpoint_rmsnorm_q6k_q8_cpu_reference_case(bool prefer_row64 = f require_kernel_subsequence( scheduled_kernel_sequence(hrx_graph, { "gfx1151", prefer_row64 }), { "qwen3_moe:qwen3_moe_rmsnorm_f32_quantize_q8_1_x4", - prefer_row64 ? "loom_libs:ggml_mul_mat_vector" : "loom_libs:ggml_mul_mat_vector_q6_q8_f32" }); + prefer_row64 ? "loom_libs:ggml_mul_mat_vector_f32_f32" : "qwen3_moe:ggml_linear_q6k_q8_1_x4" }); } else { require_kernel_subsequence( scheduled_kernel_sequence(hrx_graph, { "gfx1151", prefer_row64 }), @@ -9202,7 +9202,7 @@ static void run_arch_decode_native_f32_route_case(llm_arch arch, ggml_type type, } REQUIRE(binding.layout == ggml::hrx::kNativeWeightLayout); REQUIRE(kernel_name_for_id(dispatch.kernel.kernel_id, target) == - "loom_libs:ggml_mul_mat_vector"); + "loom_libs:ggml_mul_mat_vector_f32_f32"); const auto format = dispatch.kernel.compile_parameters.find("ggml.matmul.vector.weight_format"); REQUIRE(format != dispatch.kernel.compile_parameters.end()); REQUIRE(format->second == (type == GGML_TYPE_Q4_K ? "4" : "6")); @@ -9331,7 +9331,7 @@ static std::vector run_dense_matmul_f32_reference_case(ggml_type type, in const bool aligned_prefill = prefill_minimum && prefill_aligned; const bool aligned_q6_prefill = q6_weight && aligned_prefill; const bool short_m64 = short_q6 && n == 1024 && (k == 1024 || k == 3072); - const char * expected_kernel = tokens == 1 ? "loom_libs:ggml_mul_mat_vector" : + const char * expected_kernel = tokens == 1 ? "loom_libs:ggml_mul_mat_vector_f32_f32" : short_m64 ? "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32" : short_q6 ? "loom_libs:ggml_mul_mat_q6_k_f32_wmma_prefill_wave32" : aligned_q6_prefill ? "loom_libs:ggml_mul_mat_q6_k_f16_wmma_prefill_wave32" : @@ -9451,13 +9451,13 @@ static void run_vector_output_capacity_experiment() { constexpr int64_t kAboveOldOutputSize = 262208; constexpr int64_t kLiftedOutputSize = 524288; - run_dense_matmul_cpu_reference_case(GGML_TYPE_F32, "loom_libs:ggml_mul_mat_vector", 1, kAboveOldOutputSize, + run_dense_matmul_cpu_reference_case(GGML_TYPE_F32, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, kAboveOldOutputSize, kSmallVectorInputSize); - run_dense_matmul_unary_cpu_reference_case(GGML_TYPE_F32, "loom_libs:ggml_mul_mat_vector", 1, + run_dense_matmul_unary_cpu_reference_case(GGML_TYPE_F32, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, kAboveOldOutputSize, kSmallVectorInputSize); run_dense_matmul_postops_cpu_reference_case(GGML_TYPE_F16, "loom_libs:ggml_mul_mat_vector_bias_residual_f32_f32", 1, kAboveOldOutputSize, true, false, false, kSmallVectorInputSize); - run_dense_matmul_cpu_reference_case(GGML_TYPE_F16, "loom_libs:ggml_mul_mat_vector", 1, kLiftedOutputSize, + run_dense_matmul_cpu_reference_case(GGML_TYPE_F16, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, kLiftedOutputSize, kSmallVectorInputSize); } @@ -9727,7 +9727,7 @@ static void register_dense_matmul_cases(Suite & suite) { for (const int64_t input_size : { 256, 384, 768 }) { suite.device_case("dense_matmul.q1_0.vector_tail.k" + std::to_string(input_size), [input_size] { run_dense_matmul_cpu_reference_case( - GGML_TYPE_Q1_0, "loom_libs:ggml_mul_mat_vector", 1, 65, input_size); + GGML_TYPE_Q1_0, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, 65, input_size); }); } @@ -9753,7 +9753,7 @@ static void register_dense_matmul_cases(Suite & suite) { [tokens, outputs] { run_dense_matmul_cpu_reference_case(GGML_TYPE_Q4_K, tokens == 1 ? - "loom_libs:ggml_mul_mat_vector" : + "loom_libs:ggml_mul_mat_vector_f32_f32" : "loom_libs:ggml_mul_mat_skinny_input_f32_publish_f32", tokens, outputs); }); @@ -9782,7 +9782,7 @@ static void register_dense_matmul_cases(Suite & suite) { 128); }); suite.device_case("dense_matmul.q4_k.vector.tokens1.outputs128", [] { - run_dense_matmul_cpu_reference_case(GGML_TYPE_Q4_K, "loom_libs:ggml_mul_mat_vector", 1, 128); + run_dense_matmul_cpu_reference_case(GGML_TYPE_Q4_K, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, 128); }); for (const ggml_type type : { GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS }) { suite.device_case("dense_matmul." + type_name(type) + ".codebook.vector.tokens1.outputs128", [type] { @@ -9850,7 +9850,7 @@ static void register_dense_matmul_cases(Suite & suite) { }); } suite.device_case("dense_matmul.iq4_xs.vector.tokens1.outputs128", [] { - run_dense_matmul_cpu_reference_case(GGML_TYPE_IQ4_XS, "loom_libs:ggml_mul_mat_vector", 1, 128); + run_dense_matmul_cpu_reference_case(GGML_TYPE_IQ4_XS, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, 128); }); suite.device_case("dense_matmul.iq4_xs.tiled.tokens64.outputs128", [] { run_dense_matmul_cpu_reference_case(GGML_TYPE_IQ4_XS, @@ -9882,7 +9882,7 @@ static void register_dense_matmul_cases(Suite & suite) { { GGML_TYPE_Q1_0, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_IQ2_S, GGML_TYPE_Q6_K, GGML_TYPE_Q8_0, GGML_TYPE_IQ4_NL, GGML_TYPE_F16, GGML_TYPE_BF16 }) { const char * vector_kernel = type == GGML_TYPE_IQ2_S ? "loom_libs:ggml_mul_mat_vector_iq2_s_f32_f32" : - "loom_libs:ggml_mul_mat_vector"; + "loom_libs:ggml_mul_mat_vector_f32_f32"; const int64_t outputs = type == GGML_TYPE_IQ2_S ? 640 : (type == GGML_TYPE_IQ4_NL ? 1024 : 128); const int64_t input = type == GGML_TYPE_IQ4_NL ? 640 : kQwenHiddenSize; const bool tiled_route = type == GGML_TYPE_IQ2_S || type == GGML_TYPE_IQ4_NL; @@ -9912,20 +9912,20 @@ static void register_dense_matmul_cases(Suite & suite) { 128, 544); }); suite.device_case("dense_matmul.q5_0.vector.tokens1.outputs256.input640", [] { - run_dense_matmul_cpu_reference_case(GGML_TYPE_Q5_0, "loom_libs:ggml_mul_mat_vector", 1, 256, 640); + run_dense_matmul_cpu_reference_case(GGML_TYPE_Q5_0, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, 256, 640); }); suite.device_case("dense_matmul.q8_1.zero_weight.skinny", [] { run_dense_matmul_zero_weight_case(GGML_TYPE_Q8_1, "loom_libs:ggml_mul_mat_skinny_input_f32_publish_f32"); }); suite.device_case("dense_matmul.q8_1.zero_weight.vector", [] { - run_dense_matmul_zero_weight_case(GGML_TYPE_Q8_1, "loom_libs:ggml_mul_mat_vector", 1); + run_dense_matmul_zero_weight_case(GGML_TYPE_Q8_1, "loom_libs:ggml_mul_mat_vector_f32_f32", 1); }); suite.device_case("dense_matmul.f32.skinny.tokens2.outputs256", [] { run_dense_matmul_cpu_reference_case(GGML_TYPE_F32, "loom_libs:ggml_mul_mat_skinny_input_f32_publish_f32", 2, 256); }); suite.device_case("dense_matmul.f32.vector.tokens1.outputs256", [] { - run_dense_matmul_cpu_reference_case(GGML_TYPE_F32, "loom_libs:ggml_mul_mat_vector", 1, 256); + run_dense_matmul_cpu_reference_case(GGML_TYPE_F32, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, 256); }); suite.device_case("dense_matmul_unary.f32.tokens2.outputs128", [] { run_dense_matmul_unary_cpu_reference_case(GGML_TYPE_F32, "loom_libs:ggml_mul_mat_skinny_input_f32_publish_f32", @@ -10221,24 +10221,24 @@ static void register_dense_matmul_cases(Suite & suite) { suite.device_case( "dense_matmul.q4_k.contiguous.tokens" + std::to_string(tokens) + ".outputs256.input5120", [tokens] { run_dense_matmul_cpu_reference_case(GGML_TYPE_Q4_K, - tokens == 1 ? "loom_libs:ggml_mul_mat_vector" : + tokens == 1 ? "loom_libs:ggml_mul_mat_vector_f32_f32" : "loom_libs:ggml_mul_mat_skinny_input_f32_publish_f32", tokens, 256, 5120, true); }); suite.device_case( "dense_matmul.q6_k.contiguous.tokens" + std::to_string(tokens) + ".outputs256.input5120", [tokens] { run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, - tokens == 1 ? "loom_libs:ggml_mul_mat_vector" : + tokens == 1 ? "loom_libs:ggml_mul_mat_vector_f32_f32" : "loom_libs:ggml_mul_mat_skinny_input_f32_publish_f32", tokens, 256, 5120, true, true); }); } suite.device_case("dense_matmul.q6_k.vector_q6.tokens1.outputs256.input5120", [] { - run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_vector", 1, 256, 5120, + run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, 256, 5120, true); }); suite.device_case("dense_matmul.q6_k.vector_q6.tokens1.outputs320.input512", [] { - run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_vector", 1, 320, 512, + run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, 320, 512, true); }); } @@ -10368,7 +10368,7 @@ static void register_rmsnorm_and_scheduling_cases(Suite & suite) { }); suite.device_case("q4_prefill_source_eligibility", [] { run_q4_prefill_source_eligibility_case(); }); suite.device_case("uploaded_native_q6_vector", [] { - run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_vector", 1, 128, + run_dense_matmul_cpu_reference_case(GGML_TYPE_Q6_K, "loom_libs:ggml_mul_mat_vector_f32_f32", 1, 128, 2048, false, false, GGML_TYPE_COUNT, true); }); suite.device_case("uploaded_native_gated_q4", [] {