From b67e7f3083219b0feb1a9f594f23bc2cd939db41 Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Tue, 3 Nov 2026 14:07:40 +0000 Subject: [PATCH 01/10] feat(hrx): support full-row max pool and standalone clamp for Mellum --- ggml/src/ggml-hrx/CMakeLists.txt | 2 + .../common/dispatch-common.cpp | 2 + .../common/dispatch-pool-1d.cpp | 78 +++++++++++++ .../common/dispatch-pool-1d.h | 9 ++ .../common/dispatch-unary.cpp | 57 +++++++++ ggml/src/ggml-hrx/ggml-hrx.cpp | 30 +++++ ggml/src/ggml-hrx/graph/op-params.cpp | 17 +++ ggml/src/ggml-hrx/graph/op-params.h | 8 ++ .../kernels/loom-libs/manifest.json | 84 ++++++++++++++ .../loom-libs/ops/elementwise/clamp_f32.loom | 36 ++++++ .../ops/reduction/pool_1d_full_max_f32.loom | 35 ++++++ tests/hrx-backend-test.cpp | 109 ++++++++++++++++++ 12 files changed, 467 insertions(+) create mode 100644 ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.cpp create mode 100644 ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.h create mode 100644 ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/elementwise/clamp_f32.loom create mode 100644 ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/reduction/pool_1d_full_max_f32.loom diff --git a/ggml/src/ggml-hrx/CMakeLists.txt b/ggml/src/ggml-hrx/CMakeLists.txt index 805545884c96..85641e9cca84 100644 --- a/ggml/src/ggml-hrx/CMakeLists.txt +++ b/ggml/src/ggml-hrx/CMakeLists.txt @@ -227,6 +227,8 @@ ggml_add_backend_library(ggml-hrx dispatch_registration/common/dispatch-mul-mat-common.h dispatch_registration/common/dispatch-mul-mat-weight-format.h dispatch_registration/common/dispatch-mul-mat.h + dispatch_registration/common/dispatch-pool-1d.cpp + dispatch_registration/common/dispatch-pool-1d.h dispatch_registration/common/dispatch-rope-set-rows.cpp dispatch_registration/common/dispatch-rope-set-rows.h dispatch_registration/common/dispatch-rope-utils.h diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-common.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-common.cpp index 9ccaebe22270..e68dfcd6d590 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-common.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-common.cpp @@ -10,6 +10,7 @@ #include "dispatch-glu.h" #include "dispatch-mul-mat-id.h" #include "dispatch-mul-mat.h" +#include "dispatch-pool-1d.h" #include "dispatch-rmsnorm.h" #include "dispatch-rope-set-rows.h" #include "dispatch-scale.h" @@ -28,6 +29,7 @@ void register_common_dispatches(DispatchRegistryBuilder & registry) { register_glu_dispatches(registry); register_mul_mat_id_dispatches(registry); register_mul_mat_dispatches(registry); + register_pool_1d_dispatch(registry); register_rope_set_rows_dispatches(registry); register_scale_dispatch(registry); register_unary_dispatch(registry); diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.cpp new file mode 100644 index 000000000000..7abe4f3c72f9 --- /dev/null +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.cpp @@ -0,0 +1,78 @@ +#include "dispatch-pool-1d.h" + +#include "ggml.h" +#include "kernel-corpus/kernel-corpus-catalog-verify.h" + +#include +#include + +namespace ggml::hrx { +namespace { + +static constexpr KernelCatalogRef kFullMaxPoolF32Kernel = + GGML_HRX_KERNEL_REF("loom_libs", "ggml_pool_1d_full_max_f32"); + +static bool match_full_max_pool_1d_f32(const DispatchMatchContext & context, DispatchMatch & match) { + const GraphNode * node = context.root_node; + if (node == nullptr) { + return false; + } + const bool one_input = node->inputs.size() == 1; + if (!one_input) { + return false; + } + + const Pool1DParams * params = op_params_as(node->params); + const Value * input = context.graph.values().find(node->inputs[0]); + const Value * output = context.graph.values().find(node->output); + const bool has_params = params != nullptr; + const bool has_input = input != nullptr; + const bool has_output = output != nullptr; + const bool all_present = has_params && has_input && has_output; + if (!all_present) { + return false; + } + + const bool f32_values = input->type == GGML_TYPE_F32 && output->type == GGML_TYPE_F32; + const bool contiguous_values = input->contiguous && output->contiguous; + const bool full_row = output->ne[0] == 1 && input->ne[0] == params->kernel; + const bool matching_rows = input->ne[1] == output->ne[1] && input->ne[2] == output->ne[2] && + input->ne[3] == output->ne[3]; + const bool exact_window = params->stride == params->kernel && params->padding == 0; + const bool bounded_width = input->ne[0] > 0 && input->ne[0] <= 32768; + const bool bounded_rows = output->element_count > 0 && output->element_count <= 1048576; + const bool bounded_input = input->element_count <= 134217728; + const bool bounded = bounded_width && bounded_rows && bounded_input; + const bool separate_storage = input->storage_root != output->storage_root; + const bool supported = params->op == GGML_OP_POOL_MAX && f32_values && contiguous_values && full_row && + matching_rows && exact_window && bounded && separate_storage; + if (!supported) { + return false; + } + + Dispatch dispatch; + dispatch.kernel = make_kernel_specialization(kFullMaxPoolF32Kernel); + dispatch.kernel.integer_parameters.emplace("row_count", output->element_count); + dispatch.kernel.integer_parameters.emplace("row_width", input->ne[0]); + dispatch.bindings.push_back({ input->id, 0, input->byte_count }); + dispatch.bindings.push_back({ output->id, 0, output->byte_count }); + + match.covered_nodes.push_back(context.root_index); + match.dispatches.push_back(std::move(dispatch)); + return true; +} + +} // namespace + +void register_pool_1d_dispatch(DispatchRegistryBuilder & registry) { + registry.add({ + "common.pool_1d_full_max_f32", + GGML_OP_POOL_1D, + DispatchMatchKind::SingleOp, + 0, + DispatchSource::Common, + match_full_max_pool_1d_f32, + }); +} + +} // namespace ggml::hrx diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.h b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.h new file mode 100644 index 000000000000..81d357295d2e --- /dev/null +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.h @@ -0,0 +1,9 @@ +#pragma once + +#include "dispatch_registration/dispatch-registry.h" + +namespace ggml::hrx { + +void register_pool_1d_dispatch(DispatchRegistryBuilder & registry); + +} // namespace ggml::hrx diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-unary.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-unary.cpp index 03cbc2b8b232..ead43da84824 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-unary.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-unary.cpp @@ -16,6 +16,7 @@ namespace { static constexpr KernelCatalogRef kUnaryF32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_unary_f32"); static constexpr KernelCatalogRef kScaleBiasF32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_scale_bias_f32"); +static constexpr KernelCatalogRef kClampF32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_clamp_f32"); static bool same_shape(const Value & lhs, const Value & rhs) { for (int i = 0; i < GGML_MAX_DIMS; ++i) { @@ -118,6 +119,54 @@ static bool match_unary_f32_dispatch(const DispatchMatchContext & context, Dispa return true; } +static bool match_clamp_f32_dispatch(const DispatchMatchContext & context, DispatchMatch & match) { + const GraphNode * node = context.root_node; + if (node == nullptr) { + return false; + } + const bool one_input = node->inputs.size() == 1; + if (!one_input) { + return false; + } + + const ClampParams * params = op_params_as(node->params); + const Value * input = graph_value(context.graph, node->inputs[0]); + const Value * output = graph_value(context.graph, node->output); + const bool has_params = params != nullptr; + const bool has_input = input != nullptr; + const bool has_output = output != nullptr; + const bool all_present = has_params && has_input && has_output; + if (!all_present) { + return false; + } + + const bool finite_limits = std::isfinite(params->min) && std::isfinite(params->max); + const bool ordered_limits = params->min <= params->max; + const bool f32_values = input->type == GGML_TYPE_F32 && output->type == GGML_TYPE_F32; + const bool contiguous_values = input->contiguous && output->contiguous; + const bool matching_shape = same_shape(*input, *output); + const bool safe_storage = scale_storage_is_safe(*input, *output); + const bool bounded = output->element_count > 0 && + static_cast(output->element_count) <= 134217728; + const bool supported = finite_limits && ordered_limits && f32_values && contiguous_values && + matching_shape && safe_storage && bounded; + if (!supported) { + return false; + } + + Dispatch dispatch; + dispatch.kernel = make_kernel_specialization(kClampF32Kernel); + dispatch.kernel.integer_parameters.emplace("element_count", output->element_count); + dispatch.kernel.compile_parameters.emplace("ggml.clamp_f32.min", float_config_value(params->min)); + dispatch.kernel.compile_parameters.emplace("ggml.clamp_f32.max", float_config_value(params->max)); + dispatch.bindings.push_back({ input->id, 0, input->byte_count }); + dispatch.bindings.push_back({ output->id, 0, output->byte_count }); + + match.covered_nodes.push_back(context.root_index); + match.dispatches.push_back(std::move(dispatch)); + return true; +} + static void register_unary_dispatch_for(DispatchRegistryBuilder & registry, ggml_op root_op) { registry.add({ "common.unary_f32", @@ -132,6 +181,14 @@ static void register_unary_dispatch_for(DispatchRegistryBuilder & registry, ggml } // namespace void register_unary_dispatch(DispatchRegistryBuilder & registry) { + registry.add({ + "common.clamp_f32", + GGML_OP_CLAMP, + DispatchMatchKind::SingleOp, + 0, + DispatchSource::Common, + match_clamp_f32_dispatch, + }); registry.add({ "common.scale_bias_f32", GGML_OP_SCALE, diff --git a/ggml/src/ggml-hrx/ggml-hrx.cpp b/ggml/src/ggml-hrx/ggml-hrx.cpp index 6be3802203f0..5c8742693f32 100644 --- a/ggml/src/ggml-hrx/ggml-hrx.cpp +++ b/ggml/src/ggml-hrx/ggml-hrx.cpp @@ -1139,6 +1139,33 @@ static bool supported_unary_f32_tensor(const ggml_tensor * op) { return ggml::hrx::unary_kind_supported(unary_kind); } +static bool supported_full_max_pool_1d_f32_tensor(const ggml_tensor * op) { + if (op == nullptr) { + return false; + } + const bool pool_op = op->op == GGML_OP_POOL_1D; + const bool has_input = op->src[0] != nullptr; + const bool pool_with_input = pool_op && has_input; + if (!pool_with_input) { + return false; + } + + const ggml_tensor * input = op->src[0]; + const bool f32_values = input->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + const bool contiguous_values = ggml_is_contiguous(input) && ggml_is_contiguous(op); + const bool full_row = op->ne[0] == 1 && input->ne[0] == ggml_get_op_params_i32(op, 1); + const bool matching_rows = input->ne[1] == op->ne[1] && input->ne[2] == op->ne[2] && + input->ne[3] == op->ne[3]; + const bool max_pool = ggml_get_op_params_i32(op, 0) == GGML_OP_POOL_MAX; + const bool exact_window = ggml_get_op_params_i32(op, 2) == input->ne[0] && + ggml_get_op_params_i32(op, 3) == 0; + const bool bounded_width = input->ne[0] > 0 && input->ne[0] <= 32768; + const bool bounded_rows = ggml_nelements(op) > 0 && ggml_nelements(op) <= 1048576; + const bool bounded_input = ggml_nelements(input) <= 134217728; + const bool bounded = bounded_width && bounded_rows && bounded_input; + return f32_values && contiguous_values && full_row && matching_rows && max_pool && exact_window && bounded; +} + static bool device_supports_op(ggml_backend_dev_t device, const ggml_tensor * op) { GGML_UNUSED(device); if (op == nullptr) { @@ -1171,6 +1198,9 @@ static bool device_supports_op(ggml_backend_dev_t device, const ggml_tensor * op if (supported_scale_f32_tensor(op)) { return true; } + if (supported_full_max_pool_1d_f32_tensor(op)) { + return true; + } if (supported_qwen_attention_projection_get_rows_tensor(op)) { return true; } diff --git a/ggml/src/ggml-hrx/graph/op-params.cpp b/ggml/src/ggml-hrx/graph/op-params.cpp index bf3374f973bf..3179f71e1cd2 100644 --- a/ggml/src/ggml-hrx/graph/op-params.cpp +++ b/ggml/src/ggml-hrx/graph/op-params.cpp @@ -48,6 +48,14 @@ static bool clamp_params_equivalent(const OpParams & lhs, const OpParams & rhs) nearly_equal(lhs_params->max, rhs_params->max); } +static bool pool_1d_params_equivalent(const OpParams & lhs, const OpParams & rhs) { + const Pool1DParams * lhs_params = op_params_as(lhs); + const Pool1DParams * rhs_params = op_params_as(rhs); + return lhs_params != nullptr && rhs_params != nullptr && lhs_params->op == rhs_params->op && + lhs_params->kernel == rhs_params->kernel && lhs_params->stride == rhs_params->stride && + lhs_params->padding == rhs_params->padding; +} + static bool glu_params_equivalent(const OpParams & lhs, const OpParams & rhs) { const GluParams * lhs_params = op_params_as(lhs); const GluParams * rhs_params = op_params_as(rhs); @@ -279,6 +287,13 @@ OpParams import_op_params(const ggml_tensor & tensor) { ggml_get_op_params_f32(&tensor, 0), ggml_get_op_params_f32(&tensor, 1), }; + case GGML_OP_POOL_1D: + return Pool1DParams{ + static_cast(ggml_get_op_params_i32(&tensor, 0)), + ggml_get_op_params_i32(&tensor, 1), + ggml_get_op_params_i32(&tensor, 2), + ggml_get_op_params_i32(&tensor, 3), + }; case GGML_OP_GLU: return GluParams{ ggml_get_glu_op(&tensor), ggml_get_op_params_i32(&tensor, 1) != 0 }; case GGML_OP_SCALE: @@ -323,6 +338,8 @@ bool op_params_equivalent(ggml_op op, const OpParams & lhs, const OpParams & rhs return argsort_params_equivalent(lhs, rhs); case GGML_OP_CLAMP: return clamp_params_equivalent(lhs, rhs); + case GGML_OP_POOL_1D: + return pool_1d_params_equivalent(lhs, rhs); case GGML_OP_GLU: return binary_params_equivalent(lhs, rhs) || glu_params_equivalent(lhs, rhs); case GGML_OP_SCALE: diff --git a/ggml/src/ggml-hrx/graph/op-params.h b/ggml/src/ggml-hrx/graph/op-params.h index 73f5a9122322..630bb71f720e 100644 --- a/ggml/src/ggml-hrx/graph/op-params.h +++ b/ggml/src/ggml-hrx/graph/op-params.h @@ -35,6 +35,13 @@ struct ClampParams { float max = 0.0f; }; +struct Pool1DParams { + ggml_op_pool op = GGML_OP_POOL_MAX; + int32_t kernel = 0; + int32_t stride = 0; + int32_t padding = 0; +}; + struct GluParams { ggml_glu_op op = GGML_GLU_OP_REGLU; bool swapped = false; @@ -120,6 +127,7 @@ using OpParams = std::variant< SoftMaxParams, ArgsortParams, ClampParams, + Pool1DParams, GluParams, ScaleParams, BinaryParams, diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json index 11c7e39e7b2c..9002ec9e5a38 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json @@ -20,6 +20,12 @@ { "path": "ops/elementwise/unary_f32.loom" }, + { + "path": "ops/elementwise/clamp_f32.loom" + }, + { + "path": "ops/reduction/pool_1d_full_max_f32.loom" + }, { "path": "ops/elementwise/binary_swiglu_symmetric_i4_k32.loom" }, @@ -1238,6 +1244,84 @@ "motifs/copy/concat_dim0.loom" ] }, + { + "name": "ggml_clamp_f32", + "family": "loom_libs", + "symbol": "ggml_clamp_f32", + "source": "ops/elementwise/clamp_f32.loom", + "workload_parameters": [ + { + "name": "element_count", + "type": "index" + } + ], + "launch_parameters": [ + { + "name": "element_count", + "type": "index" + } + ], + "bindings": [ + "input", + "output" + ], + "binding_access": [ + "read", + "read_write" + ], + "target_selector": "", + "compile_recipe": { + "mode": "direct", + "primary_sources": [ + "ops/elementwise/clamp_f32.loom" + ], + "library_sources": [] + }, + "compile_dependencies": [] + }, + { + "name": "ggml_pool_1d_full_max_f32", + "family": "loom_libs", + "symbol": "ggml_pool_1d_full_max_f32", + "source": "ops/reduction/pool_1d_full_max_f32.loom", + "workload_parameters": [ + { + "name": "row_count", + "type": "index" + }, + { + "name": "row_width", + "type": "index" + } + ], + "launch_parameters": [ + { + "name": "row_count", + "type": "index" + }, + { + "name": "row_width", + "type": "index" + } + ], + "bindings": [ + "input", + "output" + ], + "binding_access": [ + "read", + "write" + ], + "target_selector": "", + "compile_recipe": { + "mode": "direct", + "primary_sources": [ + "ops/reduction/pool_1d_full_max_f32.loom" + ], + "library_sources": [] + }, + "compile_dependencies": [] + }, { "name": "ggml_scale_bias_f32", "family": "loom_libs", diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/elementwise/clamp_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/elementwise/clamp_f32.loom new file mode 100644 index 000000000000..beddb6715efc --- /dev/null +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/elementwise/clamp_f32.loom @@ -0,0 +1,36 @@ +// Copyright 2026 The HRX Authors +// SPDX-License-Identifier: Apache-2.0 + +amdgpu.target @ggml_clamp_f32_gfx11_wave64 {subgroup_size = 64} + +config.decl @ggml.clamp_f32.min : f32 +config.decl @ggml.clamp_f32.max : f32 + +kernel.def target(@ggml_clamp_f32_gfx11_wave64) export("ggml_clamp_f32") @ggml_clamp_f32(%element_count: index) { + %one = index.constant 1 : index + %twofiftysix = index.constant 256 : index + %rounding = index.constant 255 : index + %rounded = index.add %element_count, %rounding : index + %workgroup_count = index.div %rounded, %twofiftysix : index + kernel.launch.config workgroups(%workgroup_count, %one, %one) workgroup_size(%twofiftysix, %one, %one) : index +} launch(%element_count: index, %input: buffer, %output: buffer) { + %count = index.assume %element_count [range(%element_count, 1, 134217728)] : index + %workgroup = kernel.workgroup.id : index + %workitem = kernel.workitem.id : index + %twofiftysix = index.constant 256 : index + %base = index.mul %workgroup, %twofiftysix : index + %linear0 = index.add %base, %workitem : index + %linear = index.assume %linear0 [range(%linear0, 0, 134217983)] : index + %in_bounds = index.cmp ult, %linear, %count : index + %minimum = config.get @ggml.clamp_f32.min : f32 + %maximum = config.get @ggml.clamp_f32.max : f32 + %zero_offset = index.constant 0 : offset + %input_view = buffer.view %input[%zero_offset] : buffer -> view<[%count]xf32> + %output_view = buffer.view %output[%zero_offset] : buffer -> view<[%count]xf32> + scf.if %in_bounds { + %value = view.load %input_view[%linear] : view<[%count]xf32> -> f32 + %result = scalar.clampf %value, %minimum, %maximum : f32 + view.store %result, %output_view[%linear] : f32, view<[%count]xf32> + } + kernel.return +} diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/reduction/pool_1d_full_max_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/reduction/pool_1d_full_max_f32.loom new file mode 100644 index 000000000000..dc6588e17b97 --- /dev/null +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/reduction/pool_1d_full_max_f32.loom @@ -0,0 +1,35 @@ +// Copyright 2026 The HRX Authors +// SPDX-License-Identifier: Apache-2.0 + +amdgpu.target @ggml_pool_1d_full_max_f32_gfx11_wave64 {subgroup_size = 64} + +kernel.def target(@ggml_pool_1d_full_max_f32_gfx11_wave64) export("ggml_pool_1d_full_max_f32") @ggml_pool_1d_full_max_f32(%row_count: index, %row_width: index) { + %one = index.constant 1 : index + %twofiftysix = index.constant 256 : index + kernel.launch.config workgroups(%row_count, %one, %one) workgroup_size(%twofiftysix, %one, %one) : index +} launch(%row_count: index, %row_width: index, %input: buffer, %output: buffer) where [range(%row_count, 1, 1048576), range(%row_width, 1, 32768)] { + %rows = index.assume %row_count [range(%row_count, 1, 1048576)] : index + %width = index.assume %row_width [range(%row_width, 1, 32768)] : index + %row = kernel.workgroup.id : index + %lane = kernel.workitem.id : index + %zero = index.constant 0 : index + %stride = index.constant 256 : index + %negative_max = scalar.constant -3.4028234663852886e+38 : f32 + %element_count = index.mul %rows, %width : index + %row_base = index.mul %row, %width : index + %zero_offset = index.constant 0 : offset + %input_view = buffer.view %input[%zero_offset] : buffer -> view<[%element_count]xf32> + %output_view = buffer.view %output[%zero_offset] : buffer -> view<[%rows]xf32> + %maximum = scf.for %column = [%lane to %width step %stride](%prior = %negative_max : f32) -> (f32) { + %offset = index.add %row_base, %column : index + %value = view.load %input_view[%offset] : view<[%element_count]xf32> -> f32 + %next = scalar.maxnumf %value, %prior : f32 + scf.yield %next : f32 + } + %row_maximum = kernel.workgroup.reduce %maximum : f32 + %is_first = index.cmp eq, %lane, %zero : index + scf.if %is_first { + view.store %row_maximum, %output_view[%row] : f32, view<[%rows]xf32> + } + kernel.return +} diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index b964c18a7c77..17011f352f77 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -1565,6 +1565,41 @@ static void run_scale_f32_dispatch_checks() { ggml_free(ctx); } +static void run_mellum_scaling_dispatch_checks() { + ggml_init_params params = {}; + params.mem_size = 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + + ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 896, 8, 16); + ggml_tensor * pooled = ggml_pool_1d(ctx, input, GGML_OP_POOL_MAX, 896, 896, 0); + ggml_tensor * clamp_input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1, 8, 16); + ggml_tensor * clamped = ggml_clamp(ctx, clamp_input, 0.0f, 127.0f); + REQUIRE(input != nullptr); + REQUIRE(pooled != nullptr); + REQUIRE(clamp_input != nullptr); + REQUIRE(clamped != nullptr); + + const ggml::hrx::Dispatch pool_dispatch = schedule_single_dispatch_for_tensor(ctx, pooled); + REQUIRE(kernel_name_for_id(pool_dispatch.kernel.kernel_id) == "loom_libs:ggml_pool_1d_full_max_f32"); + REQUIRE(pool_dispatch.kernel.integer_parameters.at("row_count") == 128); + REQUIRE(pool_dispatch.kernel.integer_parameters.at("row_width") == 896); + + const ggml::hrx::Dispatch clamp_dispatch = schedule_single_dispatch_for_tensor(ctx, clamped); + REQUIRE(kernel_name_for_id(clamp_dispatch.kernel.kernel_id) == "loom_libs:ggml_clamp_f32"); + REQUIRE(clamp_dispatch.kernel.integer_parameters.at("element_count") == 128); + require_compile_parameter(clamp_dispatch, "ggml.clamp_f32.min", expected_config_value(0.0f)); + require_compile_parameter(clamp_dispatch, "ggml.clamp_f32.max", expected_config_value(127.0f)); + + ggml_tensor * partial_pool = ggml_pool_1d(ctx, input, GGML_OP_POOL_MAX, 32, 32, 0); + ggml_tensor * average_pool = ggml_pool_1d(ctx, input, GGML_OP_POOL_AVG, 896, 896, 0); + REQUIRE(!can_schedule_tensor(ctx, partial_pool)); + REQUIRE(!can_schedule_tensor(ctx, average_pool)); + + ggml_free(ctx); +} + static void require_scale_add_dispatch(ggml_context * ctx, ggml_tensor * input, ggml_tensor * residual, @@ -13796,6 +13831,78 @@ static void run_scale_f32() { ggml_backend_free(backend); } +static std::vector run_mellum_scaling_numerical_case(ggml_backend_t backend, int64_t token_count) { + ggml_init_params params = {}; + params.mem_size = 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + + constexpr int64_t row_width = 896; + constexpr int64_t route_count = 8; + ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, row_width, route_count, token_count); + ggml_tensor * absolute = ggml_abs(ctx, input); + ggml_tensor * pooled = ggml_pool_1d(ctx, absolute, GGML_OP_POOL_MAX, row_width, row_width, 0); + ggml_tensor * clamped = ggml_clamp(ctx, pooled, 0.0f, 127.0f); + REQUIRE(input != nullptr); + REQUIRE(absolute != nullptr); + REQUIRE(pooled != nullptr); + REQUIRE(clamped != nullptr); + + ggml_cgraph * graph = ggml_new_graph(ctx); + REQUIRE(graph != nullptr); + ggml_build_forward_expand(graph, clamped); + + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); + REQUIRE(buffer != nullptr); + + std::vector input_data(row_width * route_count * token_count); + for (size_t i = 0; i < input_data.size(); ++i) { + input_data[i] = static_cast((i * 37) % 251) * 0.0625f - 7.5f; + } + input_data[0] = 0.0f; + input_data[row_width - 1] = -256.0f; + for (int64_t column = 0; column < row_width; ++column) { + input_data[row_width + column] = 0.0f; + input_data[2 * row_width + column] = 0.000125f; + } + input_data[2 * row_width - 1] = -0.000125f; + ggml_backend_tensor_set(input, input_data.data(), 0, input_data.size() * sizeof(float)); + + REQUIRE(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(backend); + + std::vector output(route_count * token_count); + ggml_backend_tensor_get(clamped, output.data(), 0, output.size() * sizeof(float)); + + ggml_backend_buffer_free(buffer); + ggml_free(ctx); + return output; +} + +static void run_mellum_scaling_numerics() { + ggml_backend_t hrx = ggml_backend_hrx_init(0); + ggml_backend_t cpu = ggml_backend_cpu_init(); + REQUIRE(hrx != nullptr); + REQUIRE(cpu != nullptr); + + for (int64_t token_count : { 1, 16 }) { + const std::vector expected = run_mellum_scaling_numerical_case(cpu, token_count); + const std::vector actual = run_mellum_scaling_numerical_case(hrx, token_count); + const std::vector repeated = run_mellum_scaling_numerical_case(hrx, token_count); + REQUIRE(actual.size() == expected.size()); + REQUIRE(repeated.size() == expected.size()); + for (size_t i = 0; i < expected.size(); ++i) { + REQUIRE(std::isfinite(actual[i])); + REQUIRE(actual[i] == expected[i]); + REQUIRE(repeated[i] == actual[i]); + } + } + + ggml_backend_free(cpu); + ggml_backend_free(hrx); +} + static void run_scale_f32_inplace() { ggml_backend_t backend = ggml_backend_hrx_init(0); REQUIRE(backend != nullptr); @@ -14641,6 +14748,7 @@ static void register_hrx_backend_host_cases(test_runner::Suite & suite) { run_qwen_weighted_reduce_next_rmsnorm_q8_publication_checks(); }); suite.host_case("common_mul_mat_id_postops_dispatch", [] { run_common_mul_mat_id_postops_dispatch_checks(); }); + suite.host_case("mellum_scaling_dispatch", [] { run_mellum_scaling_dispatch_checks(); }); suite.host_case("alias_value_import", [] { run_alias_value_import_checks(); }); suite.host_case("multi_dispatch", [] { run_multi_dispatch_checks(); }); suite.host_case("layout_alias_scheduler_elision", [] { run_layout_alias_scheduler_elision_checks(); }); @@ -14812,6 +14920,7 @@ static void register_hrx_backend_device_cases(test_runner::Suite & suite) { suite.device_case("rope_scale_f32_numerics", [] { run_rope_scale_f32_numerics(); }); suite.device_case("gather_add_rmsnorm_f32_numerics", [] { run_gather_add_rmsnorm_f32_numerics(); }); suite.device_case("scale_f32", [] { run_scale_f32(); }); + suite.device_case("mellum_scaling_numerics", [] { run_mellum_scaling_numerics(); }); suite.device_case("scale_f32_inplace", [] { run_scale_f32_inplace(); }); suite.device_case("two_independent_add_f32", [] { run_two_independent_add_f32(); }); suite.device_case("chained_add_f32", [] { run_chained_add_f32(); }); From c8c010a3faced41b68c87912b1acd47030f72b56 Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Tue, 3 Nov 2026 14:18:24 +0000 Subject: [PATCH 02/10] Narrow Mellum clamp dispatch and test scaling numerics --- .../common/dispatch-unary.cpp | 6 +- tests/hrx-backend-test.cpp | 100 ++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-unary.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-unary.cpp index ead43da84824..2c4bb9778d1f 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-unary.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-unary.cpp @@ -140,15 +140,15 @@ static bool match_clamp_f32_dispatch(const DispatchMatchContext & context, Dispa return false; } - const bool finite_limits = std::isfinite(params->min) && std::isfinite(params->max); - const bool ordered_limits = params->min <= params->max; + const bool mellum_amax_limits = params->min == 0.0f && params->max == 127.0f; + const bool scalar_per_row = input->ne[0] == 1 && output->ne[0] == 1; const bool f32_values = input->type == GGML_TYPE_F32 && output->type == GGML_TYPE_F32; const bool contiguous_values = input->contiguous && output->contiguous; const bool matching_shape = same_shape(*input, *output); const bool safe_storage = scale_storage_is_safe(*input, *output); const bool bounded = output->element_count > 0 && static_cast(output->element_count) <= 134217728; - const bool supported = finite_limits && ordered_limits && f32_values && contiguous_values && + const bool supported = mellum_amax_limits && scalar_per_row && f32_values && contiguous_values && matching_shape && safe_storage && bounded; if (!supported) { return false; diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index 17011f352f77..43221757e2bc 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -1592,6 +1592,12 @@ static void run_mellum_scaling_dispatch_checks() { require_compile_parameter(clamp_dispatch, "ggml.clamp_f32.min", expected_config_value(0.0f)); require_compile_parameter(clamp_dispatch, "ggml.clamp_f32.max", expected_config_value(127.0f)); + ggml_tensor * wide_clamp_input = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 8); + ggml_tensor * wide_clamp = ggml_clamp(ctx, wide_clamp_input, 0.0f, 127.0f); + ggml_tensor * other_bounds = ggml_clamp(ctx, clamp_input, 0.0f, 1.0f); + REQUIRE(!can_schedule_tensor(ctx, wide_clamp)); + REQUIRE(!can_schedule_tensor(ctx, other_bounds)); + ggml_tensor * partial_pool = ggml_pool_1d(ctx, input, GGML_OP_POOL_MAX, 32, 32, 0); ggml_tensor * average_pool = ggml_pool_1d(ctx, input, GGML_OP_POOL_AVG, 896, 896, 0); REQUIRE(!can_schedule_tensor(ctx, partial_pool)); @@ -13903,6 +13909,99 @@ static void run_mellum_scaling_numerics() { ggml_backend_free(hrx); } +static std::vector run_full_max_pool_numerical_case(ggml_backend_t backend, int64_t row_width) { + ggml_context * ctx = ggml_init({ 1024 * 1024, nullptr, true }); + REQUIRE(ctx != nullptr); + constexpr int64_t row_count = 8; + ggml_tensor * input = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, row_width, row_count); + ggml_tensor * output = ggml_pool_1d(ctx, input, GGML_OP_POOL_MAX, row_width, row_width, 0); + REQUIRE(input != nullptr); + REQUIRE(output != nullptr); + ggml_cgraph * graph = ggml_new_graph(ctx); + REQUIRE(graph != nullptr); + ggml_build_forward_expand(graph, output); + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); + REQUIRE(buffer != nullptr); + + std::vector input_data(row_width * row_count, -4.0f); + for (int64_t column = 0; column < row_width; ++column) { + input_data[row_width + column] = 0.0f; + input_data[2 * row_width + column] = -0.000125f; + } + input_data[row_width - 1] = -1.0f; + input_data[2 * row_width + row_width / 2] = 0.000125f; + input_data[3 * row_width] = 100000.0f; + input_data[3 * row_width + row_width - 1] = -100000.0f; + ggml_backend_tensor_set(input, input_data.data(), 0, input_data.size() * sizeof(float)); + + REQUIRE(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(backend); + std::vector result(row_count); + ggml_backend_tensor_get(output, result.data(), 0, result.size() * sizeof(float)); + + ggml_backend_buffer_free(buffer); + ggml_free(ctx); + return result; +} + +static std::vector run_clamp_numerical_case(ggml_backend_t backend, bool inplace) { + ggml_context * ctx = ggml_init({ 1024 * 1024, nullptr, true }); + REQUIRE(ctx != nullptr); + ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1, 8, 1); + ggml_tensor * output = inplace ? ggml_clamp_inplace(ctx, input, 0.0f, 127.0f) : + ggml_clamp(ctx, input, 0.0f, 127.0f); + REQUIRE(input != nullptr); + REQUIRE(output != nullptr); + ggml_cgraph * graph = ggml_new_graph(ctx); + REQUIRE(graph != nullptr); + ggml_build_forward_expand(graph, output); + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); + REQUIRE(buffer != nullptr); + + const std::array input_data = { -1000.0f, -1.0f, 0.0f, 0.000125f, 126.0f, 127.0f, 128.0f, 1000.0f }; + ggml_backend_tensor_set(input, input_data.data(), 0, sizeof(input_data)); + REQUIRE(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(backend); + std::vector result(input_data.size()); + ggml_backend_tensor_get(output, result.data(), 0, result.size() * sizeof(float)); + + ggml_backend_buffer_free(buffer); + ggml_free(ctx); + return result; +} + +static void run_mellum_scaling_direct_numerics() { + ggml_backend_t hrx = ggml_backend_hrx_init(0); + ggml_backend_t cpu = ggml_backend_cpu_init(); + REQUIRE(hrx != nullptr); + REQUIRE(cpu != nullptr); + + for (int64_t row_width : { 64, 896 }) { + const std::vector expected = run_full_max_pool_numerical_case(cpu, row_width); + const std::vector actual = run_full_max_pool_numerical_case(hrx, row_width); + const std::vector repeated = run_full_max_pool_numerical_case(hrx, row_width); + REQUIRE(expected[0] == -1.0f); + REQUIRE(expected[1] == 0.0f); + REQUIRE(expected[2] == 0.000125f); + REQUIRE(expected[3] == 100000.0f); + REQUIRE(actual == expected); + REQUIRE(repeated == actual); + } + + for (bool inplace : { false, true }) { + const std::vector expected = run_clamp_numerical_case(cpu, inplace); + const std::vector actual = run_clamp_numerical_case(hrx, inplace); + const std::vector repeated = run_clamp_numerical_case(hrx, inplace); + REQUIRE(actual == expected); + REQUIRE(repeated == actual); + REQUIRE(actual[0] == 0.0f); + REQUIRE(actual[7] == 127.0f); + } + + ggml_backend_free(cpu); + ggml_backend_free(hrx); +} + static void run_scale_f32_inplace() { ggml_backend_t backend = ggml_backend_hrx_init(0); REQUIRE(backend != nullptr); @@ -14921,6 +15020,7 @@ static void register_hrx_backend_device_cases(test_runner::Suite & suite) { suite.device_case("gather_add_rmsnorm_f32_numerics", [] { run_gather_add_rmsnorm_f32_numerics(); }); suite.device_case("scale_f32", [] { run_scale_f32(); }); suite.device_case("mellum_scaling_numerics", [] { run_mellum_scaling_numerics(); }); + suite.device_case("mellum_scaling_direct_numerics", [] { run_mellum_scaling_direct_numerics(); }); suite.device_case("scale_f32_inplace", [] { run_scale_f32_inplace(); }); suite.device_case("two_independent_add_f32", [] { run_two_independent_add_f32(); }); suite.device_case("chained_add_f32", [] { run_chained_add_f32(); }); From dc1ed82f045a5a7e078d9f58c870405901f8821a Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Tue, 3 Nov 2026 14:20:44 +0000 Subject: [PATCH 03/10] Name Mellum clamp bound by exponent contract --- .../ggml-hrx/dispatch_registration/common/dispatch-unary.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-unary.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-unary.cpp index 2c4bb9778d1f..4036061d86fa 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-unary.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-unary.cpp @@ -140,7 +140,7 @@ static bool match_clamp_f32_dispatch(const DispatchMatchContext & context, Dispa return false; } - const bool mellum_amax_limits = params->min == 0.0f && params->max == 127.0f; + const bool exponent_limits = params->min == 0.0f && params->max == 127.0f; const bool scalar_per_row = input->ne[0] == 1 && output->ne[0] == 1; const bool f32_values = input->type == GGML_TYPE_F32 && output->type == GGML_TYPE_F32; const bool contiguous_values = input->contiguous && output->contiguous; @@ -148,7 +148,7 @@ static bool match_clamp_f32_dispatch(const DispatchMatchContext & context, Dispa const bool safe_storage = scale_storage_is_safe(*input, *output); const bool bounded = output->element_count > 0 && static_cast(output->element_count) <= 134217728; - const bool supported = mellum_amax_limits && scalar_per_row && f32_values && contiguous_values && + const bool supported = exponent_limits && scalar_per_row && f32_values && contiguous_values && matching_shape && safe_storage && bounded; if (!supported) { return false; From e93f49526187928600192af1672d56f21f57d69d Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Tue, 3 Nov 2026 16:24:33 +0000 Subject: [PATCH 04/10] ggml-hrx: handle partial K blocks in routed down matmul --- ...inny_input_f32_dequant_accumulate_f32.loom | 30 ++-- ...iled_input_f32_dequant_accumulate_f32.loom | 30 ++-- ..._f32_postops_next_rmsnorm_publish_f32.loom | 2 +- .../skinny_input_f32_postops_publish_f32.loom | 2 +- .../skinny_input_f32_publish_f32.loom | 2 +- ..._f32_postops_next_rmsnorm_publish_f32.loom | 2 +- .../tiled_input_f32_postops_publish_f32.loom | 2 +- .../tiled_input_f32_publish_f32.loom | 2 +- tests/hrx-backend-test.cpp | 159 +++++++++++++++++- 9 files changed, 200 insertions(+), 31 deletions(-) diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul_id/skinny_input_f32_dequant_accumulate_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul_id/skinny_input_f32_dequant_accumulate_f32.loom index 2c59b2d7d059..43bb423c20db 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul_id/skinny_input_f32_dequant_accumulate_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul_id/skinny_input_f32_dequant_accumulate_f32.loom @@ -9,7 +9,7 @@ template.decl @ggml.matmul_id.skinny_f32_f32.finish_tile(%arg0: index, %arg1: in template.decl @ggml.matmul_id.skinny_f32_f32.publish_vector4(%arg0: i1, %arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index, %arg6: index, %arg7: index, %arg8: vector<4xf32>, %arg9: buffer, %arg10: buffer, %arg11: buffer, %arg12: buffer, %arg13: buffer, %arg14: buffer, %arg15: buffer) -func.decl @ggml_dequant_weight_tile_bytes(%weight_format: index) -> (offset) +func.decl @ggml_dequant_weight_row_bytes(%weight_format: index, %hidden_size: index) -> (offset) func.decl @ggml_iq4nl_table_i8() -> (vector<16xi8>) @@ -44,6 +44,7 @@ func.def inline @ggml_moe_unpack_expert_partition_descriptor(%descriptor: i32) - } template.def<@ggml.matmul_id.skinny_f32_f32.core> device @ggml_mul_mat_id_skinny_f32_f32_core(%weight_format: index, %token_count: index, %input_size: index, %output_size: index, %route_count: index, %input_route_count: index, %expert_count: index, %epsilon: f32, %channel_tile: index, %partition_ordinal: index, %input: buffer, %expert_table: buffer, %partition_table: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer) { + %bounded_input_size = index.assume %input_size [range(%input_size, 256, 32768), multiple_of(%input_size, 32)] : index %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048)] : index %bounded_route_count = index.assume %route_count [range(%route_count, 1, 32)] : index %bounded_input_route_count = index.assume %input_route_count [range(%input_route_count, 1, 32), le(%input_route_count, %bounded_route_count)] : index @@ -63,6 +64,7 @@ template.def<@ggml.matmul_id.skinny_f32_f32.core> device @ggml_mul_mat_id_skinny %c40 = index.constant 40 : index %c63 = index.constant 63 : index %c64 = index.constant 64 : index + %c255 = index.constant 255 : index %c256 = index.constant 256 : index %c0_offset = index.constant 0 : offset %c4_bytes = index.constant 4 : offset @@ -73,6 +75,7 @@ template.def<@ggml.matmul_id.skinny_f32_f32.core> device @ggml_mul_mat_id_skinny %result_stage_bytes = index.constant 1024 : offset %c0_i32 = scalar.constant 0 : i32 %cn1_i32 = scalar.constant -1 : i32 + %c0_f32x4 = vector.constant 0.0 : vector<4xf32> %c0_f16x4 = vector.constant 0.0 : vector<4xf16> %zero_accumulator = vector.constant 0.0 : vector<8xf16> %m = index.constant 16 : index @@ -87,12 +90,12 @@ template.def<@ggml.matmul_id.skinny_f32_f32.core> device @ggml_mul_mat_id_skinny %has_more_assignment_partitions = index.cmp ugt, %assignment_partition_count, %bounded_expert_count : index %launch_partition_count = scf.select %has_more_assignment_partitions, %assignment_partition_count, %bounded_expert_count : index %assignment_table_byte_base = index.scale %bounded_expert_count, %c4_bytes : index, offset -> offset - %weight_block_bytes = func.call @ggml_dequant_weight_tile_bytes(%weight_format) : (index) -> (offset) - %quant_block_count = index.div %input_size, %c256 : index - %weight_row_bytes = index.scale %quant_block_count, %weight_block_bytes : index, offset -> offset + %padded_input_size = index.add %bounded_input_size, %c255 : index + %quant_block_count = index.div %padded_input_size, %c256 : index + %weight_row_bytes = func.call @ggml_dequant_weight_row_bytes(%weight_format, %bounded_input_size) : (index, index) -> (offset) %weight_expert_bytes = index.scale %bounded_output_size, %weight_row_bytes : index, offset -> offset %input_noalias, %expert_table_noalias, %partition_table_noalias, %weight_noalias = buffer.assume.noalias %input, %expert_table, %partition_table, %weight : buffer, buffer, buffer, buffer - %input_view = buffer.view %input_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]x[%bounded_input_route_count]x[%input_size]xf32> + %input_view = buffer.view %input_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]x[%bounded_input_route_count]x[%bounded_input_size]xf32> %assignment_view = buffer.view %expert_table_noalias[%assignment_table_byte_base] : buffer -> view<[%bounded_expert_count]x[%bounded_token_count]xi32> %partition_count_view = buffer.view %partition_table_noalias[%c0_offset] : buffer -> view<1xi32> %partition_descriptor_view = buffer.view %partition_table_noalias[%c4_bytes] : buffer -> view<[%maximum_partition_count]xi32> @@ -172,11 +175,13 @@ template.def<@ggml.matmul_id.skinny_f32_f32.core> device @ggml_mul_mat_id_skinny %local_row = index.assume %local_row0 [range(%local_row0, 0, 63)] : index %channel = index.add %channel_tile_base, %local_row : index %valid_channel = index.cmp ult, %channel, %bounded_output_size : index - %weight_values = scf.if %valid_channel -> (vector<4xf16>) { + %weight_k = index.add %k_origin, %load_k : index + %valid_k = index.cmp ult, %weight_k, %bounded_input_size : index + %valid_weight = scalar.andi %valid_channel, %valid_k : i1 + %weight_values = scf.if %valid_weight -> (vector<4xf16>) { %channel_byte_add = index.scale %channel, %weight_row_bytes : index, offset -> offset %row_byte_base = index.add %expert_byte_base, %channel_byte_add : offset - %weight_k = index.add %k_origin, %load_k : index - %decoded = func.call @ggml_dequant_f16_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight_noalias, %row_byte_base, %input_size, %quant_block, %quant_group, %load_packet, %weight_k) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index) -> (vector<4xf16>) + %decoded = func.call @ggml_dequant_f16_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight_noalias, %row_byte_base, %bounded_input_size, %quant_block, %quant_group, %load_packet, %weight_k) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index) -> (vector<4xf16>) scf.yield %decoded : vector<4xf16> } else { scf.yield %c0_f16x4 : vector<4xf16> @@ -187,7 +192,10 @@ template.def<@ggml.matmul_id.skinny_f32_f32.core> device @ggml_mul_mat_id_skinny %activation_row = index.assume %local_row [range(%local_row, 0, 31)] : index %assignment_i32 = view.load %route_stage_view[%activation_row] : view<32xi32> -> i32 %valid_assignment = scalar.cmpi sge, %assignment_i32, %c0_i32 : i32 - %activation_values = scf.if %valid_assignment -> (vector<4xf16>) { + %input_k = index.add %k_origin, %load_k : index + %valid_input = index.cmp ult, %input_k, %bounded_input_size : index + %valid_activation = scalar.andi %valid_assignment, %valid_input : i1 + %activation_values = scf.if %valid_activation -> (vector<4xf16>) { %assignment0 = index.cast %assignment_i32 : i32 to index %assignment = index.assume %assignment0 [range(%assignment0, 0, 65535)] : index %bounded_assignment, %bounded_assignment_count = index.assume %assignment, %assignment_count [lt(%assignment, %assignment_count)] : index, index @@ -196,8 +204,8 @@ template.def<@ggml.matmul_id.skinny_f32_f32.core> device @ggml_mul_mat_id_skinny %token, %input_token_count = index.assume %token0, %bounded_token_count [lt(%token0, %bounded_token_count)] : index, index %input_route0 = index.rem %route0, %bounded_input_route_count : index %input_route = index.assume %input_route0 [range(%input_route0, 0, 31), lt(%input_route0, %bounded_input_route_count)] : index - %input_k = index.add %k_origin, %load_k : index - %loaded = vector.load %input_view[%token, %input_route, %input_k] : view<[%bounded_token_count]x[%bounded_input_route_count]x[%input_size]xf32> -> vector<4xf32> + %mask = vector.mask.range [%input_k to %bounded_input_size step %c1] : index -> vector<4xi1> + %loaded = vector.load.mask %input_view[%token, %input_route, %input_k], %mask, %c0_f32x4 : view<[%bounded_token_count]x[%bounded_input_route_count]x[%bounded_input_size]xf32>, vector<4xi1>, vector<4xf32> %converted = vector.fptrunc %loaded : vector<4xf32> to vector<4xf16> scf.yield %converted : vector<4xf16> } else { diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul_id/tiled_input_f32_dequant_accumulate_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul_id/tiled_input_f32_dequant_accumulate_f32.loom index 359fcf33e146..422aa66c4221 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul_id/tiled_input_f32_dequant_accumulate_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul_id/tiled_input_f32_dequant_accumulate_f32.loom @@ -9,7 +9,7 @@ template.decl @ggml.matmul_id.tiled_f32_f32.finish_tile(%arg0: index, %arg1: ind template.decl @ggml.matmul_id.tiled_f32_f32.publish_vector4(%arg0: i1, %arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index, %arg6: index, %arg7: index, %arg8: vector<4xf32>, %arg9: buffer, %arg10: buffer, %arg11: buffer, %arg12: buffer, %arg13: buffer, %arg14: buffer, %arg15: buffer) -func.decl @ggml_dequant_weight_tile_bytes(%weight_format: index) -> (offset) +func.decl @ggml_dequant_weight_row_bytes(%weight_format: index, %hidden_size: index) -> (offset) func.decl @ggml_iq4nl_table_i8() -> (vector<16xi8>) @@ -44,6 +44,7 @@ func.def inline @ggml_moe_unpack_expert_partition_descriptor(%descriptor: i32) - } template.def<@ggml.matmul_id.tiled_f32_f32.core> device @ggml_mul_mat_id_tiled_f32_f32_core(%weight_format: index, %token_count: index, %input_size: index, %output_size: index, %route_count: index, %input_route_count: index, %expert_count: index, %epsilon: f32, %channel_tile: index, %partition_ordinal: index, %input: buffer, %expert_table: buffer, %partition_table: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer) { + %bounded_input_size = index.assume %input_size [range(%input_size, 256, 32768), multiple_of(%input_size, 32)] : index %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048)] : index %bounded_route_count = index.assume %route_count [range(%route_count, 1, 32)] : index %bounded_input_route_count = index.assume %input_route_count [range(%input_route_count, 1, 32), le(%input_route_count, %bounded_route_count)] : index @@ -63,6 +64,7 @@ template.def<@ggml.matmul_id.tiled_f32_f32.core> device @ggml_mul_mat_id_tiled_f %c40 = index.constant 40 : index %c63 = index.constant 63 : index %c64 = index.constant 64 : index + %c255 = index.constant 255 : index %c256 = index.constant 256 : index %c0_offset = index.constant 0 : offset %c4_bytes = index.constant 4 : offset @@ -73,6 +75,7 @@ template.def<@ggml.matmul_id.tiled_f32_f32.core> device @ggml_mul_mat_id_tiled_f %result_stage_bytes = index.constant 1024 : offset %c0_i32 = scalar.constant 0 : i32 %cn1_i32 = scalar.constant -1 : i32 + %c0_f32x4 = vector.constant 0.0 : vector<4xf32> %c0_f16x4 = vector.constant 0.0 : vector<4xf16> %zero_accumulator = vector.constant 0.0 : vector<8xf16> %m = index.constant 16 : index @@ -87,12 +90,12 @@ template.def<@ggml.matmul_id.tiled_f32_f32.core> device @ggml_mul_mat_id_tiled_f %has_more_assignment_partitions = index.cmp ugt, %assignment_partition_count, %bounded_expert_count : index %launch_partition_count = scf.select %has_more_assignment_partitions, %assignment_partition_count, %bounded_expert_count : index %assignment_table_byte_base = index.scale %bounded_expert_count, %c4_bytes : index, offset -> offset - %weight_block_bytes = func.call @ggml_dequant_weight_tile_bytes(%weight_format) : (index) -> (offset) - %quant_block_count = index.div %input_size, %c256 : index - %weight_row_bytes = index.scale %quant_block_count, %weight_block_bytes : index, offset -> offset + %padded_input_size = index.add %bounded_input_size, %c255 : index + %quant_block_count = index.div %padded_input_size, %c256 : index + %weight_row_bytes = func.call @ggml_dequant_weight_row_bytes(%weight_format, %bounded_input_size) : (index, index) -> (offset) %weight_expert_bytes = index.scale %bounded_output_size, %weight_row_bytes : index, offset -> offset %input_noalias, %expert_table_noalias, %partition_table_noalias, %weight_noalias = buffer.assume.noalias %input, %expert_table, %partition_table, %weight : buffer, buffer, buffer, buffer - %input_view = buffer.view %input_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]x[%bounded_input_route_count]x[%input_size]xf32> + %input_view = buffer.view %input_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]x[%bounded_input_route_count]x[%bounded_input_size]xf32> %assignment_view = buffer.view %expert_table_noalias[%assignment_table_byte_base] : buffer -> view<[%bounded_expert_count]x[%bounded_token_count]xi32> %partition_count_view = buffer.view %partition_table_noalias[%c0_offset] : buffer -> view<1xi32> %partition_descriptor_view = buffer.view %partition_table_noalias[%c4_bytes] : buffer -> view<[%maximum_partition_count]xi32> @@ -172,11 +175,13 @@ template.def<@ggml.matmul_id.tiled_f32_f32.core> device @ggml_mul_mat_id_tiled_f %local_row = index.assume %local_row0 [range(%local_row0, 0, 63)] : index %channel = index.add %channel_tile_base, %local_row : index %valid_channel = index.cmp ult, %channel, %bounded_output_size : index - %weight_values = scf.if %valid_channel -> (vector<4xf16>) { + %weight_k = index.add %k_origin, %load_k : index + %valid_k = index.cmp ult, %weight_k, %bounded_input_size : index + %valid_weight = scalar.andi %valid_channel, %valid_k : i1 + %weight_values = scf.if %valid_weight -> (vector<4xf16>) { %channel_byte_add = index.scale %channel, %weight_row_bytes : index, offset -> offset %row_byte_base = index.add %expert_byte_base, %channel_byte_add : offset - %weight_k = index.add %k_origin, %load_k : index - %decoded = func.call @ggml_dequant_f16_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight_noalias, %row_byte_base, %input_size, %quant_block, %quant_group, %load_packet, %weight_k) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index) -> (vector<4xf16>) + %decoded = func.call @ggml_dequant_f16_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight_noalias, %row_byte_base, %bounded_input_size, %quant_block, %quant_group, %load_packet, %weight_k) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index) -> (vector<4xf16>) scf.yield %decoded : vector<4xf16> } else { scf.yield %c0_f16x4 : vector<4xf16> @@ -187,7 +192,10 @@ template.def<@ggml.matmul_id.tiled_f32_f32.core> device @ggml_mul_mat_id_tiled_f %activation_row = index.assume %local_row [range(%local_row, 0, 31)] : index %assignment_i32 = view.load %route_stage_view[%activation_row] : view<32xi32> -> i32 %valid_assignment = scalar.cmpi sge, %assignment_i32, %c0_i32 : i32 - %activation_values = scf.if %valid_assignment -> (vector<4xf16>) { + %input_k = index.add %k_origin, %load_k : index + %valid_input = index.cmp ult, %input_k, %bounded_input_size : index + %valid_activation = scalar.andi %valid_assignment, %valid_input : i1 + %activation_values = scf.if %valid_activation -> (vector<4xf16>) { %assignment0 = index.cast %assignment_i32 : i32 to index %assignment = index.assume %assignment0 [range(%assignment0, 0, 65535)] : index %bounded_assignment, %bounded_assignment_count = index.assume %assignment, %assignment_count [lt(%assignment, %assignment_count)] : index, index @@ -196,8 +204,8 @@ template.def<@ggml.matmul_id.tiled_f32_f32.core> device @ggml_mul_mat_id_tiled_f %token, %input_token_count = index.assume %token0, %bounded_token_count [lt(%token0, %bounded_token_count)] : index, index %input_route0 = index.rem %route0, %bounded_input_route_count : index %input_route = index.assume %input_route0 [range(%input_route0, 0, 31), lt(%input_route0, %bounded_input_route_count)] : index - %input_k = index.add %k_origin, %load_k : index - %loaded = vector.load %input_view[%token, %input_route, %input_k] : view<[%bounded_token_count]x[%bounded_input_route_count]x[%input_size]xf32> -> vector<4xf32> + %mask = vector.mask.range [%input_k to %bounded_input_size step %c1] : index -> vector<4xi1> + %loaded = vector.load.mask %input_view[%token, %input_route, %input_k], %mask, %c0_f32x4 : view<[%bounded_token_count]x[%bounded_input_route_count]x[%bounded_input_size]xf32>, vector<4xi1>, vector<4xf32> %converted = vector.fptrunc %loaded : vector<4xf32> to vector<4xf16> scf.yield %converted : vector<4xf16> } else { diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_postops_next_rmsnorm_publish_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_postops_next_rmsnorm_publish_f32.loom index c25b9210877e..837ae0b4fd87 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_postops_next_rmsnorm_publish_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_postops_next_rmsnorm_publish_f32.loom @@ -14,7 +14,7 @@ template.decl @ggml.matmul_id.skinny_f32_f32.store_vector4(%arg0: index, %arg1: amdgpu.target @ggml_mul_mat_id_skinny_input_f32_postops_next_rmsnorm_publish_f32_gfx11_wave64 {subgroup_size = 64} -config.decl @ggml.mul_mat_id_postops.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 256)] +config.decl @ggml.mul_mat_id_postops.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 32)] config.decl @ggml.mul_mat_id_postops.output_size : %value: index where [range(%value, 128, 32768), multiple_of(%value, 128)] diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_postops_publish_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_postops_publish_f32.loom index a2184804de65..77b214a5a1d8 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_postops_publish_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_postops_publish_f32.loom @@ -12,7 +12,7 @@ template.decl @ggml.matmul_id.skinny_f32_f32.store_vector4(%arg0: index, %arg1: amdgpu.target @ggml_mul_mat_id_skinny_input_f32_postops_publish_f32_gfx11_wave64 {subgroup_size = 64} -config.decl @ggml.mul_mat_id_postops.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 256)] +config.decl @ggml.mul_mat_id_postops.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 32)] config.decl @ggml.mul_mat_id_postops.output_size : %value: index where [range(%value, 1, 262144)] diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_publish_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_publish_f32.loom index d6a1a16cbf01..0acfbcddfe4a 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_publish_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_publish_f32.loom @@ -11,7 +11,7 @@ template.decl @ggml.matmul_id.skinny_f32_f32.publish_vector4(%publish_word: i1, amdgpu.target @ggml_mul_mat_id_skinny_input_f32_publish_f32_gfx11_wave64 {subgroup_size = 64} -config.decl @ggml.mul_mat_id.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 256)] +config.decl @ggml.mul_mat_id.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 32)] config.decl @ggml.mul_mat_id.output_size : %value: index where [range(%value, 1, 262144)] diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_postops_next_rmsnorm_publish_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_postops_next_rmsnorm_publish_f32.loom index 0c60dd9fd87c..791c90b7c9a0 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_postops_next_rmsnorm_publish_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_postops_next_rmsnorm_publish_f32.loom @@ -14,7 +14,7 @@ template.decl @ggml.matmul_id.tiled_f32_f32.store_vector4(%arg0: index, %arg1: i amdgpu.target @ggml_mul_mat_id_tiled_input_f32_postops_next_rmsnorm_publish_f32_gfx11_wave64 {subgroup_size = 64} -config.decl @ggml.mul_mat_id_postops.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 256)] +config.decl @ggml.mul_mat_id_postops.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 32)] config.decl @ggml.mul_mat_id_postops.output_size : %value: index where [range(%value, 128, 32768), multiple_of(%value, 128)] diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_postops_publish_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_postops_publish_f32.loom index 8d9596b1d063..a6b13afe0cff 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_postops_publish_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_postops_publish_f32.loom @@ -12,7 +12,7 @@ template.decl @ggml.matmul_id.tiled_f32_f32.store_vector4(%arg0: index, %arg1: i amdgpu.target @ggml_mul_mat_id_tiled_input_f32_postops_publish_f32_gfx11_wave64 {subgroup_size = 64} -config.decl @ggml.mul_mat_id_postops.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 256)] +config.decl @ggml.mul_mat_id_postops.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 32)] config.decl @ggml.mul_mat_id_postops.output_size : %value: index where [range(%value, 1, 262144)] diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_publish_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_publish_f32.loom index 9d92fd4d15cc..28ee2469795a 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_publish_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_publish_f32.loom @@ -11,7 +11,7 @@ template.decl @ggml.matmul_id.tiled_f32_f32.publish_vector4(%publish_word: i1, % amdgpu.target @ggml_mul_mat_id_tiled_input_f32_publish_f32_gfx11_wave64 {subgroup_size = 64} -config.decl @ggml.mul_mat_id.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 256)] +config.decl @ggml.mul_mat_id.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 32)] config.decl @ggml.mul_mat_id.output_size : %value: index where [range(%value, 1, 262144)] diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index 43221757e2bc..e708753c3a6c 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -10158,14 +10158,15 @@ static CommonMulMatIdPostOpsTensors build_common_mul_mat_id_postops_graph(ggml_c bool include_residual, bool include_rmsnorm, ggml_type weight_type = GGML_TYPE_Q4_K, - int64_t token_count = 4) { + int64_t token_count = 4, + int64_t input_size = 256, + int64_t input_routes = 1) { CommonMulMatIdPostOpsTensors tensors; constexpr int64_t route_count = 8; - constexpr int64_t input_size = 256; constexpr int64_t output_size = 128; constexpr int64_t expert_count = 16; tensors.route_ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, route_count, token_count); - tensors.input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, input_size, 1, token_count); + tensors.input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, input_size, input_routes, token_count); tensors.weight = ggml_new_tensor_3d(ctx, weight_type, input_size, output_size, expert_count); REQUIRE(tensors.route_ids != nullptr); REQUIRE(tensors.input != nullptr); @@ -14002,6 +14003,144 @@ static void run_mellum_scaling_direct_numerics() { ggml_backend_free(hrx); } +struct RoutedKTailOutputs { + std::vector raw; + std::vector normalized; +}; + +static RoutedKTailOutputs run_routed_k_tail_case(ggml_backend_t backend, + ggml_type weight_type, + int64_t input_size, + int64_t token_count, + int postops) { + ggml_context * ctx = ggml_init({ 4 * 1024 * 1024, nullptr, true }); + REQUIRE(ctx != nullptr); + const bool has_postops = postops != 0; + const bool has_rmsnorm = postops == 2; + const CommonMulMatIdPostOpsTensors tensors = build_common_mul_mat_id_postops_graph( + ctx, has_postops, has_postops, has_rmsnorm, weight_type, token_count, input_size, 8); + // HRX may keep internal values in private storage; both observed values must be external outputs. + ggml_set_output(tensors.residual_output); + ggml_set_output(tensors.normalized_output); + ggml_cgraph * graph = ggml_new_graph(ctx); + REQUIRE(graph != nullptr); + ggml_build_forward_expand(graph, tensors.normalized_output); + + // Require each of the six shared exports, so another dispatch cannot hide a missing tail path. + ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*graph); + REQUIRE(imported.valid()); + std::vector covered(imported.graph.nodes().size(), false); + ggml::hrx::CommandPlan plan; + ggml::hrx::DispatchMatch match; + const size_t root = producer_index_for_tensor(imported.graph, tensors.projection); + REQUIRE(match_dispatch_at_index(imported.graph, plan, covered, root, match)); + REQUIRE(match.dispatches.size() == 3); + const char * expected_kernel = has_postops ? common_mul_mat_id_postops_f32_kernel_name(token_count, has_rmsnorm) : + common_mul_mat_id_f32_kernel_name(token_count); + REQUIRE(kernel_name_for_id(match.dispatches.back().kernel.kernel_id) == expected_kernel); + + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); + REQUIRE(buffer != nullptr); + const int64_t output_size = tensors.weight->ne[1]; + const int64_t expert_count = tensors.weight->ne[2]; + const int64_t route_count = tensors.route_ids->ne[0]; + std::vector weights(ggml_nelements(tensors.weight)); + for (int64_t row = 0; row < output_size * expert_count; ++row) { + for (int64_t column = 0; column < input_size; ++column) { + const int64_t packet = column % 32; + float value; + if (weight_type == GGML_TYPE_Q8_0) { + const int code = packet == 0 ? 127 : packet == 1 ? -127 : packet == 2 ? 0 : + static_cast((row * 3 + column + column / 32) % 31) - 15; + value = static_cast(code) / 256.0f; + } else { + REQUIRE(weight_type == GGML_TYPE_Q5_1); + const int code = packet == 0 ? 0 : packet == 1 ? 31 : packet == 2 ? 0 : + static_cast((row * 3 + column + column / 32) % 32); + value = static_cast(code) / 64.0f; + } + weights[row * input_size + column] = value; + } + } + std::vector packed(ggml_nbytes(tensors.weight)); + REQUIRE(ggml_quantize_chunk(weight_type, weights.data(), packed.data(), 0, + output_size * expert_count, input_size, nullptr) == packed.size()); + ggml_backend_tensor_set(tensors.weight, packed.data(), 0, packed.size()); + + // Zero-weight anchors fix the CPU's activation quantization scale to an exact binary value. + // The remaining sparse products and sums are also exact in F16 WMMA. Two contributions are + // in the final partial block, and weight rows vary across channels and experts. + std::vector input(ggml_nelements(tensors.input), 0.0f); + std::vector ids(ggml_nelements(tensors.route_ids)); + for (int64_t token = 0; token < token_count; ++token) { + for (int64_t route = 0; route < route_count; ++route) { + const int64_t assignment = token * route_count + route; + for (int64_t column = 2; column < input_size; column += 32) { + input[assignment * input_size + column] = 127.0f / 16.0f; + } + input[assignment * input_size + 3] = static_cast(route + 1) / 16.0f; + input[assignment * input_size + input_size - 5] = static_cast(token % 3 + 1) / 16.0f; + input[assignment * input_size + input_size - 37] = -1.0f / 16.0f; + ids[assignment] = static_cast((token * 3 + route) % expert_count); + } + } + ggml_backend_tensor_set(tensors.input, input.data(), 0, input.size() * sizeof(float)); + ggml_backend_tensor_set(tensors.route_ids, ids.data(), 0, ids.size() * sizeof(int32_t)); + if (has_postops) { + std::vector bias(output_size); + std::vector residual(ggml_nelements(tensors.residual_input)); + for (size_t i = 0; i < bias.size(); ++i) { + bias[i] = static_cast(static_cast(i % 7) - 3) / 128.0f; + } + for (size_t i = 0; i < residual.size(); ++i) { + residual[i] = static_cast(static_cast(i % 13) - 6) / 256.0f; + } + ggml_backend_tensor_set(tensors.bias, bias.data(), 0, bias.size() * sizeof(float)); + ggml_backend_tensor_set(tensors.residual_input, residual.data(), 0, residual.size() * sizeof(float)); + } + if (has_rmsnorm) { + std::vector norm_weight(output_size); + for (size_t i = 0; i < norm_weight.size(); ++i) { + norm_weight[i] = 0.5f + static_cast(i % 5) / 8.0f; + } + ggml_backend_tensor_set(tensors.norm_weight, norm_weight.data(), 0, norm_weight.size() * sizeof(float)); + } + + REQUIRE(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(backend); + RoutedKTailOutputs result; + result.raw.resize(ggml_nelements(tensors.residual_output)); + result.normalized.resize(ggml_nelements(tensors.normalized_output)); + ggml_backend_tensor_get(tensors.residual_output, result.raw.data(), 0, result.raw.size() * sizeof(float)); + ggml_backend_tensor_get(tensors.normalized_output, result.normalized.data(), 0, + result.normalized.size() * sizeof(float)); + ggml_backend_buffer_free(buffer); + ggml_free(ctx); + return result; +} + +static void run_routed_k_tail_numerics(ggml_type type, int64_t input_size, int64_t token_count, int postops) { + ggml_backend_t hrx = ggml_backend_hrx_init(0); + ggml_backend_t cpu = ggml_backend_cpu_init(); + REQUIRE(hrx != nullptr); + REQUIRE(cpu != nullptr); + ggml_backend_cpu_set_n_threads(cpu, 2); + const RoutedKTailOutputs expected = run_routed_k_tail_case(cpu, type, input_size, token_count, postops); + const RoutedKTailOutputs actual = run_routed_k_tail_case(hrx, type, input_size, token_count, postops); + const RoutedKTailOutputs repeated = run_routed_k_tail_case(hrx, type, input_size, token_count, postops); + REQUIRE(std::any_of(expected.raw.begin(), expected.raw.end(), [](float value) { return value != 0.0f; })); + REQUIRE(actual.raw == expected.raw); + REQUIRE(repeated.raw == actual.raw); + REQUIRE(repeated.normalized == actual.normalized); + for (size_t i = 0; i < actual.normalized.size(); ++i) { + REQUIRE(std::isfinite(actual.normalized[i])); + const float tolerance = 3.0e-5f + 3.0e-5f * std::fabs(expected.normalized[i]); + REQUIRE(std::fabs(actual.normalized[i] - expected.normalized[i]) <= tolerance); + } + ggml_backend_free(cpu); + ggml_backend_free(hrx); +} + static void run_scale_f32_inplace() { ggml_backend_t backend = ggml_backend_hrx_init(0); REQUIRE(backend != nullptr); @@ -15021,6 +15160,20 @@ static void register_hrx_backend_device_cases(test_runner::Suite & suite) { suite.device_case("scale_f32", [] { run_scale_f32(); }); suite.device_case("mellum_scaling_numerics", [] { run_mellum_scaling_numerics(); }); suite.device_case("mellum_scaling_direct_numerics", [] { run_mellum_scaling_direct_numerics(); }); + for (const auto & shape : { std::pair{ GGML_TYPE_Q8_0, 896 }, + { GGML_TYPE_Q8_0, 704 }, { GGML_TYPE_Q5_1, 704 }, + { GGML_TYPE_Q8_0, 1024 }, { GGML_TYPE_Q5_1, 768 } }) { + for (int64_t tokens : { 1, 16 }) { + for (int postops : { 0, 1, 2 }) { + const std::string name = "routed_k_tail." + std::string(ggml_type_name(shape.first)) + + ".k" + std::to_string(shape.second) + ".t" + std::to_string(tokens) + + ".postops" + std::to_string(postops); + suite.device_case(name, [shape, tokens, postops] { + run_routed_k_tail_numerics(shape.first, shape.second, tokens, postops); + }); + } + } + } suite.device_case("scale_f32_inplace", [] { run_scale_f32_inplace(); }); suite.device_case("two_independent_add_f32", [] { run_two_independent_add_f32(); }); suite.device_case("chained_add_f32", [] { run_chained_add_f32(); }); From 56bba51dabf772901dace9f79a79b779c2daed38 Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Tue, 3 Nov 2026 19:38:23 +0000 Subject: [PATCH 05/10] test(hrx): cover Q1_0 routed K tails and aligned controls --- tests/hrx-backend-test.cpp | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index e708753c3a6c..1e5773e6e3fc 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -14053,11 +14053,17 @@ static RoutedKTailOutputs run_routed_k_tail_case(ggml_backend_t backend, const int code = packet == 0 ? 127 : packet == 1 ? -127 : packet == 2 ? 0 : static_cast((row * 3 + column + column / 32) % 31) - 15; value = static_cast(code) / 256.0f; - } else { - REQUIRE(weight_type == GGML_TYPE_Q5_1); + } else if (weight_type == GGML_TYPE_Q5_1) { const int code = packet == 0 ? 0 : packet == 1 ? 31 : packet == 2 ? 0 : static_cast((row * 3 + column + column / 32) % 32); value = static_cast(code) / 64.0f; + } else { + REQUIRE(weight_type == GGML_TYPE_Q1_0); + const float scale = static_cast(1 + (row + column / 128) % 3) / 16.0f; + const bool is_anchor = packet < 2; + const bool positive_code = (row + column / 32 + column / 128) % 2 == 0; + const bool positive_weight = is_anchor || positive_code; + value = positive_weight ? scale : -scale; } weights[row * input_size + column] = value; } @@ -14067,7 +14073,8 @@ static RoutedKTailOutputs run_routed_k_tail_case(ggml_backend_t backend, output_size * expert_count, input_size, nullptr) == packed.size()); ggml_backend_tensor_set(tensors.weight, packed.data(), 0, packed.size()); - // Zero-weight anchors fix the CPU's activation quantization scale to an exact binary value. + // Anchors fix the CPU's activation quantization scale to an exact binary value. Q1_0 has + // no zero code, so its two anchors cancel against equal weights within each Q8_0 block. // The remaining sparse products and sums are also exact in F16 WMMA. Two contributions are // in the final partial block, and weight rows vary across channels and experts. std::vector input(ggml_nelements(tensors.input), 0.0f); @@ -14075,8 +14082,13 @@ static RoutedKTailOutputs run_routed_k_tail_case(ggml_backend_t backend, for (int64_t token = 0; token < token_count; ++token) { for (int64_t route = 0; route < route_count; ++route) { const int64_t assignment = token * route_count + route; - for (int64_t column = 2; column < input_size; column += 32) { - input[assignment * input_size + column] = 127.0f / 16.0f; + for (int64_t column = 0; column < input_size; column += 32) { + if (weight_type == GGML_TYPE_Q1_0) { + input[assignment * input_size + column] = 127.0f / 16.0f; + input[assignment * input_size + column + 1] = -127.0f / 16.0f; + } else { + input[assignment * input_size + column + 2] = 127.0f / 16.0f; + } } input[assignment * input_size + 3] = static_cast(route + 1) / 16.0f; input[assignment * input_size + input_size - 5] = static_cast(token % 3 + 1) / 16.0f; @@ -15162,7 +15174,8 @@ static void register_hrx_backend_device_cases(test_runner::Suite & suite) { suite.device_case("mellum_scaling_direct_numerics", [] { run_mellum_scaling_direct_numerics(); }); for (const auto & shape : { std::pair{ GGML_TYPE_Q8_0, 896 }, { GGML_TYPE_Q8_0, 704 }, { GGML_TYPE_Q5_1, 704 }, - { GGML_TYPE_Q8_0, 1024 }, { GGML_TYPE_Q5_1, 768 } }) { + { GGML_TYPE_Q8_0, 1024 }, { GGML_TYPE_Q5_1, 768 }, + { GGML_TYPE_Q1_0, 384 }, { GGML_TYPE_Q1_0, 512 } }) { for (int64_t tokens : { 1, 16 }) { for (int postops : { 0, 1, 2 }) { const std::string name = "routed_k_tail." + std::string(ggml_type_name(shape.first)) + From a4d8c6a474f32aa2dbd01ac039cb993bc24947df Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Thu, 5 Nov 2026 02:33:14 +0000 Subject: [PATCH 06/10] ggml-hrx: preserve caller attention metadata Remove metadata regeneration and compact-mask substitution. Reuse the merged RoPE factor fixture and graph-reuse checks for the mask and ring-cache regression; explicit factors now exercise the corrected production path. Consolidates the metadata fix and its fixture/scope follow-ups without the reverted long-context fallback. Bead: lemonade-ws-fnxc.4 Assisted-by: Codex --- .../common/dispatch-flash-attention.cpp | 39 +---- .../dispatch-qwen-attention-postprocess.cpp | 70 -------- tests/hrx-backend-test.cpp | 43 +---- tests/hrx-ops-test.cpp | 151 ++++++++---------- 4 files changed, 80 insertions(+), 223 deletions(-) diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-flash-attention.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-flash-attention.cpp index c1fec1a8a8ce..2ad2afcc7c55 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-flash-attention.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-flash-attention.cpp @@ -109,13 +109,6 @@ static bool has_flash_attention_params(const GraphNode & node) { (params->prec == GGML_PREC_DEFAULT || params->prec == GGML_PREC_F32); } -static size_t attention_mask_byte_count(int64_t query_token_count, int64_t key_value_token_count) { - if (query_token_count <= 0 || key_value_token_count <= 0) { - return 0; - } - return static_cast(query_token_count) * static_cast(key_value_token_count) * sizeof(ggml_fp16_t); -} - static std::string to_config_value(int64_t value) { return std::to_string(value); } @@ -144,8 +137,6 @@ struct FlashAttentionMatch { const Value * mask = nullptr; const Value * output = nullptr; const GraphNode * output_layout = nullptr; - ValueId mask_binding_value; - size_t mask_binding_bytes = 0; int64_t query_token_count = 0; int64_t key_value_token_count = 0; int64_t query_head_count = 0; @@ -181,7 +172,6 @@ struct DecodeSplitFlashAttentionMatch { }; static FlashAttentionMatch match_flash_attention_f32_f16(const Graph & graph, - const CommandPlan & plan, const GraphNode * node) { FlashAttentionMatch match; if (node == nullptr || node->op != GGML_OP_FLASH_ATTN_EXT || node->inputs.size() != 4) { @@ -229,20 +219,9 @@ static FlashAttentionMatch match_flash_attention_f32_f16(const Graph & gra return {}; } - int64_t key_value_token_count = key_value_capacity; - ValueId mask_binding_value = mask->id; - size_t mask_binding_bytes = mask->byte_count; - bool mask_binding_is_alternate = false; - const size_t compact_mask_bytes = attention_mask_byte_count(query_token_count, query_token_count); - const auto * compact_mask = find_alternate_value(plan, mask->id, GGML_TYPE_F16, compact_mask_bytes); - const bool mask_is_compact = mask->ne[0] == query_token_count; - const bool mask_is_capacity = mask->ne[0] == key_value_capacity; - if (compact_mask != nullptr && mask_is_capacity && mask->ne[0] > query_token_count) { - key_value_token_count = query_token_count; - mask_binding_value = compact_mask->alternate_value; - mask_binding_bytes = compact_mask->byte_count; - mask_binding_is_alternate = true; - } else if (!mask_is_compact && !mask_is_capacity) { + const int64_t key_value_token_count = key_value_capacity; + const bool mask_is_capacity = mask->ne[0] == key_value_capacity; + if (!mask_is_capacity) { return {}; } if (!is_supported_key_value_token_count(key_value_token_count)) { @@ -255,7 +234,7 @@ static FlashAttentionMatch match_flash_attention_f32_f16(const Graph & gra !has_output_layout(*output, query_head_count, value_head_size)) { return {}; } - if (!mask_binding_is_alternate && !has_mask_layout(*mask, key_value_token_count)) { + if (!has_mask_layout(*mask, key_value_token_count)) { return {}; } @@ -265,8 +244,6 @@ static FlashAttentionMatch match_flash_attention_f32_f16(const Graph & gra match.mask = mask; match.output = output; match.output_layout = find_single_layout_alias_consumer(graph, output->id); - match.mask_binding_value = mask_binding_value; - match.mask_binding_bytes = mask_binding_bytes; match.query_token_count = query_token_count; match.key_value_token_count = key_value_token_count; match.query_head_count = query_head_count; @@ -425,7 +402,7 @@ static DispatchBinding prepare_flash_attention_value(const DispatchMatchContext } static bool match_flash_attention_gate_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) { - const FlashAttentionMatch match = match_flash_attention_f32_f16(context.graph, context.plan, context.root_node); + const FlashAttentionMatch match = match_flash_attention_f32_f16(context.graph, context.root_node); if (!match.matched() || match.output_layout == nullptr || match.output_layout->op != GGML_OP_RESHAPE) { return false; } @@ -503,7 +480,7 @@ static bool match_flash_attention_gate_dispatch(const DispatchMatchContext & con dispatch.bindings.push_back({ match.query->id, 0, match.query->byte_count }); dispatch.bindings.push_back({ match.key->id, 0, match.key->byte_count }); dispatch.bindings.push_back(prepare_flash_attention_value(context, match, dispatch_match, dispatch.kernel)); - dispatch.bindings.push_back({ match.mask_binding_value, 0, match.mask_binding_bytes }); + dispatch.bindings.push_back({ match.mask->id, 0, match.mask->byte_count }); dispatch.bindings.push_back({ raw_gate->id, 0, raw_gate->byte_count }); dispatch.bindings.push_back({ output->id, 0, output->byte_count }); dispatch_match.dispatches.push_back(std::move(dispatch)); @@ -512,7 +489,7 @@ static bool match_flash_attention_gate_dispatch(const DispatchMatchContext & con static bool match_flash_attention_f32_f16_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) { - const FlashAttentionMatch match = match_flash_attention_f32_f16(context.graph, context.plan, context.root_node); + const FlashAttentionMatch match = match_flash_attention_f32_f16(context.graph, context.root_node); if (!match.matched()) { return false; } @@ -540,7 +517,7 @@ static bool match_flash_attention_f32_f16_dispatch(const DispatchMatchContext & dispatch.bindings.push_back({ match.query->id, 0, match.query->byte_count }); dispatch.bindings.push_back({ match.key->id, 0, match.key->byte_count }); dispatch.bindings.push_back(prepare_flash_attention_value(context, match, dispatch_match, dispatch.kernel)); - dispatch.bindings.push_back({ match.mask_binding_value, 0, match.mask_binding_bytes }); + dispatch.bindings.push_back({ match.mask->id, 0, match.mask->byte_count }); dispatch.bindings.push_back({ match.query->id, 0, match.query->byte_count }); dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count }); diff --git a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-attention-postprocess.cpp b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-attention-postprocess.cpp index 2466a8e711b2..d538c71d1657 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-attention-postprocess.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-attention-postprocess.cpp @@ -21,10 +21,6 @@ static constexpr KernelCatalogRef kQwenAttentionPostprocessF32F16Kernel = GGML_HRX_KERNEL_REF("qwen3_moe", "qwen3_moe_attention_postprocess_f32_f16"); static constexpr KernelCatalogRef kQwenAttentionQkvPostprocessFusedDecodeKernel = GGML_HRX_KERNEL_REF("qwen3_moe", "qwen3_moe_attention_qkv_postprocess_fused_decode"); -static constexpr KernelCatalogRef kQwenAttentionContextBaseCaptureKernel = - GGML_HRX_KERNEL_REF("qwen", "qwen_attention_context_base_capture"); -static constexpr KernelCatalogRef kQwenAttentionMetadataKernel = - GGML_HRX_KERNEL_REF("qwen3_moe", "qwen_attention_metadata"); static constexpr int64_t kQwenAttentionHeadSize = 128; static const Value * graph_value(const Graph & graph, ValueId id) { @@ -735,71 +731,11 @@ static bool append_postprocess_covered_nodes(const DispatchMatchContext & c return true; } -static bool has_attention_metadata_initialization(const CommandPlan & plan) { - for (const Dispatch & dispatch : plan.initialization_dispatches) { - if (dispatch.kernel.kernel_id == kQwenAttentionMetadataKernel.id) { - return true; - } - } - return false; -} - static ValueId next_match_transient_value(const DispatchMatchContext & context, const DispatchMatch & dispatch_match) { return ValueId(context.next_plan_value.value + static_cast(dispatch_match.transients.size()) + static_cast(dispatch_match.completion_counter_requests.size())); } -static bool append_attention_metadata_initialization(const DispatchMatchContext & context, - const AttentionPostprocessMatch & match, - DispatchMatch & dispatch_match) { - if (has_attention_metadata_initialization(context.plan)) { - return true; - } - if (!match.flash_layouts.matched() || match.flash_layouts.flash->inputs.size() != 4) { - return true; - } - - const Value * mask = graph_value(context.graph, match.flash_layouts.flash->inputs[3]); - if (mask == nullptr || mask->type != GGML_TYPE_F16 || mask->ne[0] <= 0 || mask->ne[1] != match.query.token_count) { - return false; - } - int64_t context_capacity = mask->ne[0]; - size_t mask_byte_count = mask->byte_count; - if (context_capacity > 32768 && match.query.token_count <= 32768) { - context_capacity = match.query.token_count; - mask_byte_count = static_cast(match.query.token_count) * static_cast(match.query.token_count) * - sizeof(ggml_fp16_t); - Status metadata_status; - if (!dispatch_match.metadata.append_alternate_value( - { mask->id, mask->id, GGML_TYPE_F16, mask_byte_count, "qwen.attention.compact_mask" }, - metadata_status)) { - dispatch_match.status.append(metadata_status); - return false; - } - } - - const ValueId control = next_match_transient_value(context, dispatch_match); - dispatch_match.transients.push_back({ control, "qwen.attention.control", sizeof(int32_t), 16 }); - - Dispatch context_capture; - context_capture.kernel = make_kernel_specialization(kQwenAttentionContextBaseCaptureKernel); - context_capture.bindings.push_back({ match.query.positions->id, 0, match.query.positions->byte_count }); - context_capture.bindings.push_back({ control, 0, sizeof(int32_t) }); - dispatch_match.initialization_dispatches.push_back(std::move(context_capture)); - - Dispatch metadata; - metadata.kernel = make_kernel_specialization(kQwenAttentionMetadataKernel); - metadata.kernel.integer_parameters.emplace("token_count", match.query.token_count); - metadata.kernel.integer_parameters.emplace("context_capacity", context_capacity); - metadata.bindings.push_back({ control, 0, sizeof(int32_t) }); - metadata.bindings.push_back({ match.query.positions->id, 0, match.query.positions->byte_count }); - metadata.bindings.push_back({ match.key.cache_indices->id, 0, match.key.cache_indices->byte_count }); - metadata.bindings.push_back({ match.value.cache_indices->id, 0, match.value.cache_indices->byte_count }); - metadata.bindings.push_back({ mask->id, 0, mask_byte_count }); - dispatch_match.initialization_dispatches.push_back(std::move(metadata)); - return true; -} - static const Value * projection_weight(const Graph & graph, const GraphNode * projection) { return projection == nullptr || projection->inputs.size() != 2 ? nullptr : graph_value(graph, projection->inputs[0]); @@ -869,9 +805,6 @@ static bool match_qwen_attention_qkv_postprocess_fused_decode_dispatch(const Dis &dispatch_match.status)) { return false; } - if (!append_attention_metadata_initialization(context, match, dispatch_match)) { - return false; - } const ValueId theta_value = next_match_transient_value(context, dispatch_match); dispatch_match.transients.push_back({ theta_value, "qwen.attention_qkv_decode.theta", @@ -949,9 +882,6 @@ static bool match_qwen_attention_postprocess_dispatch(const DispatchMatchContext if (!append_postprocess_covered_nodes(context, match, dispatch_match, &dispatch_match.status)) { return false; } - if (!append_attention_metadata_initialization(context, match, dispatch_match)) { - return false; - } Dispatch dispatch; const ValueId theta_value = next_match_transient_value(context, dispatch_match); diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index 1e5773e6e3fc..39d270fcfa25 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -5989,46 +5989,17 @@ static void run_qwen_attention_postprocess_dispatch_checks() { producer_index_for_tensor(imported.graph, tensors.query_reshape), match)); append_match_to_plan(plan, match, covered_nodes); REQUIRE(plan.valid()); - REQUIRE(plan.initialization_dispatches.size() == 2); - - const ggml::hrx::Dispatch & context_capture = plan.initialization_dispatches[0]; - const ggml::hrx::Dispatch & metadata = plan.initialization_dispatches[1]; - REQUIRE(kernel_name_for_id(context_capture.kernel.kernel_id) == "qwen:qwen_attention_context_base_capture"); - REQUIRE(kernel_name_for_id(metadata.kernel.kernel_id) == "qwen3_moe:qwen_attention_metadata"); - REQUIRE(context_capture.bindings.size() == 2); - REQUIRE(metadata.bindings.size() == 5); - REQUIRE(context_capture.bindings[1].value == metadata.bindings[0].value); - REQUIRE(metadata.kernel.integer_parameters.at("token_count") == token_count); - REQUIRE(metadata.kernel.integer_parameters.at("context_capacity") == cache_row_count); - - const ggml::hrx::Value * positions_value = imported.graph.values().find_tensor(tensors.positions); - const ggml::hrx::Value * key_indices_value = imported.graph.values().find_tensor(tensors.key_cache_indices); - const ggml::hrx::Value * value_indices_value = imported.graph.values().find_tensor(tensors.value_cache_indices); - const ggml::hrx::Value * mask_value = imported.graph.values().find_tensor(tensors.mask); - REQUIRE(positions_value != nullptr); - REQUIRE(key_indices_value != nullptr); - REQUIRE(value_indices_value != nullptr); - REQUIRE(mask_value != nullptr); - REQUIRE(context_capture.bindings[0].value == positions_value->id); - REQUIRE(metadata.bindings[1].value == positions_value->id); - REQUIRE(metadata.bindings[2].value == key_indices_value->id); - REQUIRE(metadata.bindings[3].value == value_indices_value->id); - REQUIRE(metadata.bindings[4].value == mask_value->id); + REQUIRE(plan.initialization_dispatches.empty()); + REQUIRE(plan.dispatches.size() == 1); + const ggml::hrx::Dispatch & postprocess = plan.dispatches.front(); + REQUIRE(postprocess.bindings[0].value == imported.graph.values().find_tensor(tensors.positions)->id); + REQUIRE(postprocess.bindings[1].value == imported.graph.values().find_tensor(tensors.key_cache_indices)->id); + REQUIRE(postprocess.bindings[2].value == imported.graph.values().find_tensor(tensors.value_cache_indices)->id); const ggml::hrx::CommandProgram commands = ggml::hrx::build_command_program(imported.graph, plan, ggml::hrx::get_qwen_kernel_corpus(), "gfx1151"); REQUIRE(commands.valid()); - REQUIRE(commands.initialization_commands.size() == 2); - REQUIRE(commands.initialization_commands[0].bindings[0].name == "positions"); - REQUIRE(commands.initialization_commands[0].bindings[1].name == "control"); - REQUIRE(commands.initialization_commands[1].bindings[0].name == "control"); - REQUIRE(commands.initialization_commands[1].bindings[4].name == "attention_mask"); - REQUIRE(ggml::hrx::find_transient_allocation(commands.transients, context_capture.bindings[1].value) != - nullptr); - - ggml::hrx::CommandProgram initialization_commands = copy_command_program_shape(commands); - initialization_commands.commands.clear(); - REQUIRE(command_program_verifies(initialization_commands)); + REQUIRE(commands.initialization_commands.empty()); } ggml_free(ctx); diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index fb3dc56f5f9e..94e356c56d55 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -4847,80 +4847,6 @@ static void run_asymmetric_flash_attention_cpu_reference_case() { restore_environment_value(kDisableQwenDispatchEnv, had_original_env, original_env_value); } -static void run_qwen_full_cache_prefill_flash_attention_scheduling_case(int64_t token_count) { - constexpr int64_t kQueryHeadCount = 32; - constexpr int64_t kKeyValueHeadCount = 4; - constexpr int64_t kFullCacheRowCount = 40960; - const size_t active_mask_byte_count = - static_cast(token_count) * static_cast(token_count) * sizeof(ggml_fp16_t); - - ggml_init_params params = {}; - params.mem_size = 128 * 1024 * 1024; - params.no_alloc = true; - ggml_context * ctx = ggml_init(params); - REQUIRE(ctx != nullptr); - - AttentionPostprocessGraph attention = - build_attention_postprocess_graph(ctx, token_count, kQueryHeadCount, kKeyValueHeadCount, kFullCacheRowCount); - ggml_tensor * flash_output = append_qwen_full_cache_flash_attention_consumer( - ctx, attention, token_count, kQueryHeadCount, kKeyValueHeadCount, kFullCacheRowCount); - - ggml_cgraph * graph = ggml_new_graph(ctx); - REQUIRE(graph != nullptr); - ggml_build_forward_expand(graph, attention.key_output); - ggml_build_forward_expand(graph, attention.value_output); - ggml_build_forward_expand(graph, flash_output); - - ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*graph); - REQUIRE(imported.valid()); - const ggml::hrx::DispatchRegistry * registry = ggml::hrx::find_dispatch_registry({ "gfx1151" }); - REQUIRE(registry != nullptr); - - std::vector covered_nodes(imported.graph.nodes().size(), false); - ggml::hrx::CommandPlan plan; - ggml::hrx::DispatchMatch postprocess_match; - match_dispatch_at_index(imported.graph, *registry, plan, covered_nodes, - producer_index_for_tensor(imported.graph, attention.query_reshape), postprocess_match); - ggml::hrx::DispatchMatch flash_match; - match_dispatch_at_index(imported.graph, *registry, plan, covered_nodes, - producer_index_for_tensor(imported.graph, flash_output), flash_match); - - const ggml::hrx::Value * mask_value = imported.graph.values().find_tensor(attention.attention_mask); - REQUIRE(mask_value != nullptr); - const ggml::hrx::CommandPlanAlternateValue * compact_mask = - ggml::hrx::find_alternate_value(plan, mask_value->id, GGML_TYPE_F16, active_mask_byte_count); - REQUIRE(compact_mask != nullptr); - - const ggml::hrx::Dispatch * metadata_dispatch = nullptr; - for (const ggml::hrx::Dispatch & dispatch : plan.initialization_dispatches) { - if (kernel_name_for_id(dispatch.kernel.kernel_id) == "qwen3_moe:qwen_attention_metadata") { - metadata_dispatch = &dispatch; - } - } - REQUIRE(metadata_dispatch != nullptr); - REQUIRE(metadata_dispatch->kernel.integer_parameters.at("token_count") == token_count); - REQUIRE(metadata_dispatch->kernel.integer_parameters.at("context_capacity") == token_count); - REQUIRE(metadata_dispatch->bindings.size() == 5); - REQUIRE(metadata_dispatch->bindings[4].value == compact_mask->alternate_value); - REQUIRE(metadata_dispatch->bindings[4].length == active_mask_byte_count); - - const ggml::hrx::Dispatch * flash_dispatch = nullptr; - for (const ggml::hrx::Dispatch & dispatch : plan.dispatches) { - if (kernel_name_for_id(dispatch.kernel.kernel_id) == "loom_libs:ggml_flash_attention_f32_f16_wmma") { - flash_dispatch = &dispatch; - } - } - REQUIRE(flash_dispatch != nullptr); - REQUIRE(flash_dispatch->kernel.integer_parameters.at("query_token_count") == token_count); - REQUIRE(flash_dispatch->kernel.integer_parameters.at("key_value_token_count") == token_count); - REQUIRE(flash_dispatch->bindings.size() == 6); - REQUIRE(flash_dispatch->kernel.compile_parameters.at("ggml.flash_attention.apply_gate") == "0"); - REQUIRE(flash_dispatch->bindings[3].value == compact_mask->alternate_value); - REQUIRE(flash_dispatch->bindings[3].length == active_mask_byte_count); - - ggml_free(ctx); -} - static void run_qwen_decode_split_flash_attention_scheduling_case(int64_t query_token_count, int64_t key_value_token_count, bool use_common_dispatch = false, @@ -7659,7 +7585,7 @@ static void run_rope_unit_factors_cpu_reference_case() { } static std::pair, std::vector> run_attention_postprocess_cpu_reference_case( - bool check_decode = false, bool implicit_rope = false) { + bool check_decode = false, bool implicit_rope = false, bool check_metadata_reuse = false) { ggml_backend_t cpu_backend = init_cpu_backend(); ggml_backend_t hrx_backend = ggml_backend_hrx_init(0); REQUIRE(hrx_backend != nullptr); @@ -7688,7 +7614,16 @@ static std::pair, std::vector> run_attention_postproce build_attention_postprocess_graph(hrx_ctx, token_count, query_head_count, key_value_head_count, cache_row_count, explicit_factors); - + ggml_tensor * cpu_flash = nullptr; + ggml_tensor * hrx_flash = nullptr; + if (check_metadata_reuse) { + cpu_flash = append_qwen_full_cache_flash_attention_consumer( + cpu_ctx, cpu, token_count, query_head_count, key_value_head_count, cache_row_count); + hrx_flash = append_qwen_full_cache_flash_attention_consumer( + hrx_ctx, hrx, token_count, query_head_count, key_value_head_count, cache_row_count); + ggml_set_output(cpu.query_output); + ggml_set_output(hrx.query_output); + } auto hrx_query_buffer = allocate_test_weight(hrx_backend, hrx.query_weight); auto hrx_key_buffer = allocate_test_weight(hrx_backend, hrx.key_weight); @@ -7699,16 +7634,18 @@ static std::pair, std::vector> run_attention_postproce REQUIRE(cpu_graph != nullptr); REQUIRE(hrx_graph != nullptr); // A stable graph UID exercises the cached execution contract across changing inputs. - hrx_graph->uid = check_decode ? 4003 : 4002; + hrx_graph->uid = check_metadata_reuse ? 4001 : check_decode ? 4003 : 4002; ggml_build_forward_expand(cpu_graph, cpu.query_output); ggml_build_forward_expand(cpu_graph, cpu.key_output); ggml_build_forward_expand(cpu_graph, cpu.value_output); ggml_build_forward_expand(hrx_graph, hrx.query_output); ggml_build_forward_expand(hrx_graph, hrx.key_output); ggml_build_forward_expand(hrx_graph, hrx.value_output); - - - + if (check_metadata_reuse) { + ggml_build_forward_expand(cpu_graph, cpu_flash); + ggml_build_forward_expand(hrx_graph, hrx_flash); + REQUIRE(ggml_backend_supports_op(hrx_backend, hrx_flash)); + } require_kernel_subsequence(scheduled_kernel_sequence(hrx_graph), { check_decode ? "qwen3_moe:qwen3_moe_attention_qkv_postprocess_fused_decode" : "qwen3_moe:qwen3_moe_attention_postprocess_f32_f16" }); @@ -7767,18 +7704,33 @@ static std::pair, std::vector> run_attention_postproce std::vector unit_query; std::vector unit_key_cache; for (int round = 0; round < rounds; ++round) { - if (round == 1 && explicit_factors) { + const bool update_factors = round == 1 && explicit_factors; + if (update_factors) { for (size_t channel = 0; channel < factors.size(); ++channel) { factors[channel] = channel % 3 == 0 ? 0.5f : channel % 3 == 1 ? 2.0f : 4.0f; } set_tensor_pair_bytes(cpu_backend, cpu.freq_factors, hrx_backend, hrx.freq_factors, factors.data(), factors.size() * sizeof(float)); } - if (round == 2) { + std::vector supplied_mask; + if (check_metadata_reuse) { + supplied_mask.assign(token_count * cache_row_count, ggml_fp32_to_fp16(-INFINITY)); + for (int64_t token = 0; token < token_count; ++token) { + positions[token] = static_cast(6 + round * token_count + token); + cache_indices[token] = positions[token] % cache_row_count; + // A window of one retains exactly the current physical row, including after wrap. + supplied_mask[token * cache_row_count + cache_indices[token]] = ggml_fp32_to_fp16(0.0f); + } + set_tensor_pair_bytes(cpu_backend, cpu.attention_mask, hrx_backend, hrx.attention_mask, + supplied_mask.data(), supplied_mask.size() * sizeof(ggml_fp16_t)); + } else if (round == 2) { for (int64_t token = 0; token < token_count; ++token) { positions[token] += static_cast(cache_row_count); cache_indices[token] = positions[token] % cache_row_count; } + } + const bool update_metadata = check_metadata_reuse || round == 2; + if (update_metadata) { set_tensor_pair_bytes(cpu_backend, cpu.positions, hrx_backend, hrx.positions, positions.data(), positions.size() * sizeof(int32_t)); set_tensor_pair_bytes(cpu_backend, cpu.key_cache_indices, hrx_backend, hrx.key_cache_indices, @@ -7799,10 +7751,11 @@ static std::pair, std::vector> run_attention_postproce const auto actual_query = get_f32_tensor(hrx_backend, hrx.query_output); const auto actual_key_cache = get_f32_tensor(hrx_backend, hrx.key_cache); + const bool check_factor_rotation = round == 1 && !check_metadata_reuse; if (round == 0) { unit_query = actual_query; unit_key_cache = actual_key_cache; - } else if (round == 1) { + } else if (check_factor_rotation) { auto rotated = [](const std::vector & base, int64_t row, int64_t head_count, int64_t position, const std::vector & divisors) { std::vector expected = base; @@ -7850,7 +7803,7 @@ static std::pair, std::vector> run_attention_postproce actual_factors.size() * sizeof(float)); REQUIRE(actual_factors == factors); } - if (round == 2) { + if (update_metadata) { std::vector actual_positions(positions.size()); std::vector actual_key_indices(cache_indices.size()); std::vector actual_value_indices(cache_indices.size()); @@ -7864,6 +7817,25 @@ static std::pair, std::vector> run_attention_postproce REQUIRE(actual_key_indices == cache_indices); REQUIRE(actual_value_indices == cache_indices); } + if (check_metadata_reuse) { + std::vector actual_mask(supplied_mask.size()); + ggml_backend_tensor_get(hrx.attention_mask, actual_mask.data(), 0, + actual_mask.size() * sizeof(ggml_fp16_t)); + REQUIRE(actual_mask == supplied_mask); + const auto cached_values = get_f32_tensor(hrx_backend, hrx.value_cache); + std::vector expected(token_count * query_size); + for (int64_t token = 0; token < token_count; ++token) { + for (int64_t head = 0; head < query_head_count; ++head) { + const int64_t value_head = head / (query_head_count / key_value_head_count); + for (int64_t channel = 0; channel < kQwenFlashHeadSize; ++channel) { + expected[token * query_size + head * kQwenFlashHeadSize + channel] = + cached_values[cache_indices[token] * key_value_size + + value_head * kQwenFlashHeadSize + channel]; + } + } + } + require_close(get_f32_tensor(hrx_backend, hrx_flash), expected, 2.0e-3f, 1.0e-3f); + } ggml_backend_hrx_cache_stats stats = {}; REQUIRE(ggml_backend_hrx_get_cache_stats(hrx_backend, &stats)); if (round == 0) { @@ -8179,15 +8151,22 @@ static void run_decode_attention_qkv_scheduling_case() { REQUIRE(ctx != nullptr); AttentionPostprocessGraph graph = build_decode_attention_qkv_graph(ctx); + ggml_tensor * flash = append_qwen_full_cache_flash_attention_consumer(ctx, graph, 1, 32, 4, 1024); ggml_cgraph * cgraph = ggml_new_graph(ctx); REQUIRE(cgraph != nullptr); ggml_build_forward_expand(cgraph, graph.query_output); ggml_build_forward_expand(cgraph, graph.key_output); ggml_build_forward_expand(cgraph, graph.value_output); + ggml_build_forward_expand(cgraph, flash); require_kernel_subsequence(scheduled_kernel_sequence(cgraph), { "qwen3_moe:qwen3_moe_rmsnorm_f32_quantize_q8_1_x4", "qwen3_moe:qwen3_moe_attention_qkv_postprocess_fused_decode" }); + auto imported = ggml::hrx::import_ggml_graph(*cgraph); + REQUIRE(imported.valid()); + ggml::hrx::DispatchScheduler scheduler; + REQUIRE(scheduler.schedule_graph(imported.graph, { "gfx1151" })); + REQUIRE(scheduler.plan().initialization_dispatches.empty()); ggml_free(ctx); } @@ -8913,6 +8892,8 @@ static void register_rmsnorm_and_scheduling_cases(Suite & suite) { require_close(explicit_unit.second, implicit.second, 1.0e-3f, 1.0e-3f); }); suite.device_case("attention_postprocess.decode_factors", [] { run_attention_postprocess_cpu_reference_case(true); }); + suite.device_case("attention_postprocess.metadata_reuse", + [] { run_attention_postprocess_cpu_reference_case(false, false, true); }); suite.device_case("routed_moe.q4_k.next_rmsnorm", [] { run_routed_moe_cpu_reference_case(GGML_TYPE_Q4_K, true); }); suite.host_case("routed_moe.codebook_admission", [] { run_routed_codebook_admission_checks(); }); suite.device_case("routed_pair.iq3_xxs.iq3_xxs.skinny.cpu_reference", [] { @@ -9032,8 +9013,6 @@ static void register_flash_attention_cases(Suite & suite) { [] { run_qwen_decode_attention_output_next_q8_scheduling_case(false); }); suite.device_case("qwen_decode_attention_output_next_q8.selectors", [] { run_qwen_decode_attention_output_next_q8_scheduling_case(true); }); - suite.device_case("qwen_full_cache_prefill_flash_attention.tokens512", - [] { run_qwen_full_cache_prefill_flash_attention_scheduling_case(512); }); } static void register_hrx_ops_cases(Suite & suite) { From 0f65778f72f8e6ad05fded6b1afbd7df6c946906 Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Thu, 5 Nov 2026 03:25:44 +0000 Subject: [PATCH 07/10] test(hrx): separate metadata reuse from RoPE factor checks Keep explicit factors fixed in the metadata regression and share input initialization. Restore the independent RoPE loop without a metadata mode or skipped scalar assertion. Assisted-by: Codex --- tests/hrx-ops-test.cpp | 283 ++++++++++++++++++++++++++++------------- 1 file changed, 197 insertions(+), 86 deletions(-) diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index 94e356c56d55..eceafc2165db 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -7584,8 +7584,50 @@ static void run_rope_unit_factors_cpu_reference_case() { ggml_backend_free(cpu_backend); } +static void initialize_attention_postprocess_data(ggml_backend_t cpu_backend, + const AttentionPostprocessGraph & cpu, + ggml_backend_t hrx_backend, + const AttentionPostprocessGraph & hrx) { + const int64_t token_count = cpu.positions->ne[0]; + const int64_t query_size = cpu.query_weight->ne[1]; + const int64_t key_value_size = cpu.key_weight->ne[1]; + const int64_t cache_row_count = cpu.key_cache->ne[1]; + const std::vector input = make_pattern_f32(kQwenHiddenSize * token_count, 10, 0.01f); + const std::vector query_w = make_quantized_rows(GGML_TYPE_Q4_K, kQwenHiddenSize, query_size, 11); + const std::vector key_w = make_quantized_rows(GGML_TYPE_Q4_K, kQwenHiddenSize, key_value_size, 12); + const std::vector value_w = make_quantized_rows(GGML_TYPE_Q6_K, kQwenHiddenSize, key_value_size, 13); + const std::vector query_nw = make_weight(kQwenFlashHeadSize); + const std::vector key_nw = make_pattern_f32(kQwenFlashHeadSize, 14, 0.02f); + const std::vector key_cache(static_cast(key_value_size * cache_row_count), + ggml_fp32_to_fp16(0.0f)); + const std::vector value_cache(static_cast(key_value_size * cache_row_count), + ggml_fp32_to_fp16(0.0f)); + + if (cpu.hidden_state != nullptr) { + const auto attention_weight = make_weight(kQwenHiddenSize); + set_tensor_pair_bytes(cpu_backend, cpu.hidden_state, hrx_backend, hrx.hidden_state, + input.data(), input.size() * sizeof(float)); + set_tensor_pair_bytes(cpu_backend, cpu.attention_weight, hrx_backend, hrx.attention_weight, + attention_weight.data(), attention_weight.size() * sizeof(float)); + } else { + set_tensor_pair_bytes(cpu_backend, cpu.input, hrx_backend, hrx.input, + input.data(), input.size() * sizeof(float)); + } + set_tensor_pair_bytes(cpu_backend, cpu.query_weight, hrx_backend, hrx.query_weight, query_w.data(), query_w.size()); + set_tensor_pair_bytes(cpu_backend, cpu.key_weight, hrx_backend, hrx.key_weight, key_w.data(), key_w.size()); + set_tensor_pair_bytes(cpu_backend, cpu.value_weight, hrx_backend, hrx.value_weight, value_w.data(), value_w.size()); + set_tensor_pair_bytes(cpu_backend, cpu.query_norm_weight, hrx_backend, hrx.query_norm_weight, query_nw.data(), + query_nw.size() * sizeof(float)); + set_tensor_pair_bytes(cpu_backend, cpu.key_norm_weight, hrx_backend, hrx.key_norm_weight, key_nw.data(), + key_nw.size() * sizeof(float)); + set_tensor_pair_bytes(cpu_backend, cpu.key_cache, hrx_backend, hrx.key_cache, key_cache.data(), + key_cache.size() * sizeof(ggml_fp16_t)); + set_tensor_pair_bytes(cpu_backend, cpu.value_cache, hrx_backend, hrx.value_cache, value_cache.data(), + value_cache.size() * sizeof(ggml_fp16_t)); +} + static std::pair, std::vector> run_attention_postprocess_cpu_reference_case( - bool check_decode = false, bool implicit_rope = false, bool check_metadata_reuse = false) { + bool check_decode = false, bool implicit_rope = false) { ggml_backend_t cpu_backend = init_cpu_backend(); ggml_backend_t hrx_backend = ggml_backend_hrx_init(0); REQUIRE(hrx_backend != nullptr); @@ -7614,17 +7656,6 @@ static std::pair, std::vector> run_attention_postproce build_attention_postprocess_graph(hrx_ctx, token_count, query_head_count, key_value_head_count, cache_row_count, explicit_factors); - ggml_tensor * cpu_flash = nullptr; - ggml_tensor * hrx_flash = nullptr; - if (check_metadata_reuse) { - cpu_flash = append_qwen_full_cache_flash_attention_consumer( - cpu_ctx, cpu, token_count, query_head_count, key_value_head_count, cache_row_count); - hrx_flash = append_qwen_full_cache_flash_attention_consumer( - hrx_ctx, hrx, token_count, query_head_count, key_value_head_count, cache_row_count); - ggml_set_output(cpu.query_output); - ggml_set_output(hrx.query_output); - } - auto hrx_query_buffer = allocate_test_weight(hrx_backend, hrx.query_weight); auto hrx_key_buffer = allocate_test_weight(hrx_backend, hrx.key_weight); auto hrx_value_buffer = allocate_test_weight(hrx_backend, hrx.value_weight); @@ -7634,18 +7665,14 @@ static std::pair, std::vector> run_attention_postproce REQUIRE(cpu_graph != nullptr); REQUIRE(hrx_graph != nullptr); // A stable graph UID exercises the cached execution contract across changing inputs. - hrx_graph->uid = check_metadata_reuse ? 4001 : check_decode ? 4003 : 4002; + hrx_graph->uid = check_decode ? 4003 : 4002; ggml_build_forward_expand(cpu_graph, cpu.query_output); ggml_build_forward_expand(cpu_graph, cpu.key_output); ggml_build_forward_expand(cpu_graph, cpu.value_output); ggml_build_forward_expand(hrx_graph, hrx.query_output); ggml_build_forward_expand(hrx_graph, hrx.key_output); ggml_build_forward_expand(hrx_graph, hrx.value_output); - if (check_metadata_reuse) { - ggml_build_forward_expand(cpu_graph, cpu_flash); - ggml_build_forward_expand(hrx_graph, hrx_flash); - REQUIRE(ggml_backend_supports_op(hrx_backend, hrx_flash)); - } + require_kernel_subsequence(scheduled_kernel_sequence(hrx_graph), { check_decode ? "qwen3_moe:qwen3_moe_attention_qkv_postprocess_fused_decode" : "qwen3_moe:qwen3_moe_attention_postprocess_f32_f16" }); @@ -7655,45 +7682,15 @@ static std::pair, std::vector> run_attention_postproce REQUIRE(cpu_buffer != nullptr); REQUIRE(hrx_buffer != nullptr); - const std::vector input = make_pattern_f32(kQwenHiddenSize * token_count, 10, 0.01f); - const std::vector query_w = make_quantized_rows(GGML_TYPE_Q4_K, kQwenHiddenSize, query_size, 11); - const std::vector key_w = make_quantized_rows(GGML_TYPE_Q4_K, kQwenHiddenSize, key_value_size, 12); - const std::vector value_w = make_quantized_rows(GGML_TYPE_Q6_K, kQwenHiddenSize, key_value_size, 13); - const std::vector query_nw = make_weight(kQwenFlashHeadSize); - const std::vector key_nw = make_pattern_f32(kQwenFlashHeadSize, 14, 0.02f); std::vector positions = check_decode ? std::vector{ 8 } : std::vector{ 8, 9 }; std::vector factors(static_cast(kQwenFlashHeadSize / 2), 1.0f); - const std::vector key_cache(static_cast(key_value_size * cache_row_count), - ggml_fp32_to_fp16(0.0f)); - const std::vector value_cache(static_cast(key_value_size * cache_row_count), - ggml_fp32_to_fp16(0.0f)); std::vector cache_indices = make_i64_mod_data(token_count, cache_row_count); - if (check_decode) { - const auto attention_weight = make_weight(kQwenHiddenSize); - set_tensor_pair_bytes(cpu_backend, cpu.hidden_state, hrx_backend, hrx.hidden_state, - input.data(), input.size() * sizeof(float)); - set_tensor_pair_bytes(cpu_backend, cpu.attention_weight, hrx_backend, hrx.attention_weight, - attention_weight.data(), attention_weight.size() * sizeof(float)); - } else { - set_tensor_pair_bytes(cpu_backend, cpu.input, hrx_backend, hrx.input, - input.data(), input.size() * sizeof(float)); - } - set_tensor_pair_bytes(cpu_backend, cpu.query_weight, hrx_backend, hrx.query_weight, query_w.data(), query_w.size()); - set_tensor_pair_bytes(cpu_backend, cpu.key_weight, hrx_backend, hrx.key_weight, key_w.data(), key_w.size()); - set_tensor_pair_bytes(cpu_backend, cpu.value_weight, hrx_backend, hrx.value_weight, value_w.data(), value_w.size()); - set_tensor_pair_bytes(cpu_backend, cpu.query_norm_weight, hrx_backend, hrx.query_norm_weight, query_nw.data(), - query_nw.size() * sizeof(float)); - set_tensor_pair_bytes(cpu_backend, cpu.key_norm_weight, hrx_backend, hrx.key_norm_weight, key_nw.data(), - key_nw.size() * sizeof(float)); + initialize_attention_postprocess_data(cpu_backend, cpu, hrx_backend, hrx); set_tensor_pair_bytes(cpu_backend, cpu.positions, hrx_backend, hrx.positions, positions.data(), positions.size() * sizeof(int32_t)); set_tensor_pair_bytes(cpu_backend, cpu.freq_factors, hrx_backend, hrx.freq_factors, factors.data(), factors.size() * sizeof(float)); - set_tensor_pair_bytes(cpu_backend, cpu.key_cache, hrx_backend, hrx.key_cache, key_cache.data(), - key_cache.size() * sizeof(ggml_fp16_t)); - set_tensor_pair_bytes(cpu_backend, cpu.value_cache, hrx_backend, hrx.value_cache, value_cache.data(), - value_cache.size() * sizeof(ggml_fp16_t)); set_tensor_pair_bytes(cpu_backend, cpu.key_cache_indices, hrx_backend, hrx.key_cache_indices, cache_indices.data(), cache_indices.size() * sizeof(int64_t)); set_tensor_pair_bytes(cpu_backend, cpu.value_cache_indices, hrx_backend, hrx.value_cache_indices, @@ -7704,7 +7701,8 @@ static std::pair, std::vector> run_attention_postproce std::vector unit_query; std::vector unit_key_cache; for (int round = 0; round < rounds; ++round) { - const bool update_factors = round == 1 && explicit_factors; + const bool factor_change_round = round == 1; + const bool update_factors = factor_change_round && explicit_factors; if (update_factors) { for (size_t channel = 0; channel < factors.size(); ++channel) { factors[channel] = channel % 3 == 0 ? 0.5f : channel % 3 == 1 ? 2.0f : 4.0f; @@ -7712,25 +7710,11 @@ static std::pair, std::vector> run_attention_postproce set_tensor_pair_bytes(cpu_backend, cpu.freq_factors, hrx_backend, hrx.freq_factors, factors.data(), factors.size() * sizeof(float)); } - std::vector supplied_mask; - if (check_metadata_reuse) { - supplied_mask.assign(token_count * cache_row_count, ggml_fp32_to_fp16(-INFINITY)); - for (int64_t token = 0; token < token_count; ++token) { - positions[token] = static_cast(6 + round * token_count + token); - cache_indices[token] = positions[token] % cache_row_count; - // A window of one retains exactly the current physical row, including after wrap. - supplied_mask[token * cache_row_count + cache_indices[token]] = ggml_fp32_to_fp16(0.0f); - } - set_tensor_pair_bytes(cpu_backend, cpu.attention_mask, hrx_backend, hrx.attention_mask, - supplied_mask.data(), supplied_mask.size() * sizeof(ggml_fp16_t)); - } else if (round == 2) { + if (round == 2) { for (int64_t token = 0; token < token_count; ++token) { positions[token] += static_cast(cache_row_count); cache_indices[token] = positions[token] % cache_row_count; } - } - const bool update_metadata = check_metadata_reuse || round == 2; - if (update_metadata) { set_tensor_pair_bytes(cpu_backend, cpu.positions, hrx_backend, hrx.positions, positions.data(), positions.size() * sizeof(int32_t)); set_tensor_pair_bytes(cpu_backend, cpu.key_cache_indices, hrx_backend, hrx.key_cache_indices, @@ -7751,11 +7735,10 @@ static std::pair, std::vector> run_attention_postproce const auto actual_query = get_f32_tensor(hrx_backend, hrx.query_output); const auto actual_key_cache = get_f32_tensor(hrx_backend, hrx.key_cache); - const bool check_factor_rotation = round == 1 && !check_metadata_reuse; if (round == 0) { unit_query = actual_query; unit_key_cache = actual_key_cache; - } else if (check_factor_rotation) { + } else if (round == 1) { auto rotated = [](const std::vector & base, int64_t row, int64_t head_count, int64_t position, const std::vector & divisors) { std::vector expected = base; @@ -7803,7 +7786,7 @@ static std::pair, std::vector> run_attention_postproce actual_factors.size() * sizeof(float)); REQUIRE(actual_factors == factors); } - if (update_metadata) { + if (round == 2) { std::vector actual_positions(positions.size()); std::vector actual_key_indices(cache_indices.size()); std::vector actual_value_indices(cache_indices.size()); @@ -7817,25 +7800,154 @@ static std::pair, std::vector> run_attention_postproce REQUIRE(actual_key_indices == cache_indices); REQUIRE(actual_value_indices == cache_indices); } - if (check_metadata_reuse) { - std::vector actual_mask(supplied_mask.size()); - ggml_backend_tensor_get(hrx.attention_mask, actual_mask.data(), 0, - actual_mask.size() * sizeof(ggml_fp16_t)); - REQUIRE(actual_mask == supplied_mask); - const auto cached_values = get_f32_tensor(hrx_backend, hrx.value_cache); - std::vector expected(token_count * query_size); - for (int64_t token = 0; token < token_count; ++token) { - for (int64_t head = 0; head < query_head_count; ++head) { - const int64_t value_head = head / (query_head_count / key_value_head_count); - for (int64_t channel = 0; channel < kQwenFlashHeadSize; ++channel) { - expected[token * query_size + head * kQwenFlashHeadSize + channel] = - cached_values[cache_indices[token] * key_value_size + - value_head * kQwenFlashHeadSize + channel]; - } + ggml_backend_hrx_cache_stats stats = {}; + REQUIRE(ggml_backend_hrx_get_cache_stats(hrx_backend, &stats)); + if (round == 0) { + first_stats = stats; + } else { + REQUIRE(stats.prepared_program_builds == first_stats.prepared_program_builds); + REQUIRE(stats.graph_program_builds == first_stats.graph_program_builds); + REQUIRE(stats.graph_program_hits > first_stats.graph_program_hits); + } + } + + ggml_backend_buffer_free(cpu_buffer); + ggml_backend_buffer_free(hrx_buffer); + ggml_free(cpu_ctx); + ggml_free(hrx_ctx); + ggml_backend_free(cpu_backend); + ggml_backend_free(hrx_backend); + return { unit_query, unit_key_cache }; +} + +static void run_attention_postprocess_metadata_reuse_case() { + ggml_backend_t cpu_backend = init_cpu_backend(); + ggml_backend_t hrx_backend = ggml_backend_hrx_init(0); + REQUIRE(hrx_backend != nullptr); + + ggml_init_params params = {}; + params.mem_size = 32 * 1024 * 1024; + params.no_alloc = true; + ggml_context * cpu_ctx = ggml_init(params); + ggml_context * hrx_ctx = ggml_init(params); + REQUIRE(cpu_ctx != nullptr); + REQUIRE(hrx_ctx != nullptr); + + constexpr int64_t token_count = 2; + constexpr int64_t query_head_count = 4; + constexpr int64_t key_value_head_count = 2; + constexpr int64_t cache_row_count = 8; + constexpr int64_t query_size = query_head_count * kQwenFlashHeadSize; + constexpr int64_t key_value_size = key_value_head_count * kQwenFlashHeadSize; + AttentionPostprocessGraph cpu = build_attention_postprocess_graph( + cpu_ctx, token_count, query_head_count, key_value_head_count, cache_row_count); + AttentionPostprocessGraph hrx = build_attention_postprocess_graph( + hrx_ctx, token_count, query_head_count, key_value_head_count, cache_row_count); + ggml_tensor * cpu_flash = append_qwen_full_cache_flash_attention_consumer( + cpu_ctx, cpu, token_count, query_head_count, key_value_head_count, cache_row_count); + ggml_tensor * hrx_flash = append_qwen_full_cache_flash_attention_consumer( + hrx_ctx, hrx, token_count, query_head_count, key_value_head_count, cache_row_count); + ggml_set_output(cpu.query_output); + ggml_set_output(hrx.query_output); + + auto hrx_query_buffer = allocate_test_weight(hrx_backend, hrx.query_weight); + auto hrx_key_buffer = allocate_test_weight(hrx_backend, hrx.key_weight); + auto hrx_value_buffer = allocate_test_weight(hrx_backend, hrx.value_weight); + + ggml_cgraph * cpu_graph = ggml_new_graph(cpu_ctx); + ggml_cgraph * hrx_graph = ggml_new_graph(hrx_ctx); + REQUIRE(cpu_graph != nullptr); + REQUIRE(hrx_graph != nullptr); + // A stable graph UID exercises the cached execution contract across changing inputs. + hrx_graph->uid = 4001; + ggml_build_forward_expand(cpu_graph, cpu.query_output); + ggml_build_forward_expand(cpu_graph, cpu.key_output); + ggml_build_forward_expand(cpu_graph, cpu.value_output); + ggml_build_forward_expand(hrx_graph, hrx.query_output); + ggml_build_forward_expand(hrx_graph, hrx.key_output); + ggml_build_forward_expand(hrx_graph, hrx.value_output); + ggml_build_forward_expand(cpu_graph, cpu_flash); + ggml_build_forward_expand(hrx_graph, hrx_flash); + REQUIRE(ggml_backend_supports_op(hrx_backend, hrx_flash)); + require_kernel_subsequence(scheduled_kernel_sequence(hrx_graph), + { "qwen3_moe:qwen3_moe_attention_postprocess_f32_f16" }); + + ggml_backend_buffer_t cpu_buffer = ggml_backend_alloc_ctx_tensors(cpu_ctx, cpu_backend); + ggml_backend_buffer_t hrx_buffer = ggml_backend_alloc_ctx_tensors(hrx_ctx, hrx_backend); + REQUIRE(cpu_buffer != nullptr); + REQUIRE(hrx_buffer != nullptr); + + initialize_attention_postprocess_data(cpu_backend, cpu, hrx_backend, hrx); + // Keep factors fixed so each execution changes only caller attention metadata. + const std::vector factors(static_cast(kQwenFlashHeadSize / 2), 1.0f); + set_tensor_pair_bytes(cpu_backend, cpu.freq_factors, hrx_backend, hrx.freq_factors, + factors.data(), factors.size() * sizeof(float)); + std::vector positions(token_count); + std::vector cache_indices(token_count); + ggml_backend_hrx_cache_stats first_stats = {}; + for (int round = 0; round < 3; ++round) { + std::vector supplied_mask(token_count * cache_row_count, ggml_fp32_to_fp16(-INFINITY)); + for (int64_t token = 0; token < token_count; ++token) { + positions[token] = static_cast(6 + round * token_count + token); + cache_indices[token] = positions[token] % cache_row_count; + // A window of one retains exactly the current physical row, including after wrap. + supplied_mask[token * cache_row_count + cache_indices[token]] = ggml_fp32_to_fp16(0.0f); + } + set_tensor_pair_bytes(cpu_backend, cpu.attention_mask, hrx_backend, hrx.attention_mask, + supplied_mask.data(), supplied_mask.size() * sizeof(ggml_fp16_t)); + set_tensor_pair_bytes(cpu_backend, cpu.positions, hrx_backend, hrx.positions, + positions.data(), positions.size() * sizeof(int32_t)); + set_tensor_pair_bytes(cpu_backend, cpu.key_cache_indices, hrx_backend, hrx.key_cache_indices, + cache_indices.data(), cache_indices.size() * sizeof(int64_t)); + set_tensor_pair_bytes(cpu_backend, cpu.value_cache_indices, hrx_backend, hrx.value_cache_indices, + cache_indices.data(), cache_indices.size() * sizeof(int64_t)); + REQUIRE(ggml_backend_graph_compute(cpu_backend, cpu_graph) == GGML_STATUS_SUCCESS); + REQUIRE(ggml_backend_graph_compute(hrx_backend, hrx_graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(cpu_backend); + ggml_backend_synchronize(hrx_backend); + require_close(get_f32_tensor(hrx_backend, hrx.query_output), get_f32_tensor(cpu_backend, cpu.query_output), 2.0f, + 5.0e-2f); + require_close(get_f32_tensor(hrx_backend, hrx.key_output), get_f32_tensor(cpu_backend, cpu.key_output), 2.0f, + 5.0e-2f); + require_close(get_f32_tensor(hrx_backend, hrx.value_output), get_f32_tensor(cpu_backend, cpu.value_output), 2.0f, + 5.0e-2f); + + std::vector actual_factors(factors.size()); + ggml_backend_tensor_get(hrx.freq_factors, actual_factors.data(), 0, + actual_factors.size() * sizeof(float)); + REQUIRE(actual_factors == factors); + + std::vector actual_positions(positions.size()); + std::vector actual_key_indices(cache_indices.size()); + std::vector actual_value_indices(cache_indices.size()); + ggml_backend_tensor_get(hrx.positions, actual_positions.data(), 0, + actual_positions.size() * sizeof(int32_t)); + ggml_backend_tensor_get(hrx.key_cache_indices, actual_key_indices.data(), 0, + actual_key_indices.size() * sizeof(int64_t)); + ggml_backend_tensor_get(hrx.value_cache_indices, actual_value_indices.data(), 0, + actual_value_indices.size() * sizeof(int64_t)); + REQUIRE(actual_positions == positions); + REQUIRE(actual_key_indices == cache_indices); + REQUIRE(actual_value_indices == cache_indices); + + std::vector actual_mask(supplied_mask.size()); + ggml_backend_tensor_get(hrx.attention_mask, actual_mask.data(), 0, + actual_mask.size() * sizeof(ggml_fp16_t)); + REQUIRE(actual_mask == supplied_mask); + + const auto cached_values = get_f32_tensor(hrx_backend, hrx.value_cache); + std::vector expected(token_count * query_size); + for (int64_t token = 0; token < token_count; ++token) { + for (int64_t head = 0; head < query_head_count; ++head) { + const int64_t value_head = head / (query_head_count / key_value_head_count); + for (int64_t channel = 0; channel < kQwenFlashHeadSize; ++channel) { + expected[token * query_size + head * kQwenFlashHeadSize + channel] = + cached_values[cache_indices[token] * key_value_size + + value_head * kQwenFlashHeadSize + channel]; } } - require_close(get_f32_tensor(hrx_backend, hrx_flash), expected, 2.0e-3f, 1.0e-3f); } + require_close(get_f32_tensor(hrx_backend, hrx_flash), expected, 2.0e-3f, 1.0e-3f); ggml_backend_hrx_cache_stats stats = {}; REQUIRE(ggml_backend_hrx_get_cache_stats(hrx_backend, &stats)); if (round == 0) { @@ -7853,7 +7965,6 @@ static std::pair, std::vector> run_attention_postproce ggml_free(hrx_ctx); ggml_backend_free(cpu_backend); ggml_backend_free(hrx_backend); - return { unit_query, unit_key_cache }; } static void run_routed_moe_cpu_reference_case(ggml_type down_weight_type, bool include_next_rmsnorm) { @@ -8893,7 +9004,7 @@ static void register_rmsnorm_and_scheduling_cases(Suite & suite) { }); suite.device_case("attention_postprocess.decode_factors", [] { run_attention_postprocess_cpu_reference_case(true); }); suite.device_case("attention_postprocess.metadata_reuse", - [] { run_attention_postprocess_cpu_reference_case(false, false, true); }); + [] { run_attention_postprocess_metadata_reuse_case(); }); suite.device_case("routed_moe.q4_k.next_rmsnorm", [] { run_routed_moe_cpu_reference_case(GGML_TYPE_Q4_K, true); }); suite.host_case("routed_moe.codebook_admission", [] { run_routed_codebook_admission_checks(); }); suite.device_case("routed_pair.iq3_xxs.iq3_xxs.skinny.cpu_reference", [] { From 534c30d3c3e5f8c34aafbed85d4d5aa15ce72da5 Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Thu, 5 Nov 2026 04:44:59 +0000 Subject: [PATCH 08/10] experiment: fuse activation max scaling and division --- .../common/dispatch-pool-1d.cpp | 172 ++++++++++++++++++ .../kernels/loom-libs/manifest.json | 52 ++++++ .../ops/reduction/amax_scale_f32.loom | 60 ++++++ tests/hrx-backend-test.cpp | 123 +++++++++++++ 4 files changed, 407 insertions(+) create mode 100644 ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/reduction/amax_scale_f32.loom diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.cpp index 7abe4f3c72f9..c8ef14521ecb 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.cpp @@ -1,6 +1,7 @@ #include "dispatch-pool-1d.h" #include "ggml.h" +#include "graph/graph-matcher.h" #include "kernel-corpus/kernel-corpus-catalog-verify.h" #include @@ -12,6 +13,169 @@ namespace { static constexpr KernelCatalogRef kFullMaxPoolF32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_pool_1d_full_max_f32"); +static constexpr KernelCatalogRef kAmaxScaleF32Kernel = + GGML_HRX_KERNEL_REF("loom_libs", "ggml_amax_scale_f32"); + +static bool matches_unary(const GraphNode & node, UnaryKind kind) { + const UnaryParams * params = op_params_as(node.params); + if (params == nullptr) { + return false; + } + return params->op == kind; +} + +static bool matches_scale(const GraphNode & node, float factor) { + const ScaleParams * params = op_params_as(node.params); + if (params == nullptr) { + return false; + } + const bool same_factor = params->scale == factor; + const bool no_bias = params->bias == 0.0f; + return same_factor && no_bias; +} + +static bool match_amax_scale_f32(const DispatchMatchContext & context, DispatchMatch & match) { + const GraphNode * node = context.root_node; + if (node == nullptr) { + return false; + } + const Graph & graph = context.graph; + // Intermediate values must stay private: this dispatch publishes only x/s and s. + const ggml_op ops[] = { GGML_OP_UNARY, GGML_OP_POOL_1D, GGML_OP_SCALE, GGML_OP_LOG, + GGML_OP_SCALE, GGML_OP_UNARY, GGML_OP_CLAMP, GGML_OP_SCALE, GGML_OP_UNARY }; + std::vector chain; + for (size_t i = 0; i < 9; ++i) { + const bool right_op = node->op == ops[i]; + const bool one_input = node->inputs.size() == 1; + const bool expected_node = right_op && one_input; + if (!expected_node) { + return false; + } + const Value * value = graph.values().find(node->output); + if (value == nullptr) { + return false; + } + const bool f32 = value->type == GGML_TYPE_F32; + const bool contiguous = value->contiguous; + const bool supported_value = f32 && contiguous; + if (!supported_value) { + return false; + } + chain.push_back(node); + if (i == 8) { + break; + } + const auto & consumers = graph.index().consumers(node->output); + const bool private_value = value->kind == ValueKind::Transient; + const bool one_consumer = consumers.size() == 1; + const bool can_elide = private_value && one_consumer; + if (!can_elide) { + return false; + } + node = consumers.front(); + } + const bool abs_input = matches_unary(*chain[0], UnaryKind::Abs); + const bool scale_amax = matches_scale(*chain[2], 1.0f / 1024.0f); + const bool logarithm = matches_unary(*chain[3], UnaryKind::Log); + const bool scale_log = matches_scale(*chain[4], 1.4426950408889634f); + const bool ceil_log = matches_unary(*chain[5], UnaryKind::Ceil); + const bool scale_exp = matches_scale(*chain[7], 0.6931471805599453f); + const bool exponential = matches_unary(*chain[8], UnaryKind::Exp); + const bool exact_math = abs_input && scale_amax && logarithm && scale_log && ceil_log && scale_exp && exponential; + if (!exact_math) { + return false; + } + const Pool1DParams * pool = op_params_as(chain[1]->params); + const ClampParams * clamp = op_params_as(chain[6]->params); + const Value * input = graph.values().find(chain[0]->inputs[0]); + const bool has_pool = pool != nullptr; + const bool has_clamp = clamp != nullptr; + const bool has_input = input != nullptr; + const bool all_present = has_pool && has_clamp && has_input; + if (!all_present) { + return false; + } + const bool max_pool = pool->op == GGML_OP_POOL_MAX; + const bool full_width = pool->kernel == input->ne[0]; + const bool full_stride = pool->stride == pool->kernel; + const bool no_padding = pool->padding == 0; + const bool clamp_min = clamp->min == 0.0f; + const bool clamp_max = clamp->max == 127.0f; + const bool input_f32 = input->type == GGML_TYPE_F32; + const bool bounded_width = input->ne[0] > 0 && input->ne[0] <= 32768; + const bool bounded_elements = input->element_count > 0 && input->element_count <= 134217728; + const bool supported = max_pool && full_width && full_stride && no_padding && clamp_min && clamp_max && + input_f32 && input->contiguous && bounded_width && bounded_elements; + if (!supported) { + return false; + } + const Value * scale = graph.values().find(chain.back()->output); + const int64_t rows = input->element_count / input->ne[0]; + const bool bounded_rows = rows <= 1048576; + if (!bounded_rows) { + return false; + } + for (size_t i = 0; i < chain.size(); ++i) { + const Value * value = graph.values().find(chain[i]->output); + auto expected_shape = input->ne; + if (i > 0) { + expected_shape[0] = 1; + } + const bool same_shape = value->ne == expected_shape; + const bool separate_input = value->storage_root != input->storage_root; + const bool valid_output = same_shape && separate_input; + if (!valid_output) { + return false; + } + } + const GraphNode * division = nullptr; + for (const GraphNode * consumer : graph.index().consumers(scale->id)) { + const bool is_div = consumer->op == GGML_OP_DIV; + const bool two_inputs = consumer->inputs.size() == 2; + const bool candidate = is_div && two_inputs; + if (!candidate) { + continue; + } + const bool numerator = consumer->inputs[0] == input->id; + const bool denominator = consumer->inputs[1] == scale->id; + const bool exact_division = numerator && denominator; + if (exact_division) { + division = consumer; + break; + } + } + if (division == nullptr) { + return false; + } + const Value * output = graph.values().find(division->output); + if (output == nullptr) { + return false; + } + const bool output_f32 = output->type == GGML_TYPE_F32; + const bool same_shape = output->ne == input->ne; + const bool separate_input = output->storage_root != input->storage_root; + const bool separate_scale = output->storage_root != scale->storage_root; + const bool valid_output = output_f32 && output->contiguous && same_shape && separate_input && separate_scale; + if (!valid_output) { + return false; + } + chain.push_back(division); + for (const GraphNode * covered : chain) { + if (!append_covered_node_index_once(graph, context.covered_nodes, covered, match.covered_nodes)) { + return false; + } + } + Dispatch dispatch; + dispatch.kernel = make_kernel_specialization(kAmaxScaleF32Kernel); + dispatch.kernel.integer_parameters.emplace("row_count", rows); + dispatch.kernel.integer_parameters.emplace("row_width", input->ne[0]); + dispatch.bindings.push_back({ input->id, 0, input->byte_count }); + dispatch.bindings.push_back({ output->id, 0, output->byte_count }); + dispatch.bindings.push_back({ scale->id, 0, scale->byte_count }); + match.dispatches.push_back(std::move(dispatch)); + return true; +} + static bool match_full_max_pool_1d_f32(const DispatchMatchContext & context, DispatchMatch & match) { const GraphNode * node = context.root_node; if (node == nullptr) { @@ -65,6 +229,14 @@ static bool match_full_max_pool_1d_f32(const DispatchMatchContext & context, Dis } // namespace void register_pool_1d_dispatch(DispatchRegistryBuilder & registry) { + registry.add({ + "common.amax_scale_f32", + GGML_OP_UNARY, + DispatchMatchKind::Fused, + 400, + DispatchSource::Common, + match_amax_scale_f32, + }); registry.add({ "common.pool_1d_full_max_f32", GGML_OP_POOL_1D, diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json index 9002ec9e5a38..73ce5b7d5ffe 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json @@ -658,6 +658,9 @@ }, { "path": "motifs/matmul/tiled_pair_postprocess_f32.loom" + }, + { + "path": "ops/reduction/amax_scale_f32.loom" } ], "exports": [ @@ -9666,6 +9669,55 @@ "motifs/bf16_f16.loom", "motifs/f32_f16.loom" ] + }, + { + "name": "ggml_amax_scale_f32", + "family": "loom_libs", + "symbol": "ggml_amax_scale_f32", + "source": "ops/reduction/amax_scale_f32.loom", + "workload_parameters": [ + { + "name": "row_count", + "type": "index" + }, + { + "name": "row_width", + "type": "index" + } + ], + "launch_parameters": [ + { + "name": "row_count", + "type": "index" + }, + { + "name": "row_width", + "type": "index" + } + ], + "bindings": [ + "input", + "output", + "scale" + ], + "binding_access": [ + "read", + "write", + "write" + ], + "target_selector": "", + "compile_recipe": { + "mode": "direct", + "primary_sources": [ + "ops/reduction/amax_scale_f32.loom" + ], + "library_sources": [ + "motifs/elementwise/unary_apply.loom" + ] + }, + "compile_dependencies": [ + "motifs/elementwise/unary_apply.loom" + ] } ], "link_modules": [ diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/reduction/amax_scale_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/reduction/amax_scale_f32.loom new file mode 100644 index 000000000000..f46f11d02ae9 --- /dev/null +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/reduction/amax_scale_f32.loom @@ -0,0 +1,60 @@ +// Copyright 2026 The HRX Authors +// SPDX-License-Identifier: Apache-2.0 + +template.decl @ggml.unary_f32.apply(%arg0: index, %arg1: f32) -> (f32) + +amdgpu.target @ggml_amax_scale_f32_gfx11_wave64 {subgroup_size = 64} + +kernel.def target(@ggml_amax_scale_f32_gfx11_wave64) export("ggml_amax_scale_f32") @ggml_amax_scale_f32(%row_count: index, %row_width: index) { + %one = index.constant 1 : index + %twofiftysix = index.constant 256 : index + kernel.launch.config workgroups(%row_count, %one, %one) workgroup_size(%twofiftysix, %one, %one) : index +} launch(%row_count: index, %row_width: index, %input: buffer, %output: buffer, %scale: buffer) where [range(%row_count, 1, 1048576), range(%row_width, 1, 32768)] { + %rows = index.assume %row_count [range(%row_count, 1, 1048576)] : index + %width = index.assume %row_width [range(%row_width, 1, 32768)] : index + %row = kernel.workgroup.id : index + %lane = kernel.workitem.id : index + %zero = index.constant 0 : index + %stride = index.constant 256 : index + %zero_f32 = scalar.constant 0.0 : f32 + %abs_op = index.constant 1 : index + %log_op = index.constant 22 : index + %ceil_op = index.constant 7 : index + %exp_op = index.constant 18 : index + %element_count = index.mul %rows, %width : index + %row_base = index.mul %row, %width : index + %zero_offset = index.constant 0 : offset + %input_view = buffer.view %input[%zero_offset] : buffer -> view<[%element_count]xf32> + %output_view = buffer.view %output[%zero_offset] : buffer -> view<[%element_count]xf32> + %scale_view = buffer.view %scale[%zero_offset] : buffer -> view<[%rows]xf32> + %maximum = scf.for %column = [%lane to %width step %stride](%prior = %zero_f32 : f32) -> (f32) { + %offset = index.add %row_base, %column : index + %value = view.load %input_view[%offset] : view<[%element_count]xf32> -> f32 + %absolute = template.apply<@ggml.unary_f32.apply>(%abs_op, %value) : (index, f32) -> (f32) + %next = scalar.maxnumf %absolute, %prior : f32 + scf.yield %next : f32 + } + %row_maximum = kernel.workgroup.reduce %maximum : f32 + %inverse_limit = scalar.constant 0.0009765625 : f32 + %log2_e = scalar.constant 1.4426950408889634 : f32 + %ln_2 = scalar.constant 0.6931471805599453 : f32 + %max_exponent = scalar.constant 127.0 : f32 + %relative = scalar.mulf %row_maximum, %inverse_limit : f32 + %logarithm = template.apply<@ggml.unary_f32.apply>(%log_op, %relative) : (index, f32) -> (f32) + %log2 = scalar.mulf %logarithm, %log2_e : f32 + %rounded = template.apply<@ggml.unary_f32.apply>(%ceil_op, %log2) : (index, f32) -> (f32) + %exponent = scalar.clampf %rounded, %zero_f32, %max_exponent : f32 + %natural_exponent = scalar.mulf %exponent, %ln_2 : f32 + %factor = template.apply<@ggml.unary_f32.apply>(%exp_op, %natural_exponent) : (index, f32) -> (f32) + %is_first = index.cmp eq, %lane, %zero : index + scf.if %is_first { + view.store %factor, %scale_view[%row] : f32, view<[%rows]xf32> + } + scf.for %column = [%lane to %width step %stride] { + %offset = index.add %row_base, %column : index + %value = view.load %input_view[%offset] : view<[%element_count]xf32> -> f32 + %normalized = scalar.divf %value, %factor : f32 + view.store %normalized, %output_view[%offset] : f32, view<[%element_count]xf32> + } + kernel.return +} diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index 39d270fcfa25..a298372370a7 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -13809,6 +13809,127 @@ static void run_scale_f32() { ggml_backend_free(backend); } +struct AmaxScaleTestGraph { + ggml_tensor * input; + ggml_tensor * absolute; + ggml_tensor * scale; + ggml_tensor * output; +}; + +static AmaxScaleTestGraph build_amax_scale_test_graph(ggml_context * ctx, int64_t width, int64_t tokens, + float limit_inverse = 1.0f / 1024.0f) { + ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, width, 8, tokens); + ggml_tensor * absolute = ggml_abs(ctx, input); + ggml_tensor * pooled = ggml_pool_1d(ctx, absolute, GGML_OP_POOL_MAX, width, width, 0); + ggml_tensor * scale = ggml_scale(ctx, pooled, limit_inverse); + scale = ggml_log(ctx, scale); + scale = ggml_scale(ctx, scale, 1.4426950408889634f); + scale = ggml_ceil(ctx, scale); + scale = ggml_clamp(ctx, scale, 0.0f, 127.0f); + scale = ggml_scale(ctx, scale, 0.6931471805599453f); + scale = ggml_exp(ctx, scale); + return { input, absolute, scale, ggml_div(ctx, input, scale) }; +} + +static void run_amax_scale_fusion_dispatch_checks() { + for (int variant = 0; variant < 4; ++variant) { + ggml_context * ctx = ggml_init({ 1024 * 1024, nullptr, true }); + REQUIRE(ctx != nullptr); + const float inverse = variant == 1 ? 1.0f / 512.0f : 1.0f / 1024.0f; + const auto tensors = build_amax_scale_test_graph(ctx, 896, 16, inverse); + if (variant == 2) { + ggml_set_output(tensors.absolute); + } + ggml_cgraph * graph = ggml_new_graph(ctx); + ggml_build_forward_expand(graph, tensors.output); + // The scale remains observable and feeds the eventual output rescaling. + ggml_set_output(tensors.scale); + ggml_build_forward_expand(graph, ggml_mul(ctx, tensors.output, tensors.scale)); + if (variant == 3) { + ggml_build_forward_expand(graph, ggml_scale(ctx, tensors.absolute, 2.0f)); + } + auto imported = ggml::hrx::import_ggml_graph(*graph); + REQUIRE(imported.valid()); + ggml::hrx::DispatchScheduler scheduler; + REQUIRE(scheduler.schedule_graph(imported.graph, test_dispatch_target())); + size_t fused_count = 0; + for (const auto & dispatch : scheduler.plan().dispatches) { + if (kernel_name_for_id(dispatch.kernel.kernel_id) == "loom_libs:ggml_amax_scale_f32") { + ++fused_count; + REQUIRE(dispatch.bindings.size() == 3); + REQUIRE(dispatch.kernel.integer_parameters.at("row_width") == 896); + REQUIRE(dispatch.kernel.integer_parameters.at("row_count") == 128); + } + } + REQUIRE(fused_count == (variant == 0 ? 1 : 0)); + if (variant == 0) { + REQUIRE(scheduler.plan().dispatches.size() == 2); + } + ggml_free(ctx); + } +} + +static std::vector run_amax_scale_fusion_case(ggml_backend_t backend, int64_t width, + int64_t tokens, bool expose_intermediate) { + ggml_context * ctx = ggml_init({ 1024 * 1024, nullptr, true }); + REQUIRE(ctx != nullptr); + const auto tensors = build_amax_scale_test_graph(ctx, width, tokens); + if (expose_intermediate) { + ggml_set_output(tensors.absolute); + } + ggml_set_output(tensors.scale); + ggml_cgraph * graph = ggml_new_graph(ctx); + ggml_build_forward_expand(graph, tensors.output); + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); + REQUIRE(buffer != nullptr); + const float amplitudes[] = { 0.0f, 0.000125f, 1024.0f, std::nextafter(1024.0f, 0.0f), + std::nextafter(1024.0f, 2048.0f), 65536.0f, 1.0e6f, 1.0e30f }; + std::vector input(width * 8 * tokens); + for (size_t i = 0; i < input.size(); ++i) { + const size_t row = i / width; + const size_t column = i % width; + const float fraction = static_cast(column % 17) / 16.0f; + input[i] = amplitudes[row % 8] * fraction * (column % 2 ? -1.0f : 1.0f); + } + ggml_backend_tensor_set(tensors.input, input.data(), 0, input.size() * sizeof(float)); + REQUIRE(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(backend); + const size_t rows = 8 * tokens; + std::vector output(input.size() + rows); + ggml_backend_tensor_get(tensors.output, output.data(), 0, input.size() * sizeof(float)); + ggml_backend_tensor_get(tensors.scale, output.data() + input.size(), 0, rows * sizeof(float)); + for (size_t i = 0; i < input.size(); ++i) { + const float recovered = output[i] * output[input.size() + i / width]; + REQUIRE(std::isfinite(output[i])); + REQUIRE(std::fabs(recovered - input[i]) <= 3.0e-6f * std::fabs(input[i]) + 1.0e-7f); + } + ggml_backend_buffer_free(buffer); + ggml_free(ctx); + return output; +} + +static void run_amax_scale_fusion_numerics() { + ggml_backend_t cpu = ggml_backend_cpu_init(); + ggml_backend_t hrx = ggml_backend_hrx_init(0); + REQUIRE(cpu != nullptr); + REQUIRE(hrx != nullptr); + for (int64_t width : { 64, 896 }) { + for (int64_t tokens : { 1, 16 }) { + const auto reference = run_amax_scale_fusion_case(cpu, width, tokens, false); + const auto unfused = run_amax_scale_fusion_case(hrx, width, tokens, true); + const auto fused = run_amax_scale_fusion_case(hrx, width, tokens, false); + REQUIRE(fused.size() == reference.size()); + for (size_t i = 0; i < fused.size(); ++i) { + REQUIRE(std::isfinite(fused[i])); + REQUIRE(fused[i] == unfused[i]); + REQUIRE(std::fabs(fused[i] - reference[i]) <= 3.0e-6f * std::fabs(reference[i]) + 1.0e-7f); + } + } + } + ggml_backend_free(hrx); + ggml_backend_free(cpu); +} + static std::vector run_mellum_scaling_numerical_case(ggml_backend_t backend, int64_t token_count) { ggml_init_params params = {}; params.mem_size = 1024 * 1024; @@ -14969,6 +15090,7 @@ static void register_hrx_backend_host_cases(test_runner::Suite & suite) { run_qwen_weighted_reduce_next_rmsnorm_q8_publication_checks(); }); suite.host_case("common_mul_mat_id_postops_dispatch", [] { run_common_mul_mat_id_postops_dispatch_checks(); }); + suite.host_case("amax_scale_fusion_dispatch", [] { run_amax_scale_fusion_dispatch_checks(); }); suite.host_case("mellum_scaling_dispatch", [] { run_mellum_scaling_dispatch_checks(); }); suite.host_case("alias_value_import", [] { run_alias_value_import_checks(); }); suite.host_case("multi_dispatch", [] { run_multi_dispatch_checks(); }); @@ -15141,6 +15263,7 @@ static void register_hrx_backend_device_cases(test_runner::Suite & suite) { suite.device_case("rope_scale_f32_numerics", [] { run_rope_scale_f32_numerics(); }); suite.device_case("gather_add_rmsnorm_f32_numerics", [] { run_gather_add_rmsnorm_f32_numerics(); }); suite.device_case("scale_f32", [] { run_scale_f32(); }); + suite.device_case("amax_scale_fusion_numerics", [] { run_amax_scale_fusion_numerics(); }); suite.device_case("mellum_scaling_numerics", [] { run_mellum_scaling_numerics(); }); suite.device_case("mellum_scaling_direct_numerics", [] { run_mellum_scaling_direct_numerics(); }); for (const auto & shape : { std::pair{ GGML_TYPE_Q8_0, 896 }, From 7ba92fa50a4cce646c40272382ef8f433091199e Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Thu, 5 Nov 2026 05:07:33 +0000 Subject: [PATCH 09/10] test(hrx): cover fused scaling boundaries and alias exclusions --- tests/hrx-backend-test.cpp | 54 +++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index a298372370a7..a2d802d9d19c 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -13832,11 +13832,14 @@ static AmaxScaleTestGraph build_amax_scale_test_graph(ggml_context * ctx, int64_ } static void run_amax_scale_fusion_dispatch_checks() { - for (int variant = 0; variant < 4; ++variant) { + for (int variant = 0; variant < 6; ++variant) { ggml_context * ctx = ggml_init({ 1024 * 1024, nullptr, true }); REQUIRE(ctx != nullptr); const float inverse = variant == 1 ? 1.0f / 512.0f : 1.0f / 1024.0f; - const auto tensors = build_amax_scale_test_graph(ctx, 896, 16, inverse); + auto tensors = build_amax_scale_test_graph(ctx, 896, 16, inverse); + if (variant == 4) { + tensors.output = ggml_div_inplace(ctx, tensors.input, tensors.scale); + } if (variant == 2) { ggml_set_output(tensors.absolute); } @@ -13848,10 +13851,21 @@ static void run_amax_scale_fusion_dispatch_checks() { if (variant == 3) { ggml_build_forward_expand(graph, ggml_scale(ctx, tensors.absolute, 2.0f)); } + if (variant == 5) { + ggml_tensor * alias = ggml_view_1d(ctx, tensors.absolute, 896, 0); + ggml_set_output(alias); + ggml_build_forward_expand(graph, alias); + } auto imported = ggml::hrx::import_ggml_graph(*graph); REQUIRE(imported.valid()); ggml::hrx::DispatchScheduler scheduler; - REQUIRE(scheduler.schedule_graph(imported.graph, test_dispatch_target())); + const bool scheduled = scheduler.schedule_graph(imported.graph, test_dispatch_target()); + if (variant == 4) { + REQUIRE(!scheduled); + ggml_free(ctx); + continue; + } + REQUIRE(scheduled); size_t fused_count = 0; for (const auto & dispatch : scheduler.plan().dispatches) { if (kernel_name_for_id(dispatch.kernel.kernel_id) == "loom_libs:ggml_amax_scale_f32") { @@ -13870,7 +13884,7 @@ static void run_amax_scale_fusion_dispatch_checks() { } static std::vector run_amax_scale_fusion_case(ggml_backend_t backend, int64_t width, - int64_t tokens, bool expose_intermediate) { + int64_t tokens, bool expose_intermediate, bool nonfinite = false) { ggml_context * ctx = ggml_init({ 1024 * 1024, nullptr, true }); REQUIRE(ctx != nullptr); const auto tensors = build_amax_scale_test_graph(ctx, width, tokens); @@ -13891,6 +13905,16 @@ static std::vector run_amax_scale_fusion_case(ggml_backend_t backend, int const float fraction = static_cast(column % 17) / 16.0f; input[i] = amplitudes[row % 8] * fraction * (column % 2 ? -1.0f : 1.0f); } + for (size_t row = 0; row < static_cast(8 * tokens); ++row) { + input[(row + 1) * width - 1] = -amplitudes[row % 8]; + } + if (nonfinite) { + std::fill(input.begin(), input.begin() + width, std::numeric_limits::quiet_NaN()); + input[2 * width - 1] = std::numeric_limits::infinity(); + input[3 * width - 1] = -std::numeric_limits::infinity(); + input[4 * width - 1] = std::numeric_limits::quiet_NaN(); + input[5 * width - 1] = std::numeric_limits::max(); + } ggml_backend_tensor_set(tensors.input, input.data(), 0, input.size() * sizeof(float)); REQUIRE(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS); ggml_backend_synchronize(backend); @@ -13898,7 +13922,8 @@ static std::vector run_amax_scale_fusion_case(ggml_backend_t backend, int std::vector output(input.size() + rows); ggml_backend_tensor_get(tensors.output, output.data(), 0, input.size() * sizeof(float)); ggml_backend_tensor_get(tensors.scale, output.data() + input.size(), 0, rows * sizeof(float)); - for (size_t i = 0; i < input.size(); ++i) { + const size_t roundtrip_elements = nonfinite ? 0 : input.size(); + for (size_t i = 0; i < roundtrip_elements; ++i) { const float recovered = output[i] * output[input.size() + i / width]; REQUIRE(std::isfinite(output[i])); REQUIRE(std::fabs(recovered - input[i]) <= 3.0e-6f * std::fabs(input[i]) + 1.0e-7f); @@ -13913,7 +13938,7 @@ static void run_amax_scale_fusion_numerics() { ggml_backend_t hrx = ggml_backend_hrx_init(0); REQUIRE(cpu != nullptr); REQUIRE(hrx != nullptr); - for (int64_t width : { 64, 896 }) { + for (int64_t width : { 1, 64, 257, 896, 32768 }) { for (int64_t tokens : { 1, 16 }) { const auto reference = run_amax_scale_fusion_case(cpu, width, tokens, false); const auto unfused = run_amax_scale_fusion_case(hrx, width, tokens, true); @@ -13922,10 +13947,25 @@ static void run_amax_scale_fusion_numerics() { for (size_t i = 0; i < fused.size(); ++i) { REQUIRE(std::isfinite(fused[i])); REQUIRE(fused[i] == unfused[i]); - REQUIRE(std::fabs(fused[i] - reference[i]) <= 3.0e-6f * std::fabs(reference[i]) + 1.0e-7f); + const bool close_to_cpu = std::fabs(fused[i] - reference[i]) <= 3.0e-6f * std::fabs(reference[i]) + 1.0e-7f; + if (!close_to_cpu) { + std::fprintf(stderr, "amax width=%lld tokens=%lld i=%zu cpu=%.9g hrx=%.9g\n", + (long long) width, (long long) tokens, i, reference[i], fused[i]); + } + REQUIRE(close_to_cpu); } } } + // CPU pooling and HRX maxnum reductions differ on NaN rows. Fusion must preserve the existing HRX results. + const auto unfused = run_amax_scale_fusion_case(hrx, 257, 1, true, true); + const auto fused = run_amax_scale_fusion_case(hrx, 257, 1, false, true); + for (size_t i = 0; i < fused.size(); ++i) { + if (std::isnan(unfused[i])) { + REQUIRE(std::isnan(fused[i])); + } else { + REQUIRE(fused[i] == unfused[i]); + } + } ggml_backend_free(hrx); ggml_backend_free(cpu); } From e6e1b5f7ef0e219a0386ad815f0ad0d0e4c268f3 Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Thu, 5 Nov 2026 05:18:46 +0000 Subject: [PATCH 10/10] hrx: remove standalone scaling kernels superseded by fusion --- ggml/src/ggml-hrx/CMakeLists.txt | 4 +- ...ch-pool-1d.cpp => dispatch-amax-scale.cpp} | 65 +---- ...spatch-pool-1d.h => dispatch-amax-scale.h} | 2 +- .../common/dispatch-common.cpp | 4 +- .../common/dispatch-unary.cpp | 57 ---- ggml/src/ggml-hrx/ggml-hrx.cpp | 1 + .../kernels/loom-libs/manifest.json | 84 ------ .../loom-libs/ops/elementwise/clamp_f32.loom | 36 --- .../ops/reduction/pool_1d_full_max_f32.loom | 35 --- tests/hrx-backend-test.cpp | 268 +++--------------- 10 files changed, 44 insertions(+), 512 deletions(-) rename ggml/src/ggml-hrx/dispatch_registration/common/{dispatch-pool-1d.cpp => dispatch-amax-scale.cpp} (72%) rename ggml/src/ggml-hrx/dispatch_registration/common/{dispatch-pool-1d.h => dispatch-amax-scale.h} (62%) delete mode 100644 ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/elementwise/clamp_f32.loom delete mode 100644 ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/reduction/pool_1d_full_max_f32.loom diff --git a/ggml/src/ggml-hrx/CMakeLists.txt b/ggml/src/ggml-hrx/CMakeLists.txt index 85641e9cca84..3a398b18a81c 100644 --- a/ggml/src/ggml-hrx/CMakeLists.txt +++ b/ggml/src/ggml-hrx/CMakeLists.txt @@ -227,8 +227,8 @@ ggml_add_backend_library(ggml-hrx dispatch_registration/common/dispatch-mul-mat-common.h dispatch_registration/common/dispatch-mul-mat-weight-format.h dispatch_registration/common/dispatch-mul-mat.h - dispatch_registration/common/dispatch-pool-1d.cpp - dispatch_registration/common/dispatch-pool-1d.h + dispatch_registration/common/dispatch-amax-scale.cpp + dispatch_registration/common/dispatch-amax-scale.h dispatch_registration/common/dispatch-rope-set-rows.cpp dispatch_registration/common/dispatch-rope-set-rows.h dispatch_registration/common/dispatch-rope-utils.h diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-amax-scale.cpp similarity index 72% rename from ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.cpp rename to ggml/src/ggml-hrx/dispatch_registration/common/dispatch-amax-scale.cpp index c8ef14521ecb..4bae738edecd 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-amax-scale.cpp @@ -1,4 +1,4 @@ -#include "dispatch-pool-1d.h" +#include "dispatch-amax-scale.h" #include "ggml.h" #include "graph/graph-matcher.h" @@ -10,9 +10,6 @@ namespace ggml::hrx { namespace { -static constexpr KernelCatalogRef kFullMaxPoolF32Kernel = - GGML_HRX_KERNEL_REF("loom_libs", "ggml_pool_1d_full_max_f32"); - static constexpr KernelCatalogRef kAmaxScaleF32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_amax_scale_f32"); @@ -176,59 +173,9 @@ static bool match_amax_scale_f32(const DispatchMatchContext & context, DispatchM return true; } -static bool match_full_max_pool_1d_f32(const DispatchMatchContext & context, DispatchMatch & match) { - const GraphNode * node = context.root_node; - if (node == nullptr) { - return false; - } - const bool one_input = node->inputs.size() == 1; - if (!one_input) { - return false; - } - - const Pool1DParams * params = op_params_as(node->params); - const Value * input = context.graph.values().find(node->inputs[0]); - const Value * output = context.graph.values().find(node->output); - const bool has_params = params != nullptr; - const bool has_input = input != nullptr; - const bool has_output = output != nullptr; - const bool all_present = has_params && has_input && has_output; - if (!all_present) { - return false; - } - - const bool f32_values = input->type == GGML_TYPE_F32 && output->type == GGML_TYPE_F32; - const bool contiguous_values = input->contiguous && output->contiguous; - const bool full_row = output->ne[0] == 1 && input->ne[0] == params->kernel; - const bool matching_rows = input->ne[1] == output->ne[1] && input->ne[2] == output->ne[2] && - input->ne[3] == output->ne[3]; - const bool exact_window = params->stride == params->kernel && params->padding == 0; - const bool bounded_width = input->ne[0] > 0 && input->ne[0] <= 32768; - const bool bounded_rows = output->element_count > 0 && output->element_count <= 1048576; - const bool bounded_input = input->element_count <= 134217728; - const bool bounded = bounded_width && bounded_rows && bounded_input; - const bool separate_storage = input->storage_root != output->storage_root; - const bool supported = params->op == GGML_OP_POOL_MAX && f32_values && contiguous_values && full_row && - matching_rows && exact_window && bounded && separate_storage; - if (!supported) { - return false; - } - - Dispatch dispatch; - dispatch.kernel = make_kernel_specialization(kFullMaxPoolF32Kernel); - dispatch.kernel.integer_parameters.emplace("row_count", output->element_count); - dispatch.kernel.integer_parameters.emplace("row_width", input->ne[0]); - dispatch.bindings.push_back({ input->id, 0, input->byte_count }); - dispatch.bindings.push_back({ output->id, 0, output->byte_count }); - - match.covered_nodes.push_back(context.root_index); - match.dispatches.push_back(std::move(dispatch)); - return true; -} - } // namespace -void register_pool_1d_dispatch(DispatchRegistryBuilder & registry) { +void register_amax_scale_dispatch(DispatchRegistryBuilder & registry) { registry.add({ "common.amax_scale_f32", GGML_OP_UNARY, @@ -237,14 +184,6 @@ void register_pool_1d_dispatch(DispatchRegistryBuilder & registry) { DispatchSource::Common, match_amax_scale_f32, }); - registry.add({ - "common.pool_1d_full_max_f32", - GGML_OP_POOL_1D, - DispatchMatchKind::SingleOp, - 0, - DispatchSource::Common, - match_full_max_pool_1d_f32, - }); } } // namespace ggml::hrx diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.h b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-amax-scale.h similarity index 62% rename from ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.h rename to ggml/src/ggml-hrx/dispatch_registration/common/dispatch-amax-scale.h index 81d357295d2e..9aaeba4f9ad6 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-pool-1d.h +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-amax-scale.h @@ -4,6 +4,6 @@ namespace ggml::hrx { -void register_pool_1d_dispatch(DispatchRegistryBuilder & registry); +void register_amax_scale_dispatch(DispatchRegistryBuilder & registry); } // namespace ggml::hrx diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-common.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-common.cpp index e68dfcd6d590..c2a2b47e8d32 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-common.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-common.cpp @@ -10,7 +10,7 @@ #include "dispatch-glu.h" #include "dispatch-mul-mat-id.h" #include "dispatch-mul-mat.h" -#include "dispatch-pool-1d.h" +#include "dispatch-amax-scale.h" #include "dispatch-rmsnorm.h" #include "dispatch-rope-set-rows.h" #include "dispatch-scale.h" @@ -29,7 +29,7 @@ void register_common_dispatches(DispatchRegistryBuilder & registry) { register_glu_dispatches(registry); register_mul_mat_id_dispatches(registry); register_mul_mat_dispatches(registry); - register_pool_1d_dispatch(registry); + register_amax_scale_dispatch(registry); register_rope_set_rows_dispatches(registry); register_scale_dispatch(registry); register_unary_dispatch(registry); diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-unary.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-unary.cpp index 4036061d86fa..03cbc2b8b232 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-unary.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-unary.cpp @@ -16,7 +16,6 @@ namespace { static constexpr KernelCatalogRef kUnaryF32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_unary_f32"); static constexpr KernelCatalogRef kScaleBiasF32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_scale_bias_f32"); -static constexpr KernelCatalogRef kClampF32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_clamp_f32"); static bool same_shape(const Value & lhs, const Value & rhs) { for (int i = 0; i < GGML_MAX_DIMS; ++i) { @@ -119,54 +118,6 @@ static bool match_unary_f32_dispatch(const DispatchMatchContext & context, Dispa return true; } -static bool match_clamp_f32_dispatch(const DispatchMatchContext & context, DispatchMatch & match) { - const GraphNode * node = context.root_node; - if (node == nullptr) { - return false; - } - const bool one_input = node->inputs.size() == 1; - if (!one_input) { - return false; - } - - const ClampParams * params = op_params_as(node->params); - const Value * input = graph_value(context.graph, node->inputs[0]); - const Value * output = graph_value(context.graph, node->output); - const bool has_params = params != nullptr; - const bool has_input = input != nullptr; - const bool has_output = output != nullptr; - const bool all_present = has_params && has_input && has_output; - if (!all_present) { - return false; - } - - const bool exponent_limits = params->min == 0.0f && params->max == 127.0f; - const bool scalar_per_row = input->ne[0] == 1 && output->ne[0] == 1; - const bool f32_values = input->type == GGML_TYPE_F32 && output->type == GGML_TYPE_F32; - const bool contiguous_values = input->contiguous && output->contiguous; - const bool matching_shape = same_shape(*input, *output); - const bool safe_storage = scale_storage_is_safe(*input, *output); - const bool bounded = output->element_count > 0 && - static_cast(output->element_count) <= 134217728; - const bool supported = exponent_limits && scalar_per_row && f32_values && contiguous_values && - matching_shape && safe_storage && bounded; - if (!supported) { - return false; - } - - Dispatch dispatch; - dispatch.kernel = make_kernel_specialization(kClampF32Kernel); - dispatch.kernel.integer_parameters.emplace("element_count", output->element_count); - dispatch.kernel.compile_parameters.emplace("ggml.clamp_f32.min", float_config_value(params->min)); - dispatch.kernel.compile_parameters.emplace("ggml.clamp_f32.max", float_config_value(params->max)); - dispatch.bindings.push_back({ input->id, 0, input->byte_count }); - dispatch.bindings.push_back({ output->id, 0, output->byte_count }); - - match.covered_nodes.push_back(context.root_index); - match.dispatches.push_back(std::move(dispatch)); - return true; -} - static void register_unary_dispatch_for(DispatchRegistryBuilder & registry, ggml_op root_op) { registry.add({ "common.unary_f32", @@ -181,14 +132,6 @@ static void register_unary_dispatch_for(DispatchRegistryBuilder & registry, ggml } // namespace void register_unary_dispatch(DispatchRegistryBuilder & registry) { - registry.add({ - "common.clamp_f32", - GGML_OP_CLAMP, - DispatchMatchKind::SingleOp, - 0, - DispatchSource::Common, - match_clamp_f32_dispatch, - }); registry.add({ "common.scale_bias_f32", GGML_OP_SCALE, diff --git a/ggml/src/ggml-hrx/ggml-hrx.cpp b/ggml/src/ggml-hrx/ggml-hrx.cpp index 5c8742693f32..c0620720a058 100644 --- a/ggml/src/ggml-hrx/ggml-hrx.cpp +++ b/ggml/src/ggml-hrx/ggml-hrx.cpp @@ -1198,6 +1198,7 @@ static bool device_supports_op(ggml_backend_dev_t device, const ggml_tensor * op if (supported_scale_f32_tensor(op)) { return true; } + // Admit pooling for the amax-scale fusion; graph claim validates the complete chain. if (supported_full_max_pool_1d_f32_tensor(op)) { return true; } diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json index 73ce5b7d5ffe..4e1acc579461 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json @@ -20,12 +20,6 @@ { "path": "ops/elementwise/unary_f32.loom" }, - { - "path": "ops/elementwise/clamp_f32.loom" - }, - { - "path": "ops/reduction/pool_1d_full_max_f32.loom" - }, { "path": "ops/elementwise/binary_swiglu_symmetric_i4_k32.loom" }, @@ -1247,84 +1241,6 @@ "motifs/copy/concat_dim0.loom" ] }, - { - "name": "ggml_clamp_f32", - "family": "loom_libs", - "symbol": "ggml_clamp_f32", - "source": "ops/elementwise/clamp_f32.loom", - "workload_parameters": [ - { - "name": "element_count", - "type": "index" - } - ], - "launch_parameters": [ - { - "name": "element_count", - "type": "index" - } - ], - "bindings": [ - "input", - "output" - ], - "binding_access": [ - "read", - "read_write" - ], - "target_selector": "", - "compile_recipe": { - "mode": "direct", - "primary_sources": [ - "ops/elementwise/clamp_f32.loom" - ], - "library_sources": [] - }, - "compile_dependencies": [] - }, - { - "name": "ggml_pool_1d_full_max_f32", - "family": "loom_libs", - "symbol": "ggml_pool_1d_full_max_f32", - "source": "ops/reduction/pool_1d_full_max_f32.loom", - "workload_parameters": [ - { - "name": "row_count", - "type": "index" - }, - { - "name": "row_width", - "type": "index" - } - ], - "launch_parameters": [ - { - "name": "row_count", - "type": "index" - }, - { - "name": "row_width", - "type": "index" - } - ], - "bindings": [ - "input", - "output" - ], - "binding_access": [ - "read", - "write" - ], - "target_selector": "", - "compile_recipe": { - "mode": "direct", - "primary_sources": [ - "ops/reduction/pool_1d_full_max_f32.loom" - ], - "library_sources": [] - }, - "compile_dependencies": [] - }, { "name": "ggml_scale_bias_f32", "family": "loom_libs", diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/elementwise/clamp_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/elementwise/clamp_f32.loom deleted file mode 100644 index beddb6715efc..000000000000 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/elementwise/clamp_f32.loom +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2026 The HRX Authors -// SPDX-License-Identifier: Apache-2.0 - -amdgpu.target @ggml_clamp_f32_gfx11_wave64 {subgroup_size = 64} - -config.decl @ggml.clamp_f32.min : f32 -config.decl @ggml.clamp_f32.max : f32 - -kernel.def target(@ggml_clamp_f32_gfx11_wave64) export("ggml_clamp_f32") @ggml_clamp_f32(%element_count: index) { - %one = index.constant 1 : index - %twofiftysix = index.constant 256 : index - %rounding = index.constant 255 : index - %rounded = index.add %element_count, %rounding : index - %workgroup_count = index.div %rounded, %twofiftysix : index - kernel.launch.config workgroups(%workgroup_count, %one, %one) workgroup_size(%twofiftysix, %one, %one) : index -} launch(%element_count: index, %input: buffer, %output: buffer) { - %count = index.assume %element_count [range(%element_count, 1, 134217728)] : index - %workgroup = kernel.workgroup.id : index - %workitem = kernel.workitem.id : index - %twofiftysix = index.constant 256 : index - %base = index.mul %workgroup, %twofiftysix : index - %linear0 = index.add %base, %workitem : index - %linear = index.assume %linear0 [range(%linear0, 0, 134217983)] : index - %in_bounds = index.cmp ult, %linear, %count : index - %minimum = config.get @ggml.clamp_f32.min : f32 - %maximum = config.get @ggml.clamp_f32.max : f32 - %zero_offset = index.constant 0 : offset - %input_view = buffer.view %input[%zero_offset] : buffer -> view<[%count]xf32> - %output_view = buffer.view %output[%zero_offset] : buffer -> view<[%count]xf32> - scf.if %in_bounds { - %value = view.load %input_view[%linear] : view<[%count]xf32> -> f32 - %result = scalar.clampf %value, %minimum, %maximum : f32 - view.store %result, %output_view[%linear] : f32, view<[%count]xf32> - } - kernel.return -} diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/reduction/pool_1d_full_max_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/reduction/pool_1d_full_max_f32.loom deleted file mode 100644 index dc6588e17b97..000000000000 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/reduction/pool_1d_full_max_f32.loom +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2026 The HRX Authors -// SPDX-License-Identifier: Apache-2.0 - -amdgpu.target @ggml_pool_1d_full_max_f32_gfx11_wave64 {subgroup_size = 64} - -kernel.def target(@ggml_pool_1d_full_max_f32_gfx11_wave64) export("ggml_pool_1d_full_max_f32") @ggml_pool_1d_full_max_f32(%row_count: index, %row_width: index) { - %one = index.constant 1 : index - %twofiftysix = index.constant 256 : index - kernel.launch.config workgroups(%row_count, %one, %one) workgroup_size(%twofiftysix, %one, %one) : index -} launch(%row_count: index, %row_width: index, %input: buffer, %output: buffer) where [range(%row_count, 1, 1048576), range(%row_width, 1, 32768)] { - %rows = index.assume %row_count [range(%row_count, 1, 1048576)] : index - %width = index.assume %row_width [range(%row_width, 1, 32768)] : index - %row = kernel.workgroup.id : index - %lane = kernel.workitem.id : index - %zero = index.constant 0 : index - %stride = index.constant 256 : index - %negative_max = scalar.constant -3.4028234663852886e+38 : f32 - %element_count = index.mul %rows, %width : index - %row_base = index.mul %row, %width : index - %zero_offset = index.constant 0 : offset - %input_view = buffer.view %input[%zero_offset] : buffer -> view<[%element_count]xf32> - %output_view = buffer.view %output[%zero_offset] : buffer -> view<[%rows]xf32> - %maximum = scf.for %column = [%lane to %width step %stride](%prior = %negative_max : f32) -> (f32) { - %offset = index.add %row_base, %column : index - %value = view.load %input_view[%offset] : view<[%element_count]xf32> -> f32 - %next = scalar.maxnumf %value, %prior : f32 - scf.yield %next : f32 - } - %row_maximum = kernel.workgroup.reduce %maximum : f32 - %is_first = index.cmp eq, %lane, %zero : index - scf.if %is_first { - view.store %row_maximum, %output_view[%row] : f32, view<[%rows]xf32> - } - kernel.return -} diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index a2d802d9d19c..a90b8e9ea567 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -1565,47 +1565,6 @@ static void run_scale_f32_dispatch_checks() { ggml_free(ctx); } -static void run_mellum_scaling_dispatch_checks() { - ggml_init_params params = {}; - params.mem_size = 1024 * 1024; - params.no_alloc = true; - ggml_context * ctx = ggml_init(params); - REQUIRE(ctx != nullptr); - - ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 896, 8, 16); - ggml_tensor * pooled = ggml_pool_1d(ctx, input, GGML_OP_POOL_MAX, 896, 896, 0); - ggml_tensor * clamp_input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1, 8, 16); - ggml_tensor * clamped = ggml_clamp(ctx, clamp_input, 0.0f, 127.0f); - REQUIRE(input != nullptr); - REQUIRE(pooled != nullptr); - REQUIRE(clamp_input != nullptr); - REQUIRE(clamped != nullptr); - - const ggml::hrx::Dispatch pool_dispatch = schedule_single_dispatch_for_tensor(ctx, pooled); - REQUIRE(kernel_name_for_id(pool_dispatch.kernel.kernel_id) == "loom_libs:ggml_pool_1d_full_max_f32"); - REQUIRE(pool_dispatch.kernel.integer_parameters.at("row_count") == 128); - REQUIRE(pool_dispatch.kernel.integer_parameters.at("row_width") == 896); - - const ggml::hrx::Dispatch clamp_dispatch = schedule_single_dispatch_for_tensor(ctx, clamped); - REQUIRE(kernel_name_for_id(clamp_dispatch.kernel.kernel_id) == "loom_libs:ggml_clamp_f32"); - REQUIRE(clamp_dispatch.kernel.integer_parameters.at("element_count") == 128); - require_compile_parameter(clamp_dispatch, "ggml.clamp_f32.min", expected_config_value(0.0f)); - require_compile_parameter(clamp_dispatch, "ggml.clamp_f32.max", expected_config_value(127.0f)); - - ggml_tensor * wide_clamp_input = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 8); - ggml_tensor * wide_clamp = ggml_clamp(ctx, wide_clamp_input, 0.0f, 127.0f); - ggml_tensor * other_bounds = ggml_clamp(ctx, clamp_input, 0.0f, 1.0f); - REQUIRE(!can_schedule_tensor(ctx, wide_clamp)); - REQUIRE(!can_schedule_tensor(ctx, other_bounds)); - - ggml_tensor * partial_pool = ggml_pool_1d(ctx, input, GGML_OP_POOL_MAX, 32, 32, 0); - ggml_tensor * average_pool = ggml_pool_1d(ctx, input, GGML_OP_POOL_AVG, 896, 896, 0); - REQUIRE(!can_schedule_tensor(ctx, partial_pool)); - REQUIRE(!can_schedule_tensor(ctx, average_pool)); - - ggml_free(ctx); -} - static void require_scale_add_dispatch(ggml_context * ctx, ggml_tensor * input, ggml_tensor * residual, @@ -13860,7 +13819,17 @@ static void run_amax_scale_fusion_dispatch_checks() { REQUIRE(imported.valid()); ggml::hrx::DispatchScheduler scheduler; const bool scheduled = scheduler.schedule_graph(imported.graph, test_dispatch_target()); - if (variant == 4) { + if (variant != 0) { + // Without standalone pool/clamp dispatches, these graphs must reject the fusion and the full claim. + std::vector covered(imported.graph.nodes().size(), false); + ggml::hrx::CommandPlan plan; + ggml::hrx::DispatchMatch match; + const size_t root = producer_index_for_tensor(imported.graph, tensors.absolute); + if (match_dispatch_at_index(imported.graph, plan, covered, root, match)) { + for (const auto & dispatch : match.dispatches) { + REQUIRE(kernel_name_for_id(dispatch.kernel.kernel_id) != "loom_libs:ggml_amax_scale_f32"); + } + } REQUIRE(!scheduled); ggml_free(ctx); continue; @@ -13875,22 +13844,17 @@ static void run_amax_scale_fusion_dispatch_checks() { REQUIRE(dispatch.kernel.integer_parameters.at("row_count") == 128); } } - REQUIRE(fused_count == (variant == 0 ? 1 : 0)); - if (variant == 0) { - REQUIRE(scheduler.plan().dispatches.size() == 2); - } + REQUIRE(fused_count == 1); + REQUIRE(scheduler.plan().dispatches.size() == 2); ggml_free(ctx); } } static std::vector run_amax_scale_fusion_case(ggml_backend_t backend, int64_t width, - int64_t tokens, bool expose_intermediate, bool nonfinite = false) { + int64_t tokens, bool nonfinite = false) { ggml_context * ctx = ggml_init({ 1024 * 1024, nullptr, true }); REQUIRE(ctx != nullptr); const auto tensors = build_amax_scale_test_graph(ctx, width, tokens); - if (expose_intermediate) { - ggml_set_output(tensors.absolute); - } ggml_set_output(tensors.scale); ggml_cgraph * graph = ggml_new_graph(ctx); ggml_build_forward_expand(graph, tensors.output); @@ -13940,13 +13904,11 @@ static void run_amax_scale_fusion_numerics() { REQUIRE(hrx != nullptr); for (int64_t width : { 1, 64, 257, 896, 32768 }) { for (int64_t tokens : { 1, 16 }) { - const auto reference = run_amax_scale_fusion_case(cpu, width, tokens, false); - const auto unfused = run_amax_scale_fusion_case(hrx, width, tokens, true); - const auto fused = run_amax_scale_fusion_case(hrx, width, tokens, false); + const auto reference = run_amax_scale_fusion_case(cpu, width, tokens); + const auto fused = run_amax_scale_fusion_case(hrx, width, tokens); REQUIRE(fused.size() == reference.size()); for (size_t i = 0; i < fused.size(); ++i) { REQUIRE(std::isfinite(fused[i])); - REQUIRE(fused[i] == unfused[i]); const bool close_to_cpu = std::fabs(fused[i] - reference[i]) <= 3.0e-6f * std::fabs(reference[i]) + 1.0e-7f; if (!close_to_cpu) { std::fprintf(stderr, "amax width=%lld tokens=%lld i=%zu cpu=%.9g hrx=%.9g\n", @@ -13956,183 +13918,28 @@ static void run_amax_scale_fusion_numerics() { } } } - // CPU pooling and HRX maxnum reductions differ on NaN rows. Fusion must preserve the existing HRX results. - const auto unfused = run_amax_scale_fusion_case(hrx, 257, 1, true, true); - const auto fused = run_amax_scale_fusion_case(hrx, 257, 1, false, true); - for (size_t i = 0; i < fused.size(); ++i) { - if (std::isnan(unfused[i])) { - REQUIRE(std::isnan(fused[i])); - } else { - REQUIRE(fused[i] == unfused[i]); - } - } - ggml_backend_free(hrx); - ggml_backend_free(cpu); -} - -static std::vector run_mellum_scaling_numerical_case(ggml_backend_t backend, int64_t token_count) { - ggml_init_params params = {}; - params.mem_size = 1024 * 1024; - params.no_alloc = true; - ggml_context * ctx = ggml_init(params); - REQUIRE(ctx != nullptr); - - constexpr int64_t row_width = 896; - constexpr int64_t route_count = 8; - ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, row_width, route_count, token_count); - ggml_tensor * absolute = ggml_abs(ctx, input); - ggml_tensor * pooled = ggml_pool_1d(ctx, absolute, GGML_OP_POOL_MAX, row_width, row_width, 0); - ggml_tensor * clamped = ggml_clamp(ctx, pooled, 0.0f, 127.0f); - REQUIRE(input != nullptr); - REQUIRE(absolute != nullptr); - REQUIRE(pooled != nullptr); - REQUIRE(clamped != nullptr); - - ggml_cgraph * graph = ggml_new_graph(ctx); - REQUIRE(graph != nullptr); - ggml_build_forward_expand(graph, clamped); - - ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); - REQUIRE(buffer != nullptr); - - std::vector input_data(row_width * route_count * token_count); - for (size_t i = 0; i < input_data.size(); ++i) { - input_data[i] = static_cast((i * 37) % 251) * 0.0625f - 7.5f; - } - input_data[0] = 0.0f; - input_data[row_width - 1] = -256.0f; - for (int64_t column = 0; column < row_width; ++column) { - input_data[row_width + column] = 0.0f; - input_data[2 * row_width + column] = 0.000125f; - } - input_data[2 * row_width - 1] = -0.000125f; - ggml_backend_tensor_set(input, input_data.data(), 0, input_data.size() * sizeof(float)); - - REQUIRE(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS); - ggml_backend_synchronize(backend); - - std::vector output(route_count * token_count); - ggml_backend_tensor_get(clamped, output.data(), 0, output.size() * sizeof(float)); - - ggml_backend_buffer_free(buffer); - ggml_free(ctx); - return output; -} - -static void run_mellum_scaling_numerics() { - ggml_backend_t hrx = ggml_backend_hrx_init(0); - ggml_backend_t cpu = ggml_backend_cpu_init(); - REQUIRE(hrx != nullptr); - REQUIRE(cpu != nullptr); - - for (int64_t token_count : { 1, 16 }) { - const std::vector expected = run_mellum_scaling_numerical_case(cpu, token_count); - const std::vector actual = run_mellum_scaling_numerical_case(hrx, token_count); - const std::vector repeated = run_mellum_scaling_numerical_case(hrx, token_count); - REQUIRE(actual.size() == expected.size()); - REQUIRE(repeated.size() == expected.size()); - for (size_t i = 0; i < expected.size(); ++i) { - REQUIRE(std::isfinite(actual[i])); - REQUIRE(actual[i] == expected[i]); - REQUIRE(repeated[i] == actual[i]); - } - } - - ggml_backend_free(cpu); + // HRX maxnum ignores NaNs in the reduction. Division preserves each NaN input. + constexpr size_t width = 257; + constexpr size_t elements = width * 8; + const auto fused = run_amax_scale_fusion_case(hrx, width, 1, true); + for (size_t column = 0; column < width; ++column) { + REQUIRE(std::isnan(fused[column])); + } + REQUIRE(fused[elements] == 1.0f); + REQUIRE(fused[elements + 3] == 1.0f); + REQUIRE(std::isnan(fused[4 * width - 1])); + REQUIRE(fused[3 * width + 1] == -std::nextafter(1024.0f, 0.0f) / 16.0f); + REQUIRE(fused[2 * width - 1] == std::numeric_limits::infinity()); + REQUIRE(fused[3 * width - 1] == -std::numeric_limits::infinity()); + // Infinite maxima saturate the exponent at 127; the resulting scale stays finite. + for (size_t row : { 1, 2 }) { + REQUIRE(std::isfinite(fused[elements + row])); + REQUIRE(std::fabs(fused[elements + row] / std::ldexp(1.0f, 127) - 1.0f) <= 3.0e-6f); + } + REQUIRE(std::isfinite(fused[5 * width - 1])); + REQUIRE(std::fabs(fused[5 * width - 1] - 1024.0f) <= 3.0e-6f * 1024.0f); ggml_backend_free(hrx); -} - -static std::vector run_full_max_pool_numerical_case(ggml_backend_t backend, int64_t row_width) { - ggml_context * ctx = ggml_init({ 1024 * 1024, nullptr, true }); - REQUIRE(ctx != nullptr); - constexpr int64_t row_count = 8; - ggml_tensor * input = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, row_width, row_count); - ggml_tensor * output = ggml_pool_1d(ctx, input, GGML_OP_POOL_MAX, row_width, row_width, 0); - REQUIRE(input != nullptr); - REQUIRE(output != nullptr); - ggml_cgraph * graph = ggml_new_graph(ctx); - REQUIRE(graph != nullptr); - ggml_build_forward_expand(graph, output); - ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); - REQUIRE(buffer != nullptr); - - std::vector input_data(row_width * row_count, -4.0f); - for (int64_t column = 0; column < row_width; ++column) { - input_data[row_width + column] = 0.0f; - input_data[2 * row_width + column] = -0.000125f; - } - input_data[row_width - 1] = -1.0f; - input_data[2 * row_width + row_width / 2] = 0.000125f; - input_data[3 * row_width] = 100000.0f; - input_data[3 * row_width + row_width - 1] = -100000.0f; - ggml_backend_tensor_set(input, input_data.data(), 0, input_data.size() * sizeof(float)); - - REQUIRE(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS); - ggml_backend_synchronize(backend); - std::vector result(row_count); - ggml_backend_tensor_get(output, result.data(), 0, result.size() * sizeof(float)); - - ggml_backend_buffer_free(buffer); - ggml_free(ctx); - return result; -} - -static std::vector run_clamp_numerical_case(ggml_backend_t backend, bool inplace) { - ggml_context * ctx = ggml_init({ 1024 * 1024, nullptr, true }); - REQUIRE(ctx != nullptr); - ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1, 8, 1); - ggml_tensor * output = inplace ? ggml_clamp_inplace(ctx, input, 0.0f, 127.0f) : - ggml_clamp(ctx, input, 0.0f, 127.0f); - REQUIRE(input != nullptr); - REQUIRE(output != nullptr); - ggml_cgraph * graph = ggml_new_graph(ctx); - REQUIRE(graph != nullptr); - ggml_build_forward_expand(graph, output); - ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); - REQUIRE(buffer != nullptr); - - const std::array input_data = { -1000.0f, -1.0f, 0.0f, 0.000125f, 126.0f, 127.0f, 128.0f, 1000.0f }; - ggml_backend_tensor_set(input, input_data.data(), 0, sizeof(input_data)); - REQUIRE(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS); - ggml_backend_synchronize(backend); - std::vector result(input_data.size()); - ggml_backend_tensor_get(output, result.data(), 0, result.size() * sizeof(float)); - - ggml_backend_buffer_free(buffer); - ggml_free(ctx); - return result; -} - -static void run_mellum_scaling_direct_numerics() { - ggml_backend_t hrx = ggml_backend_hrx_init(0); - ggml_backend_t cpu = ggml_backend_cpu_init(); - REQUIRE(hrx != nullptr); - REQUIRE(cpu != nullptr); - - for (int64_t row_width : { 64, 896 }) { - const std::vector expected = run_full_max_pool_numerical_case(cpu, row_width); - const std::vector actual = run_full_max_pool_numerical_case(hrx, row_width); - const std::vector repeated = run_full_max_pool_numerical_case(hrx, row_width); - REQUIRE(expected[0] == -1.0f); - REQUIRE(expected[1] == 0.0f); - REQUIRE(expected[2] == 0.000125f); - REQUIRE(expected[3] == 100000.0f); - REQUIRE(actual == expected); - REQUIRE(repeated == actual); - } - - for (bool inplace : { false, true }) { - const std::vector expected = run_clamp_numerical_case(cpu, inplace); - const std::vector actual = run_clamp_numerical_case(hrx, inplace); - const std::vector repeated = run_clamp_numerical_case(hrx, inplace); - REQUIRE(actual == expected); - REQUIRE(repeated == actual); - REQUIRE(actual[0] == 0.0f); - REQUIRE(actual[7] == 127.0f); - } - ggml_backend_free(cpu); - ggml_backend_free(hrx); } struct RoutedKTailOutputs { @@ -15131,7 +14938,6 @@ static void register_hrx_backend_host_cases(test_runner::Suite & suite) { }); suite.host_case("common_mul_mat_id_postops_dispatch", [] { run_common_mul_mat_id_postops_dispatch_checks(); }); suite.host_case("amax_scale_fusion_dispatch", [] { run_amax_scale_fusion_dispatch_checks(); }); - suite.host_case("mellum_scaling_dispatch", [] { run_mellum_scaling_dispatch_checks(); }); suite.host_case("alias_value_import", [] { run_alias_value_import_checks(); }); suite.host_case("multi_dispatch", [] { run_multi_dispatch_checks(); }); suite.host_case("layout_alias_scheduler_elision", [] { run_layout_alias_scheduler_elision_checks(); }); @@ -15304,8 +15110,6 @@ static void register_hrx_backend_device_cases(test_runner::Suite & suite) { suite.device_case("gather_add_rmsnorm_f32_numerics", [] { run_gather_add_rmsnorm_f32_numerics(); }); suite.device_case("scale_f32", [] { run_scale_f32(); }); suite.device_case("amax_scale_fusion_numerics", [] { run_amax_scale_fusion_numerics(); }); - suite.device_case("mellum_scaling_numerics", [] { run_mellum_scaling_numerics(); }); - suite.device_case("mellum_scaling_direct_numerics", [] { run_mellum_scaling_direct_numerics(); }); for (const auto & shape : { std::pair{ GGML_TYPE_Q8_0, 896 }, { GGML_TYPE_Q8_0, 704 }, { GGML_TYPE_Q5_1, 704 }, { GGML_TYPE_Q8_0, 1024 }, { GGML_TYPE_Q5_1, 768 },