diff --git a/ggml/src/ggml-hrx/CMakeLists.txt b/ggml/src/ggml-hrx/CMakeLists.txt index 805545884c96..3a398b18a81c 100644 --- a/ggml/src/ggml-hrx/CMakeLists.txt +++ b/ggml/src/ggml-hrx/CMakeLists.txt @@ -227,6 +227,8 @@ ggml_add_backend_library(ggml-hrx dispatch_registration/common/dispatch-mul-mat-common.h dispatch_registration/common/dispatch-mul-mat-weight-format.h dispatch_registration/common/dispatch-mul-mat.h + dispatch_registration/common/dispatch-amax-scale.cpp + dispatch_registration/common/dispatch-amax-scale.h dispatch_registration/common/dispatch-rope-set-rows.cpp dispatch_registration/common/dispatch-rope-set-rows.h dispatch_registration/common/dispatch-rope-utils.h diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-amax-scale.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-amax-scale.cpp new file mode 100644 index 000000000000..4bae738edecd --- /dev/null +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-amax-scale.cpp @@ -0,0 +1,189 @@ +#include "dispatch-amax-scale.h" + +#include "ggml.h" +#include "graph/graph-matcher.h" +#include "kernel-corpus/kernel-corpus-catalog-verify.h" + +#include +#include + +namespace ggml::hrx { +namespace { + +static constexpr KernelCatalogRef kAmaxScaleF32Kernel = + GGML_HRX_KERNEL_REF("loom_libs", "ggml_amax_scale_f32"); + +static bool matches_unary(const GraphNode & node, UnaryKind kind) { + const UnaryParams * params = op_params_as(node.params); + if (params == nullptr) { + return false; + } + return params->op == kind; +} + +static bool matches_scale(const GraphNode & node, float factor) { + const ScaleParams * params = op_params_as(node.params); + if (params == nullptr) { + return false; + } + const bool same_factor = params->scale == factor; + const bool no_bias = params->bias == 0.0f; + return same_factor && no_bias; +} + +static bool match_amax_scale_f32(const DispatchMatchContext & context, DispatchMatch & match) { + const GraphNode * node = context.root_node; + if (node == nullptr) { + return false; + } + const Graph & graph = context.graph; + // Intermediate values must stay private: this dispatch publishes only x/s and s. + const ggml_op ops[] = { GGML_OP_UNARY, GGML_OP_POOL_1D, GGML_OP_SCALE, GGML_OP_LOG, + GGML_OP_SCALE, GGML_OP_UNARY, GGML_OP_CLAMP, GGML_OP_SCALE, GGML_OP_UNARY }; + std::vector chain; + for (size_t i = 0; i < 9; ++i) { + const bool right_op = node->op == ops[i]; + const bool one_input = node->inputs.size() == 1; + const bool expected_node = right_op && one_input; + if (!expected_node) { + return false; + } + const Value * value = graph.values().find(node->output); + if (value == nullptr) { + return false; + } + const bool f32 = value->type == GGML_TYPE_F32; + const bool contiguous = value->contiguous; + const bool supported_value = f32 && contiguous; + if (!supported_value) { + return false; + } + chain.push_back(node); + if (i == 8) { + break; + } + const auto & consumers = graph.index().consumers(node->output); + const bool private_value = value->kind == ValueKind::Transient; + const bool one_consumer = consumers.size() == 1; + const bool can_elide = private_value && one_consumer; + if (!can_elide) { + return false; + } + node = consumers.front(); + } + const bool abs_input = matches_unary(*chain[0], UnaryKind::Abs); + const bool scale_amax = matches_scale(*chain[2], 1.0f / 1024.0f); + const bool logarithm = matches_unary(*chain[3], UnaryKind::Log); + const bool scale_log = matches_scale(*chain[4], 1.4426950408889634f); + const bool ceil_log = matches_unary(*chain[5], UnaryKind::Ceil); + const bool scale_exp = matches_scale(*chain[7], 0.6931471805599453f); + const bool exponential = matches_unary(*chain[8], UnaryKind::Exp); + const bool exact_math = abs_input && scale_amax && logarithm && scale_log && ceil_log && scale_exp && exponential; + if (!exact_math) { + return false; + } + const Pool1DParams * pool = op_params_as(chain[1]->params); + const ClampParams * clamp = op_params_as(chain[6]->params); + const Value * input = graph.values().find(chain[0]->inputs[0]); + const bool has_pool = pool != nullptr; + const bool has_clamp = clamp != nullptr; + const bool has_input = input != nullptr; + const bool all_present = has_pool && has_clamp && has_input; + if (!all_present) { + return false; + } + const bool max_pool = pool->op == GGML_OP_POOL_MAX; + const bool full_width = pool->kernel == input->ne[0]; + const bool full_stride = pool->stride == pool->kernel; + const bool no_padding = pool->padding == 0; + const bool clamp_min = clamp->min == 0.0f; + const bool clamp_max = clamp->max == 127.0f; + const bool input_f32 = input->type == GGML_TYPE_F32; + const bool bounded_width = input->ne[0] > 0 && input->ne[0] <= 32768; + const bool bounded_elements = input->element_count > 0 && input->element_count <= 134217728; + const bool supported = max_pool && full_width && full_stride && no_padding && clamp_min && clamp_max && + input_f32 && input->contiguous && bounded_width && bounded_elements; + if (!supported) { + return false; + } + const Value * scale = graph.values().find(chain.back()->output); + const int64_t rows = input->element_count / input->ne[0]; + const bool bounded_rows = rows <= 1048576; + if (!bounded_rows) { + return false; + } + for (size_t i = 0; i < chain.size(); ++i) { + const Value * value = graph.values().find(chain[i]->output); + auto expected_shape = input->ne; + if (i > 0) { + expected_shape[0] = 1; + } + const bool same_shape = value->ne == expected_shape; + const bool separate_input = value->storage_root != input->storage_root; + const bool valid_output = same_shape && separate_input; + if (!valid_output) { + return false; + } + } + const GraphNode * division = nullptr; + for (const GraphNode * consumer : graph.index().consumers(scale->id)) { + const bool is_div = consumer->op == GGML_OP_DIV; + const bool two_inputs = consumer->inputs.size() == 2; + const bool candidate = is_div && two_inputs; + if (!candidate) { + continue; + } + const bool numerator = consumer->inputs[0] == input->id; + const bool denominator = consumer->inputs[1] == scale->id; + const bool exact_division = numerator && denominator; + if (exact_division) { + division = consumer; + break; + } + } + if (division == nullptr) { + return false; + } + const Value * output = graph.values().find(division->output); + if (output == nullptr) { + return false; + } + const bool output_f32 = output->type == GGML_TYPE_F32; + const bool same_shape = output->ne == input->ne; + const bool separate_input = output->storage_root != input->storage_root; + const bool separate_scale = output->storage_root != scale->storage_root; + const bool valid_output = output_f32 && output->contiguous && same_shape && separate_input && separate_scale; + if (!valid_output) { + return false; + } + chain.push_back(division); + for (const GraphNode * covered : chain) { + if (!append_covered_node_index_once(graph, context.covered_nodes, covered, match.covered_nodes)) { + return false; + } + } + Dispatch dispatch; + dispatch.kernel = make_kernel_specialization(kAmaxScaleF32Kernel); + dispatch.kernel.integer_parameters.emplace("row_count", rows); + dispatch.kernel.integer_parameters.emplace("row_width", input->ne[0]); + dispatch.bindings.push_back({ input->id, 0, input->byte_count }); + dispatch.bindings.push_back({ output->id, 0, output->byte_count }); + dispatch.bindings.push_back({ scale->id, 0, scale->byte_count }); + match.dispatches.push_back(std::move(dispatch)); + return true; +} + +} // namespace + +void register_amax_scale_dispatch(DispatchRegistryBuilder & registry) { + registry.add({ + "common.amax_scale_f32", + GGML_OP_UNARY, + DispatchMatchKind::Fused, + 400, + DispatchSource::Common, + match_amax_scale_f32, + }); +} + +} // namespace ggml::hrx diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-amax-scale.h b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-amax-scale.h new file mode 100644 index 000000000000..9aaeba4f9ad6 --- /dev/null +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-amax-scale.h @@ -0,0 +1,9 @@ +#pragma once + +#include "dispatch_registration/dispatch-registry.h" + +namespace ggml::hrx { + +void register_amax_scale_dispatch(DispatchRegistryBuilder & registry); + +} // namespace ggml::hrx diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-common.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-common.cpp index 9ccaebe22270..c2a2b47e8d32 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-common.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-common.cpp @@ -10,6 +10,7 @@ #include "dispatch-glu.h" #include "dispatch-mul-mat-id.h" #include "dispatch-mul-mat.h" +#include "dispatch-amax-scale.h" #include "dispatch-rmsnorm.h" #include "dispatch-rope-set-rows.h" #include "dispatch-scale.h" @@ -28,6 +29,7 @@ void register_common_dispatches(DispatchRegistryBuilder & registry) { register_glu_dispatches(registry); register_mul_mat_id_dispatches(registry); register_mul_mat_dispatches(registry); + register_amax_scale_dispatch(registry); register_rope_set_rows_dispatches(registry); register_scale_dispatch(registry); register_unary_dispatch(registry); diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-flash-attention.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-flash-attention.cpp index c1fec1a8a8ce..2ad2afcc7c55 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-flash-attention.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-flash-attention.cpp @@ -109,13 +109,6 @@ static bool has_flash_attention_params(const GraphNode & node) { (params->prec == GGML_PREC_DEFAULT || params->prec == GGML_PREC_F32); } -static size_t attention_mask_byte_count(int64_t query_token_count, int64_t key_value_token_count) { - if (query_token_count <= 0 || key_value_token_count <= 0) { - return 0; - } - return static_cast(query_token_count) * static_cast(key_value_token_count) * sizeof(ggml_fp16_t); -} - static std::string to_config_value(int64_t value) { return std::to_string(value); } @@ -144,8 +137,6 @@ struct FlashAttentionMatch { const Value * mask = nullptr; const Value * output = nullptr; const GraphNode * output_layout = nullptr; - ValueId mask_binding_value; - size_t mask_binding_bytes = 0; int64_t query_token_count = 0; int64_t key_value_token_count = 0; int64_t query_head_count = 0; @@ -181,7 +172,6 @@ struct DecodeSplitFlashAttentionMatch { }; static FlashAttentionMatch match_flash_attention_f32_f16(const Graph & graph, - const CommandPlan & plan, const GraphNode * node) { FlashAttentionMatch match; if (node == nullptr || node->op != GGML_OP_FLASH_ATTN_EXT || node->inputs.size() != 4) { @@ -229,20 +219,9 @@ static FlashAttentionMatch match_flash_attention_f32_f16(const Graph & gra return {}; } - int64_t key_value_token_count = key_value_capacity; - ValueId mask_binding_value = mask->id; - size_t mask_binding_bytes = mask->byte_count; - bool mask_binding_is_alternate = false; - const size_t compact_mask_bytes = attention_mask_byte_count(query_token_count, query_token_count); - const auto * compact_mask = find_alternate_value(plan, mask->id, GGML_TYPE_F16, compact_mask_bytes); - const bool mask_is_compact = mask->ne[0] == query_token_count; - const bool mask_is_capacity = mask->ne[0] == key_value_capacity; - if (compact_mask != nullptr && mask_is_capacity && mask->ne[0] > query_token_count) { - key_value_token_count = query_token_count; - mask_binding_value = compact_mask->alternate_value; - mask_binding_bytes = compact_mask->byte_count; - mask_binding_is_alternate = true; - } else if (!mask_is_compact && !mask_is_capacity) { + const int64_t key_value_token_count = key_value_capacity; + const bool mask_is_capacity = mask->ne[0] == key_value_capacity; + if (!mask_is_capacity) { return {}; } if (!is_supported_key_value_token_count(key_value_token_count)) { @@ -255,7 +234,7 @@ static FlashAttentionMatch match_flash_attention_f32_f16(const Graph & gra !has_output_layout(*output, query_head_count, value_head_size)) { return {}; } - if (!mask_binding_is_alternate && !has_mask_layout(*mask, key_value_token_count)) { + if (!has_mask_layout(*mask, key_value_token_count)) { return {}; } @@ -265,8 +244,6 @@ static FlashAttentionMatch match_flash_attention_f32_f16(const Graph & gra match.mask = mask; match.output = output; match.output_layout = find_single_layout_alias_consumer(graph, output->id); - match.mask_binding_value = mask_binding_value; - match.mask_binding_bytes = mask_binding_bytes; match.query_token_count = query_token_count; match.key_value_token_count = key_value_token_count; match.query_head_count = query_head_count; @@ -425,7 +402,7 @@ static DispatchBinding prepare_flash_attention_value(const DispatchMatchContext } static bool match_flash_attention_gate_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) { - const FlashAttentionMatch match = match_flash_attention_f32_f16(context.graph, context.plan, context.root_node); + const FlashAttentionMatch match = match_flash_attention_f32_f16(context.graph, context.root_node); if (!match.matched() || match.output_layout == nullptr || match.output_layout->op != GGML_OP_RESHAPE) { return false; } @@ -503,7 +480,7 @@ static bool match_flash_attention_gate_dispatch(const DispatchMatchContext & con dispatch.bindings.push_back({ match.query->id, 0, match.query->byte_count }); dispatch.bindings.push_back({ match.key->id, 0, match.key->byte_count }); dispatch.bindings.push_back(prepare_flash_attention_value(context, match, dispatch_match, dispatch.kernel)); - dispatch.bindings.push_back({ match.mask_binding_value, 0, match.mask_binding_bytes }); + dispatch.bindings.push_back({ match.mask->id, 0, match.mask->byte_count }); dispatch.bindings.push_back({ raw_gate->id, 0, raw_gate->byte_count }); dispatch.bindings.push_back({ output->id, 0, output->byte_count }); dispatch_match.dispatches.push_back(std::move(dispatch)); @@ -512,7 +489,7 @@ static bool match_flash_attention_gate_dispatch(const DispatchMatchContext & con static bool match_flash_attention_f32_f16_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) { - const FlashAttentionMatch match = match_flash_attention_f32_f16(context.graph, context.plan, context.root_node); + const FlashAttentionMatch match = match_flash_attention_f32_f16(context.graph, context.root_node); if (!match.matched()) { return false; } @@ -540,7 +517,7 @@ static bool match_flash_attention_f32_f16_dispatch(const DispatchMatchContext & dispatch.bindings.push_back({ match.query->id, 0, match.query->byte_count }); dispatch.bindings.push_back({ match.key->id, 0, match.key->byte_count }); dispatch.bindings.push_back(prepare_flash_attention_value(context, match, dispatch_match, dispatch.kernel)); - dispatch.bindings.push_back({ match.mask_binding_value, 0, match.mask_binding_bytes }); + dispatch.bindings.push_back({ match.mask->id, 0, match.mask->byte_count }); dispatch.bindings.push_back({ match.query->id, 0, match.query->byte_count }); dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count }); diff --git a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-attention-postprocess.cpp b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-attention-postprocess.cpp index 2466a8e711b2..d538c71d1657 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-attention-postprocess.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-qwen-attention-postprocess.cpp @@ -21,10 +21,6 @@ static constexpr KernelCatalogRef kQwenAttentionPostprocessF32F16Kernel = GGML_HRX_KERNEL_REF("qwen3_moe", "qwen3_moe_attention_postprocess_f32_f16"); static constexpr KernelCatalogRef kQwenAttentionQkvPostprocessFusedDecodeKernel = GGML_HRX_KERNEL_REF("qwen3_moe", "qwen3_moe_attention_qkv_postprocess_fused_decode"); -static constexpr KernelCatalogRef kQwenAttentionContextBaseCaptureKernel = - GGML_HRX_KERNEL_REF("qwen", "qwen_attention_context_base_capture"); -static constexpr KernelCatalogRef kQwenAttentionMetadataKernel = - GGML_HRX_KERNEL_REF("qwen3_moe", "qwen_attention_metadata"); static constexpr int64_t kQwenAttentionHeadSize = 128; static const Value * graph_value(const Graph & graph, ValueId id) { @@ -735,71 +731,11 @@ static bool append_postprocess_covered_nodes(const DispatchMatchContext & c return true; } -static bool has_attention_metadata_initialization(const CommandPlan & plan) { - for (const Dispatch & dispatch : plan.initialization_dispatches) { - if (dispatch.kernel.kernel_id == kQwenAttentionMetadataKernel.id) { - return true; - } - } - return false; -} - static ValueId next_match_transient_value(const DispatchMatchContext & context, const DispatchMatch & dispatch_match) { return ValueId(context.next_plan_value.value + static_cast(dispatch_match.transients.size()) + static_cast(dispatch_match.completion_counter_requests.size())); } -static bool append_attention_metadata_initialization(const DispatchMatchContext & context, - const AttentionPostprocessMatch & match, - DispatchMatch & dispatch_match) { - if (has_attention_metadata_initialization(context.plan)) { - return true; - } - if (!match.flash_layouts.matched() || match.flash_layouts.flash->inputs.size() != 4) { - return true; - } - - const Value * mask = graph_value(context.graph, match.flash_layouts.flash->inputs[3]); - if (mask == nullptr || mask->type != GGML_TYPE_F16 || mask->ne[0] <= 0 || mask->ne[1] != match.query.token_count) { - return false; - } - int64_t context_capacity = mask->ne[0]; - size_t mask_byte_count = mask->byte_count; - if (context_capacity > 32768 && match.query.token_count <= 32768) { - context_capacity = match.query.token_count; - mask_byte_count = static_cast(match.query.token_count) * static_cast(match.query.token_count) * - sizeof(ggml_fp16_t); - Status metadata_status; - if (!dispatch_match.metadata.append_alternate_value( - { mask->id, mask->id, GGML_TYPE_F16, mask_byte_count, "qwen.attention.compact_mask" }, - metadata_status)) { - dispatch_match.status.append(metadata_status); - return false; - } - } - - const ValueId control = next_match_transient_value(context, dispatch_match); - dispatch_match.transients.push_back({ control, "qwen.attention.control", sizeof(int32_t), 16 }); - - Dispatch context_capture; - context_capture.kernel = make_kernel_specialization(kQwenAttentionContextBaseCaptureKernel); - context_capture.bindings.push_back({ match.query.positions->id, 0, match.query.positions->byte_count }); - context_capture.bindings.push_back({ control, 0, sizeof(int32_t) }); - dispatch_match.initialization_dispatches.push_back(std::move(context_capture)); - - Dispatch metadata; - metadata.kernel = make_kernel_specialization(kQwenAttentionMetadataKernel); - metadata.kernel.integer_parameters.emplace("token_count", match.query.token_count); - metadata.kernel.integer_parameters.emplace("context_capacity", context_capacity); - metadata.bindings.push_back({ control, 0, sizeof(int32_t) }); - metadata.bindings.push_back({ match.query.positions->id, 0, match.query.positions->byte_count }); - metadata.bindings.push_back({ match.key.cache_indices->id, 0, match.key.cache_indices->byte_count }); - metadata.bindings.push_back({ match.value.cache_indices->id, 0, match.value.cache_indices->byte_count }); - metadata.bindings.push_back({ mask->id, 0, mask_byte_count }); - dispatch_match.initialization_dispatches.push_back(std::move(metadata)); - return true; -} - static const Value * projection_weight(const Graph & graph, const GraphNode * projection) { return projection == nullptr || projection->inputs.size() != 2 ? nullptr : graph_value(graph, projection->inputs[0]); @@ -869,9 +805,6 @@ static bool match_qwen_attention_qkv_postprocess_fused_decode_dispatch(const Dis &dispatch_match.status)) { return false; } - if (!append_attention_metadata_initialization(context, match, dispatch_match)) { - return false; - } const ValueId theta_value = next_match_transient_value(context, dispatch_match); dispatch_match.transients.push_back({ theta_value, "qwen.attention_qkv_decode.theta", @@ -949,9 +882,6 @@ static bool match_qwen_attention_postprocess_dispatch(const DispatchMatchContext if (!append_postprocess_covered_nodes(context, match, dispatch_match, &dispatch_match.status)) { return false; } - if (!append_attention_metadata_initialization(context, match, dispatch_match)) { - return false; - } Dispatch dispatch; const ValueId theta_value = next_match_transient_value(context, dispatch_match); diff --git a/ggml/src/ggml-hrx/ggml-hrx.cpp b/ggml/src/ggml-hrx/ggml-hrx.cpp index 6be3802203f0..c0620720a058 100644 --- a/ggml/src/ggml-hrx/ggml-hrx.cpp +++ b/ggml/src/ggml-hrx/ggml-hrx.cpp @@ -1139,6 +1139,33 @@ static bool supported_unary_f32_tensor(const ggml_tensor * op) { return ggml::hrx::unary_kind_supported(unary_kind); } +static bool supported_full_max_pool_1d_f32_tensor(const ggml_tensor * op) { + if (op == nullptr) { + return false; + } + const bool pool_op = op->op == GGML_OP_POOL_1D; + const bool has_input = op->src[0] != nullptr; + const bool pool_with_input = pool_op && has_input; + if (!pool_with_input) { + return false; + } + + const ggml_tensor * input = op->src[0]; + const bool f32_values = input->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + const bool contiguous_values = ggml_is_contiguous(input) && ggml_is_contiguous(op); + const bool full_row = op->ne[0] == 1 && input->ne[0] == ggml_get_op_params_i32(op, 1); + const bool matching_rows = input->ne[1] == op->ne[1] && input->ne[2] == op->ne[2] && + input->ne[3] == op->ne[3]; + const bool max_pool = ggml_get_op_params_i32(op, 0) == GGML_OP_POOL_MAX; + const bool exact_window = ggml_get_op_params_i32(op, 2) == input->ne[0] && + ggml_get_op_params_i32(op, 3) == 0; + const bool bounded_width = input->ne[0] > 0 && input->ne[0] <= 32768; + const bool bounded_rows = ggml_nelements(op) > 0 && ggml_nelements(op) <= 1048576; + const bool bounded_input = ggml_nelements(input) <= 134217728; + const bool bounded = bounded_width && bounded_rows && bounded_input; + return f32_values && contiguous_values && full_row && matching_rows && max_pool && exact_window && bounded; +} + static bool device_supports_op(ggml_backend_dev_t device, const ggml_tensor * op) { GGML_UNUSED(device); if (op == nullptr) { @@ -1171,6 +1198,10 @@ static bool device_supports_op(ggml_backend_dev_t device, const ggml_tensor * op if (supported_scale_f32_tensor(op)) { return true; } + // Admit pooling for the amax-scale fusion; graph claim validates the complete chain. + if (supported_full_max_pool_1d_f32_tensor(op)) { + return true; + } if (supported_qwen_attention_projection_get_rows_tensor(op)) { return true; } diff --git a/ggml/src/ggml-hrx/graph/op-params.cpp b/ggml/src/ggml-hrx/graph/op-params.cpp index bf3374f973bf..3179f71e1cd2 100644 --- a/ggml/src/ggml-hrx/graph/op-params.cpp +++ b/ggml/src/ggml-hrx/graph/op-params.cpp @@ -48,6 +48,14 @@ static bool clamp_params_equivalent(const OpParams & lhs, const OpParams & rhs) nearly_equal(lhs_params->max, rhs_params->max); } +static bool pool_1d_params_equivalent(const OpParams & lhs, const OpParams & rhs) { + const Pool1DParams * lhs_params = op_params_as(lhs); + const Pool1DParams * rhs_params = op_params_as(rhs); + return lhs_params != nullptr && rhs_params != nullptr && lhs_params->op == rhs_params->op && + lhs_params->kernel == rhs_params->kernel && lhs_params->stride == rhs_params->stride && + lhs_params->padding == rhs_params->padding; +} + static bool glu_params_equivalent(const OpParams & lhs, const OpParams & rhs) { const GluParams * lhs_params = op_params_as(lhs); const GluParams * rhs_params = op_params_as(rhs); @@ -279,6 +287,13 @@ OpParams import_op_params(const ggml_tensor & tensor) { ggml_get_op_params_f32(&tensor, 0), ggml_get_op_params_f32(&tensor, 1), }; + case GGML_OP_POOL_1D: + return Pool1DParams{ + static_cast(ggml_get_op_params_i32(&tensor, 0)), + ggml_get_op_params_i32(&tensor, 1), + ggml_get_op_params_i32(&tensor, 2), + ggml_get_op_params_i32(&tensor, 3), + }; case GGML_OP_GLU: return GluParams{ ggml_get_glu_op(&tensor), ggml_get_op_params_i32(&tensor, 1) != 0 }; case GGML_OP_SCALE: @@ -323,6 +338,8 @@ bool op_params_equivalent(ggml_op op, const OpParams & lhs, const OpParams & rhs return argsort_params_equivalent(lhs, rhs); case GGML_OP_CLAMP: return clamp_params_equivalent(lhs, rhs); + case GGML_OP_POOL_1D: + return pool_1d_params_equivalent(lhs, rhs); case GGML_OP_GLU: return binary_params_equivalent(lhs, rhs) || glu_params_equivalent(lhs, rhs); case GGML_OP_SCALE: diff --git a/ggml/src/ggml-hrx/graph/op-params.h b/ggml/src/ggml-hrx/graph/op-params.h index 73f5a9122322..630bb71f720e 100644 --- a/ggml/src/ggml-hrx/graph/op-params.h +++ b/ggml/src/ggml-hrx/graph/op-params.h @@ -35,6 +35,13 @@ struct ClampParams { float max = 0.0f; }; +struct Pool1DParams { + ggml_op_pool op = GGML_OP_POOL_MAX; + int32_t kernel = 0; + int32_t stride = 0; + int32_t padding = 0; +}; + struct GluParams { ggml_glu_op op = GGML_GLU_OP_REGLU; bool swapped = false; @@ -120,6 +127,7 @@ using OpParams = std::variant< SoftMaxParams, ArgsortParams, ClampParams, + Pool1DParams, GluParams, ScaleParams, BinaryParams, diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json index 11c7e39e7b2c..4e1acc579461 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json @@ -652,6 +652,9 @@ }, { "path": "motifs/matmul/tiled_pair_postprocess_f32.loom" + }, + { + "path": "ops/reduction/amax_scale_f32.loom" } ], "exports": [ @@ -9582,6 +9585,55 @@ "motifs/bf16_f16.loom", "motifs/f32_f16.loom" ] + }, + { + "name": "ggml_amax_scale_f32", + "family": "loom_libs", + "symbol": "ggml_amax_scale_f32", + "source": "ops/reduction/amax_scale_f32.loom", + "workload_parameters": [ + { + "name": "row_count", + "type": "index" + }, + { + "name": "row_width", + "type": "index" + } + ], + "launch_parameters": [ + { + "name": "row_count", + "type": "index" + }, + { + "name": "row_width", + "type": "index" + } + ], + "bindings": [ + "input", + "output", + "scale" + ], + "binding_access": [ + "read", + "write", + "write" + ], + "target_selector": "", + "compile_recipe": { + "mode": "direct", + "primary_sources": [ + "ops/reduction/amax_scale_f32.loom" + ], + "library_sources": [ + "motifs/elementwise/unary_apply.loom" + ] + }, + "compile_dependencies": [ + "motifs/elementwise/unary_apply.loom" + ] } ], "link_modules": [ diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul_id/skinny_input_f32_dequant_accumulate_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul_id/skinny_input_f32_dequant_accumulate_f32.loom index 2c59b2d7d059..43bb423c20db 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul_id/skinny_input_f32_dequant_accumulate_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul_id/skinny_input_f32_dequant_accumulate_f32.loom @@ -9,7 +9,7 @@ template.decl @ggml.matmul_id.skinny_f32_f32.finish_tile(%arg0: index, %arg1: in template.decl @ggml.matmul_id.skinny_f32_f32.publish_vector4(%arg0: i1, %arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index, %arg6: index, %arg7: index, %arg8: vector<4xf32>, %arg9: buffer, %arg10: buffer, %arg11: buffer, %arg12: buffer, %arg13: buffer, %arg14: buffer, %arg15: buffer) -func.decl @ggml_dequant_weight_tile_bytes(%weight_format: index) -> (offset) +func.decl @ggml_dequant_weight_row_bytes(%weight_format: index, %hidden_size: index) -> (offset) func.decl @ggml_iq4nl_table_i8() -> (vector<16xi8>) @@ -44,6 +44,7 @@ func.def inline @ggml_moe_unpack_expert_partition_descriptor(%descriptor: i32) - } template.def<@ggml.matmul_id.skinny_f32_f32.core> device @ggml_mul_mat_id_skinny_f32_f32_core(%weight_format: index, %token_count: index, %input_size: index, %output_size: index, %route_count: index, %input_route_count: index, %expert_count: index, %epsilon: f32, %channel_tile: index, %partition_ordinal: index, %input: buffer, %expert_table: buffer, %partition_table: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer) { + %bounded_input_size = index.assume %input_size [range(%input_size, 256, 32768), multiple_of(%input_size, 32)] : index %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048)] : index %bounded_route_count = index.assume %route_count [range(%route_count, 1, 32)] : index %bounded_input_route_count = index.assume %input_route_count [range(%input_route_count, 1, 32), le(%input_route_count, %bounded_route_count)] : index @@ -63,6 +64,7 @@ template.def<@ggml.matmul_id.skinny_f32_f32.core> device @ggml_mul_mat_id_skinny %c40 = index.constant 40 : index %c63 = index.constant 63 : index %c64 = index.constant 64 : index + %c255 = index.constant 255 : index %c256 = index.constant 256 : index %c0_offset = index.constant 0 : offset %c4_bytes = index.constant 4 : offset @@ -73,6 +75,7 @@ template.def<@ggml.matmul_id.skinny_f32_f32.core> device @ggml_mul_mat_id_skinny %result_stage_bytes = index.constant 1024 : offset %c0_i32 = scalar.constant 0 : i32 %cn1_i32 = scalar.constant -1 : i32 + %c0_f32x4 = vector.constant 0.0 : vector<4xf32> %c0_f16x4 = vector.constant 0.0 : vector<4xf16> %zero_accumulator = vector.constant 0.0 : vector<8xf16> %m = index.constant 16 : index @@ -87,12 +90,12 @@ template.def<@ggml.matmul_id.skinny_f32_f32.core> device @ggml_mul_mat_id_skinny %has_more_assignment_partitions = index.cmp ugt, %assignment_partition_count, %bounded_expert_count : index %launch_partition_count = scf.select %has_more_assignment_partitions, %assignment_partition_count, %bounded_expert_count : index %assignment_table_byte_base = index.scale %bounded_expert_count, %c4_bytes : index, offset -> offset - %weight_block_bytes = func.call @ggml_dequant_weight_tile_bytes(%weight_format) : (index) -> (offset) - %quant_block_count = index.div %input_size, %c256 : index - %weight_row_bytes = index.scale %quant_block_count, %weight_block_bytes : index, offset -> offset + %padded_input_size = index.add %bounded_input_size, %c255 : index + %quant_block_count = index.div %padded_input_size, %c256 : index + %weight_row_bytes = func.call @ggml_dequant_weight_row_bytes(%weight_format, %bounded_input_size) : (index, index) -> (offset) %weight_expert_bytes = index.scale %bounded_output_size, %weight_row_bytes : index, offset -> offset %input_noalias, %expert_table_noalias, %partition_table_noalias, %weight_noalias = buffer.assume.noalias %input, %expert_table, %partition_table, %weight : buffer, buffer, buffer, buffer - %input_view = buffer.view %input_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]x[%bounded_input_route_count]x[%input_size]xf32> + %input_view = buffer.view %input_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]x[%bounded_input_route_count]x[%bounded_input_size]xf32> %assignment_view = buffer.view %expert_table_noalias[%assignment_table_byte_base] : buffer -> view<[%bounded_expert_count]x[%bounded_token_count]xi32> %partition_count_view = buffer.view %partition_table_noalias[%c0_offset] : buffer -> view<1xi32> %partition_descriptor_view = buffer.view %partition_table_noalias[%c4_bytes] : buffer -> view<[%maximum_partition_count]xi32> @@ -172,11 +175,13 @@ template.def<@ggml.matmul_id.skinny_f32_f32.core> device @ggml_mul_mat_id_skinny %local_row = index.assume %local_row0 [range(%local_row0, 0, 63)] : index %channel = index.add %channel_tile_base, %local_row : index %valid_channel = index.cmp ult, %channel, %bounded_output_size : index - %weight_values = scf.if %valid_channel -> (vector<4xf16>) { + %weight_k = index.add %k_origin, %load_k : index + %valid_k = index.cmp ult, %weight_k, %bounded_input_size : index + %valid_weight = scalar.andi %valid_channel, %valid_k : i1 + %weight_values = scf.if %valid_weight -> (vector<4xf16>) { %channel_byte_add = index.scale %channel, %weight_row_bytes : index, offset -> offset %row_byte_base = index.add %expert_byte_base, %channel_byte_add : offset - %weight_k = index.add %k_origin, %load_k : index - %decoded = func.call @ggml_dequant_f16_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight_noalias, %row_byte_base, %input_size, %quant_block, %quant_group, %load_packet, %weight_k) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index) -> (vector<4xf16>) + %decoded = func.call @ggml_dequant_f16_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight_noalias, %row_byte_base, %bounded_input_size, %quant_block, %quant_group, %load_packet, %weight_k) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index) -> (vector<4xf16>) scf.yield %decoded : vector<4xf16> } else { scf.yield %c0_f16x4 : vector<4xf16> @@ -187,7 +192,10 @@ template.def<@ggml.matmul_id.skinny_f32_f32.core> device @ggml_mul_mat_id_skinny %activation_row = index.assume %local_row [range(%local_row, 0, 31)] : index %assignment_i32 = view.load %route_stage_view[%activation_row] : view<32xi32> -> i32 %valid_assignment = scalar.cmpi sge, %assignment_i32, %c0_i32 : i32 - %activation_values = scf.if %valid_assignment -> (vector<4xf16>) { + %input_k = index.add %k_origin, %load_k : index + %valid_input = index.cmp ult, %input_k, %bounded_input_size : index + %valid_activation = scalar.andi %valid_assignment, %valid_input : i1 + %activation_values = scf.if %valid_activation -> (vector<4xf16>) { %assignment0 = index.cast %assignment_i32 : i32 to index %assignment = index.assume %assignment0 [range(%assignment0, 0, 65535)] : index %bounded_assignment, %bounded_assignment_count = index.assume %assignment, %assignment_count [lt(%assignment, %assignment_count)] : index, index @@ -196,8 +204,8 @@ template.def<@ggml.matmul_id.skinny_f32_f32.core> device @ggml_mul_mat_id_skinny %token, %input_token_count = index.assume %token0, %bounded_token_count [lt(%token0, %bounded_token_count)] : index, index %input_route0 = index.rem %route0, %bounded_input_route_count : index %input_route = index.assume %input_route0 [range(%input_route0, 0, 31), lt(%input_route0, %bounded_input_route_count)] : index - %input_k = index.add %k_origin, %load_k : index - %loaded = vector.load %input_view[%token, %input_route, %input_k] : view<[%bounded_token_count]x[%bounded_input_route_count]x[%input_size]xf32> -> vector<4xf32> + %mask = vector.mask.range [%input_k to %bounded_input_size step %c1] : index -> vector<4xi1> + %loaded = vector.load.mask %input_view[%token, %input_route, %input_k], %mask, %c0_f32x4 : view<[%bounded_token_count]x[%bounded_input_route_count]x[%bounded_input_size]xf32>, vector<4xi1>, vector<4xf32> %converted = vector.fptrunc %loaded : vector<4xf32> to vector<4xf16> scf.yield %converted : vector<4xf16> } else { diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul_id/tiled_input_f32_dequant_accumulate_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul_id/tiled_input_f32_dequant_accumulate_f32.loom index 359fcf33e146..422aa66c4221 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul_id/tiled_input_f32_dequant_accumulate_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/matmul_id/tiled_input_f32_dequant_accumulate_f32.loom @@ -9,7 +9,7 @@ template.decl @ggml.matmul_id.tiled_f32_f32.finish_tile(%arg0: index, %arg1: ind template.decl @ggml.matmul_id.tiled_f32_f32.publish_vector4(%arg0: i1, %arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index, %arg6: index, %arg7: index, %arg8: vector<4xf32>, %arg9: buffer, %arg10: buffer, %arg11: buffer, %arg12: buffer, %arg13: buffer, %arg14: buffer, %arg15: buffer) -func.decl @ggml_dequant_weight_tile_bytes(%weight_format: index) -> (offset) +func.decl @ggml_dequant_weight_row_bytes(%weight_format: index, %hidden_size: index) -> (offset) func.decl @ggml_iq4nl_table_i8() -> (vector<16xi8>) @@ -44,6 +44,7 @@ func.def inline @ggml_moe_unpack_expert_partition_descriptor(%descriptor: i32) - } template.def<@ggml.matmul_id.tiled_f32_f32.core> device @ggml_mul_mat_id_tiled_f32_f32_core(%weight_format: index, %token_count: index, %input_size: index, %output_size: index, %route_count: index, %input_route_count: index, %expert_count: index, %epsilon: f32, %channel_tile: index, %partition_ordinal: index, %input: buffer, %expert_table: buffer, %partition_table: buffer, %weight: buffer, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer) { + %bounded_input_size = index.assume %input_size [range(%input_size, 256, 32768), multiple_of(%input_size, 32)] : index %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048)] : index %bounded_route_count = index.assume %route_count [range(%route_count, 1, 32)] : index %bounded_input_route_count = index.assume %input_route_count [range(%input_route_count, 1, 32), le(%input_route_count, %bounded_route_count)] : index @@ -63,6 +64,7 @@ template.def<@ggml.matmul_id.tiled_f32_f32.core> device @ggml_mul_mat_id_tiled_f %c40 = index.constant 40 : index %c63 = index.constant 63 : index %c64 = index.constant 64 : index + %c255 = index.constant 255 : index %c256 = index.constant 256 : index %c0_offset = index.constant 0 : offset %c4_bytes = index.constant 4 : offset @@ -73,6 +75,7 @@ template.def<@ggml.matmul_id.tiled_f32_f32.core> device @ggml_mul_mat_id_tiled_f %result_stage_bytes = index.constant 1024 : offset %c0_i32 = scalar.constant 0 : i32 %cn1_i32 = scalar.constant -1 : i32 + %c0_f32x4 = vector.constant 0.0 : vector<4xf32> %c0_f16x4 = vector.constant 0.0 : vector<4xf16> %zero_accumulator = vector.constant 0.0 : vector<8xf16> %m = index.constant 16 : index @@ -87,12 +90,12 @@ template.def<@ggml.matmul_id.tiled_f32_f32.core> device @ggml_mul_mat_id_tiled_f %has_more_assignment_partitions = index.cmp ugt, %assignment_partition_count, %bounded_expert_count : index %launch_partition_count = scf.select %has_more_assignment_partitions, %assignment_partition_count, %bounded_expert_count : index %assignment_table_byte_base = index.scale %bounded_expert_count, %c4_bytes : index, offset -> offset - %weight_block_bytes = func.call @ggml_dequant_weight_tile_bytes(%weight_format) : (index) -> (offset) - %quant_block_count = index.div %input_size, %c256 : index - %weight_row_bytes = index.scale %quant_block_count, %weight_block_bytes : index, offset -> offset + %padded_input_size = index.add %bounded_input_size, %c255 : index + %quant_block_count = index.div %padded_input_size, %c256 : index + %weight_row_bytes = func.call @ggml_dequant_weight_row_bytes(%weight_format, %bounded_input_size) : (index, index) -> (offset) %weight_expert_bytes = index.scale %bounded_output_size, %weight_row_bytes : index, offset -> offset %input_noalias, %expert_table_noalias, %partition_table_noalias, %weight_noalias = buffer.assume.noalias %input, %expert_table, %partition_table, %weight : buffer, buffer, buffer, buffer - %input_view = buffer.view %input_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]x[%bounded_input_route_count]x[%input_size]xf32> + %input_view = buffer.view %input_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]x[%bounded_input_route_count]x[%bounded_input_size]xf32> %assignment_view = buffer.view %expert_table_noalias[%assignment_table_byte_base] : buffer -> view<[%bounded_expert_count]x[%bounded_token_count]xi32> %partition_count_view = buffer.view %partition_table_noalias[%c0_offset] : buffer -> view<1xi32> %partition_descriptor_view = buffer.view %partition_table_noalias[%c4_bytes] : buffer -> view<[%maximum_partition_count]xi32> @@ -172,11 +175,13 @@ template.def<@ggml.matmul_id.tiled_f32_f32.core> device @ggml_mul_mat_id_tiled_f %local_row = index.assume %local_row0 [range(%local_row0, 0, 63)] : index %channel = index.add %channel_tile_base, %local_row : index %valid_channel = index.cmp ult, %channel, %bounded_output_size : index - %weight_values = scf.if %valid_channel -> (vector<4xf16>) { + %weight_k = index.add %k_origin, %load_k : index + %valid_k = index.cmp ult, %weight_k, %bounded_input_size : index + %valid_weight = scalar.andi %valid_channel, %valid_k : i1 + %weight_values = scf.if %valid_weight -> (vector<4xf16>) { %channel_byte_add = index.scale %channel, %weight_row_bytes : index, offset -> offset %row_byte_base = index.add %expert_byte_base, %channel_byte_add : offset - %weight_k = index.add %k_origin, %load_k : index - %decoded = func.call @ggml_dequant_f16_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight_noalias, %row_byte_base, %input_size, %quant_block, %quant_group, %load_packet, %weight_k) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index) -> (vector<4xf16>) + %decoded = func.call @ggml_dequant_f16_vector4(%iq4nl_table, %iq3s_grid_low0, %iq3s_grid_low1, %iq3s_grid_low2, %iq3s_grid_low3, %iq3s_grid_low4, %iq3s_grid_low5, %iq3s_grid_low6, %iq3s_grid_low7, %iq3s_grid_low8, %iq3s_grid_low9, %iq3s_grid_low10, %iq3s_grid_low11, %iq3s_grid_low12, %iq3s_grid_low13, %iq3s_grid_low14, %iq3s_grid_low15, %iq3s_grid_high0, %iq3s_grid_high1, %iq3s_grid_high2, %iq3s_grid_high3, %iq3s_grid_high4, %iq3s_grid_high5, %iq3s_grid_high6, %iq3s_grid_high7, %iq3s_grid_high8, %iq3s_grid_high9, %iq3s_grid_high10, %iq3s_grid_high11, %iq3s_grid_high12, %iq3s_grid_high13, %iq3s_grid_high14, %iq3s_grid_high15, %weight_format, %weight_noalias, %row_byte_base, %bounded_input_size, %quant_block, %quant_group, %load_packet, %weight_k) : (vector<16xi8>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, vector<32xf32>, index, buffer, offset, index, index, index, index, index) -> (vector<4xf16>) scf.yield %decoded : vector<4xf16> } else { scf.yield %c0_f16x4 : vector<4xf16> @@ -187,7 +192,10 @@ template.def<@ggml.matmul_id.tiled_f32_f32.core> device @ggml_mul_mat_id_tiled_f %activation_row = index.assume %local_row [range(%local_row, 0, 31)] : index %assignment_i32 = view.load %route_stage_view[%activation_row] : view<32xi32> -> i32 %valid_assignment = scalar.cmpi sge, %assignment_i32, %c0_i32 : i32 - %activation_values = scf.if %valid_assignment -> (vector<4xf16>) { + %input_k = index.add %k_origin, %load_k : index + %valid_input = index.cmp ult, %input_k, %bounded_input_size : index + %valid_activation = scalar.andi %valid_assignment, %valid_input : i1 + %activation_values = scf.if %valid_activation -> (vector<4xf16>) { %assignment0 = index.cast %assignment_i32 : i32 to index %assignment = index.assume %assignment0 [range(%assignment0, 0, 65535)] : index %bounded_assignment, %bounded_assignment_count = index.assume %assignment, %assignment_count [lt(%assignment, %assignment_count)] : index, index @@ -196,8 +204,8 @@ template.def<@ggml.matmul_id.tiled_f32_f32.core> device @ggml_mul_mat_id_tiled_f %token, %input_token_count = index.assume %token0, %bounded_token_count [lt(%token0, %bounded_token_count)] : index, index %input_route0 = index.rem %route0, %bounded_input_route_count : index %input_route = index.assume %input_route0 [range(%input_route0, 0, 31), lt(%input_route0, %bounded_input_route_count)] : index - %input_k = index.add %k_origin, %load_k : index - %loaded = vector.load %input_view[%token, %input_route, %input_k] : view<[%bounded_token_count]x[%bounded_input_route_count]x[%input_size]xf32> -> vector<4xf32> + %mask = vector.mask.range [%input_k to %bounded_input_size step %c1] : index -> vector<4xi1> + %loaded = vector.load.mask %input_view[%token, %input_route, %input_k], %mask, %c0_f32x4 : view<[%bounded_token_count]x[%bounded_input_route_count]x[%bounded_input_size]xf32>, vector<4xi1>, vector<4xf32> %converted = vector.fptrunc %loaded : vector<4xf32> to vector<4xf16> scf.yield %converted : vector<4xf16> } else { diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_postops_next_rmsnorm_publish_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_postops_next_rmsnorm_publish_f32.loom index c25b9210877e..837ae0b4fd87 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_postops_next_rmsnorm_publish_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_postops_next_rmsnorm_publish_f32.loom @@ -14,7 +14,7 @@ template.decl @ggml.matmul_id.skinny_f32_f32.store_vector4(%arg0: index, %arg1: amdgpu.target @ggml_mul_mat_id_skinny_input_f32_postops_next_rmsnorm_publish_f32_gfx11_wave64 {subgroup_size = 64} -config.decl @ggml.mul_mat_id_postops.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 256)] +config.decl @ggml.mul_mat_id_postops.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 32)] config.decl @ggml.mul_mat_id_postops.output_size : %value: index where [range(%value, 128, 32768), multiple_of(%value, 128)] diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_postops_publish_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_postops_publish_f32.loom index a2184804de65..77b214a5a1d8 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_postops_publish_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_postops_publish_f32.loom @@ -12,7 +12,7 @@ template.decl @ggml.matmul_id.skinny_f32_f32.store_vector4(%arg0: index, %arg1: amdgpu.target @ggml_mul_mat_id_skinny_input_f32_postops_publish_f32_gfx11_wave64 {subgroup_size = 64} -config.decl @ggml.mul_mat_id_postops.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 256)] +config.decl @ggml.mul_mat_id_postops.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 32)] config.decl @ggml.mul_mat_id_postops.output_size : %value: index where [range(%value, 1, 262144)] diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_publish_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_publish_f32.loom index d6a1a16cbf01..0acfbcddfe4a 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_publish_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_publish_f32.loom @@ -11,7 +11,7 @@ template.decl @ggml.matmul_id.skinny_f32_f32.publish_vector4(%publish_word: i1, amdgpu.target @ggml_mul_mat_id_skinny_input_f32_publish_f32_gfx11_wave64 {subgroup_size = 64} -config.decl @ggml.mul_mat_id.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 256)] +config.decl @ggml.mul_mat_id.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 32)] config.decl @ggml.mul_mat_id.output_size : %value: index where [range(%value, 1, 262144)] diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_postops_next_rmsnorm_publish_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_postops_next_rmsnorm_publish_f32.loom index 0c60dd9fd87c..791c90b7c9a0 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_postops_next_rmsnorm_publish_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_postops_next_rmsnorm_publish_f32.loom @@ -14,7 +14,7 @@ template.decl @ggml.matmul_id.tiled_f32_f32.store_vector4(%arg0: index, %arg1: i amdgpu.target @ggml_mul_mat_id_tiled_input_f32_postops_next_rmsnorm_publish_f32_gfx11_wave64 {subgroup_size = 64} -config.decl @ggml.mul_mat_id_postops.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 256)] +config.decl @ggml.mul_mat_id_postops.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 32)] config.decl @ggml.mul_mat_id_postops.output_size : %value: index where [range(%value, 128, 32768), multiple_of(%value, 128)] diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_postops_publish_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_postops_publish_f32.loom index 8d9596b1d063..a6b13afe0cff 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_postops_publish_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_postops_publish_f32.loom @@ -12,7 +12,7 @@ template.decl @ggml.matmul_id.tiled_f32_f32.store_vector4(%arg0: index, %arg1: i amdgpu.target @ggml_mul_mat_id_tiled_input_f32_postops_publish_f32_gfx11_wave64 {subgroup_size = 64} -config.decl @ggml.mul_mat_id_postops.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 256)] +config.decl @ggml.mul_mat_id_postops.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 32)] config.decl @ggml.mul_mat_id_postops.output_size : %value: index where [range(%value, 1, 262144)] diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_publish_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_publish_f32.loom index 9d92fd4d15cc..28ee2469795a 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_publish_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_publish_f32.loom @@ -11,7 +11,7 @@ template.decl @ggml.matmul_id.tiled_f32_f32.publish_vector4(%publish_word: i1, % amdgpu.target @ggml_mul_mat_id_tiled_input_f32_publish_f32_gfx11_wave64 {subgroup_size = 64} -config.decl @ggml.mul_mat_id.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 256)] +config.decl @ggml.mul_mat_id.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 32)] config.decl @ggml.mul_mat_id.output_size : %value: index where [range(%value, 1, 262144)] diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/reduction/amax_scale_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/reduction/amax_scale_f32.loom new file mode 100644 index 000000000000..f46f11d02ae9 --- /dev/null +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/reduction/amax_scale_f32.loom @@ -0,0 +1,60 @@ +// Copyright 2026 The HRX Authors +// SPDX-License-Identifier: Apache-2.0 + +template.decl @ggml.unary_f32.apply(%arg0: index, %arg1: f32) -> (f32) + +amdgpu.target @ggml_amax_scale_f32_gfx11_wave64 {subgroup_size = 64} + +kernel.def target(@ggml_amax_scale_f32_gfx11_wave64) export("ggml_amax_scale_f32") @ggml_amax_scale_f32(%row_count: index, %row_width: index) { + %one = index.constant 1 : index + %twofiftysix = index.constant 256 : index + kernel.launch.config workgroups(%row_count, %one, %one) workgroup_size(%twofiftysix, %one, %one) : index +} launch(%row_count: index, %row_width: index, %input: buffer, %output: buffer, %scale: buffer) where [range(%row_count, 1, 1048576), range(%row_width, 1, 32768)] { + %rows = index.assume %row_count [range(%row_count, 1, 1048576)] : index + %width = index.assume %row_width [range(%row_width, 1, 32768)] : index + %row = kernel.workgroup.id : index + %lane = kernel.workitem.id : index + %zero = index.constant 0 : index + %stride = index.constant 256 : index + %zero_f32 = scalar.constant 0.0 : f32 + %abs_op = index.constant 1 : index + %log_op = index.constant 22 : index + %ceil_op = index.constant 7 : index + %exp_op = index.constant 18 : index + %element_count = index.mul %rows, %width : index + %row_base = index.mul %row, %width : index + %zero_offset = index.constant 0 : offset + %input_view = buffer.view %input[%zero_offset] : buffer -> view<[%element_count]xf32> + %output_view = buffer.view %output[%zero_offset] : buffer -> view<[%element_count]xf32> + %scale_view = buffer.view %scale[%zero_offset] : buffer -> view<[%rows]xf32> + %maximum = scf.for %column = [%lane to %width step %stride](%prior = %zero_f32 : f32) -> (f32) { + %offset = index.add %row_base, %column : index + %value = view.load %input_view[%offset] : view<[%element_count]xf32> -> f32 + %absolute = template.apply<@ggml.unary_f32.apply>(%abs_op, %value) : (index, f32) -> (f32) + %next = scalar.maxnumf %absolute, %prior : f32 + scf.yield %next : f32 + } + %row_maximum = kernel.workgroup.reduce %maximum : f32 + %inverse_limit = scalar.constant 0.0009765625 : f32 + %log2_e = scalar.constant 1.4426950408889634 : f32 + %ln_2 = scalar.constant 0.6931471805599453 : f32 + %max_exponent = scalar.constant 127.0 : f32 + %relative = scalar.mulf %row_maximum, %inverse_limit : f32 + %logarithm = template.apply<@ggml.unary_f32.apply>(%log_op, %relative) : (index, f32) -> (f32) + %log2 = scalar.mulf %logarithm, %log2_e : f32 + %rounded = template.apply<@ggml.unary_f32.apply>(%ceil_op, %log2) : (index, f32) -> (f32) + %exponent = scalar.clampf %rounded, %zero_f32, %max_exponent : f32 + %natural_exponent = scalar.mulf %exponent, %ln_2 : f32 + %factor = template.apply<@ggml.unary_f32.apply>(%exp_op, %natural_exponent) : (index, f32) -> (f32) + %is_first = index.cmp eq, %lane, %zero : index + scf.if %is_first { + view.store %factor, %scale_view[%row] : f32, view<[%rows]xf32> + } + scf.for %column = [%lane to %width step %stride] { + %offset = index.add %row_base, %column : index + %value = view.load %input_view[%offset] : view<[%element_count]xf32> -> f32 + %normalized = scalar.divf %value, %factor : f32 + view.store %normalized, %output_view[%offset] : f32, view<[%element_count]xf32> + } + kernel.return +} diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index b964c18a7c77..a90b8e9ea567 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -5948,46 +5948,17 @@ static void run_qwen_attention_postprocess_dispatch_checks() { producer_index_for_tensor(imported.graph, tensors.query_reshape), match)); append_match_to_plan(plan, match, covered_nodes); REQUIRE(plan.valid()); - REQUIRE(plan.initialization_dispatches.size() == 2); - - const ggml::hrx::Dispatch & context_capture = plan.initialization_dispatches[0]; - const ggml::hrx::Dispatch & metadata = plan.initialization_dispatches[1]; - REQUIRE(kernel_name_for_id(context_capture.kernel.kernel_id) == "qwen:qwen_attention_context_base_capture"); - REQUIRE(kernel_name_for_id(metadata.kernel.kernel_id) == "qwen3_moe:qwen_attention_metadata"); - REQUIRE(context_capture.bindings.size() == 2); - REQUIRE(metadata.bindings.size() == 5); - REQUIRE(context_capture.bindings[1].value == metadata.bindings[0].value); - REQUIRE(metadata.kernel.integer_parameters.at("token_count") == token_count); - REQUIRE(metadata.kernel.integer_parameters.at("context_capacity") == cache_row_count); - - const ggml::hrx::Value * positions_value = imported.graph.values().find_tensor(tensors.positions); - const ggml::hrx::Value * key_indices_value = imported.graph.values().find_tensor(tensors.key_cache_indices); - const ggml::hrx::Value * value_indices_value = imported.graph.values().find_tensor(tensors.value_cache_indices); - const ggml::hrx::Value * mask_value = imported.graph.values().find_tensor(tensors.mask); - REQUIRE(positions_value != nullptr); - REQUIRE(key_indices_value != nullptr); - REQUIRE(value_indices_value != nullptr); - REQUIRE(mask_value != nullptr); - REQUIRE(context_capture.bindings[0].value == positions_value->id); - REQUIRE(metadata.bindings[1].value == positions_value->id); - REQUIRE(metadata.bindings[2].value == key_indices_value->id); - REQUIRE(metadata.bindings[3].value == value_indices_value->id); - REQUIRE(metadata.bindings[4].value == mask_value->id); + REQUIRE(plan.initialization_dispatches.empty()); + REQUIRE(plan.dispatches.size() == 1); + const ggml::hrx::Dispatch & postprocess = plan.dispatches.front(); + REQUIRE(postprocess.bindings[0].value == imported.graph.values().find_tensor(tensors.positions)->id); + REQUIRE(postprocess.bindings[1].value == imported.graph.values().find_tensor(tensors.key_cache_indices)->id); + REQUIRE(postprocess.bindings[2].value == imported.graph.values().find_tensor(tensors.value_cache_indices)->id); const ggml::hrx::CommandProgram commands = ggml::hrx::build_command_program(imported.graph, plan, ggml::hrx::get_qwen_kernel_corpus(), "gfx1151"); REQUIRE(commands.valid()); - REQUIRE(commands.initialization_commands.size() == 2); - REQUIRE(commands.initialization_commands[0].bindings[0].name == "positions"); - REQUIRE(commands.initialization_commands[0].bindings[1].name == "control"); - REQUIRE(commands.initialization_commands[1].bindings[0].name == "control"); - REQUIRE(commands.initialization_commands[1].bindings[4].name == "attention_mask"); - REQUIRE(ggml::hrx::find_transient_allocation(commands.transients, context_capture.bindings[1].value) != - nullptr); - - ggml::hrx::CommandProgram initialization_commands = copy_command_program_shape(commands); - initialization_commands.commands.clear(); - REQUIRE(command_program_verifies(initialization_commands)); + REQUIRE(commands.initialization_commands.empty()); } ggml_free(ctx); @@ -10117,14 +10088,15 @@ static CommonMulMatIdPostOpsTensors build_common_mul_mat_id_postops_graph(ggml_c bool include_residual, bool include_rmsnorm, ggml_type weight_type = GGML_TYPE_Q4_K, - int64_t token_count = 4) { + int64_t token_count = 4, + int64_t input_size = 256, + int64_t input_routes = 1) { CommonMulMatIdPostOpsTensors tensors; constexpr int64_t route_count = 8; - constexpr int64_t input_size = 256; constexpr int64_t output_size = 128; constexpr int64_t expert_count = 16; tensors.route_ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, route_count, token_count); - tensors.input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, input_size, 1, token_count); + tensors.input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, input_size, input_routes, token_count); tensors.weight = ggml_new_tensor_3d(ctx, weight_type, input_size, output_size, expert_count); REQUIRE(tensors.route_ids != nullptr); REQUIRE(tensors.input != nullptr); @@ -13796,6 +13768,330 @@ static void run_scale_f32() { ggml_backend_free(backend); } +struct AmaxScaleTestGraph { + ggml_tensor * input; + ggml_tensor * absolute; + ggml_tensor * scale; + ggml_tensor * output; +}; + +static AmaxScaleTestGraph build_amax_scale_test_graph(ggml_context * ctx, int64_t width, int64_t tokens, + float limit_inverse = 1.0f / 1024.0f) { + ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, width, 8, tokens); + ggml_tensor * absolute = ggml_abs(ctx, input); + ggml_tensor * pooled = ggml_pool_1d(ctx, absolute, GGML_OP_POOL_MAX, width, width, 0); + ggml_tensor * scale = ggml_scale(ctx, pooled, limit_inverse); + scale = ggml_log(ctx, scale); + scale = ggml_scale(ctx, scale, 1.4426950408889634f); + scale = ggml_ceil(ctx, scale); + scale = ggml_clamp(ctx, scale, 0.0f, 127.0f); + scale = ggml_scale(ctx, scale, 0.6931471805599453f); + scale = ggml_exp(ctx, scale); + return { input, absolute, scale, ggml_div(ctx, input, scale) }; +} + +static void run_amax_scale_fusion_dispatch_checks() { + for (int variant = 0; variant < 6; ++variant) { + ggml_context * ctx = ggml_init({ 1024 * 1024, nullptr, true }); + REQUIRE(ctx != nullptr); + const float inverse = variant == 1 ? 1.0f / 512.0f : 1.0f / 1024.0f; + auto tensors = build_amax_scale_test_graph(ctx, 896, 16, inverse); + if (variant == 4) { + tensors.output = ggml_div_inplace(ctx, tensors.input, tensors.scale); + } + if (variant == 2) { + ggml_set_output(tensors.absolute); + } + ggml_cgraph * graph = ggml_new_graph(ctx); + ggml_build_forward_expand(graph, tensors.output); + // The scale remains observable and feeds the eventual output rescaling. + ggml_set_output(tensors.scale); + ggml_build_forward_expand(graph, ggml_mul(ctx, tensors.output, tensors.scale)); + if (variant == 3) { + ggml_build_forward_expand(graph, ggml_scale(ctx, tensors.absolute, 2.0f)); + } + if (variant == 5) { + ggml_tensor * alias = ggml_view_1d(ctx, tensors.absolute, 896, 0); + ggml_set_output(alias); + ggml_build_forward_expand(graph, alias); + } + auto imported = ggml::hrx::import_ggml_graph(*graph); + REQUIRE(imported.valid()); + ggml::hrx::DispatchScheduler scheduler; + const bool scheduled = scheduler.schedule_graph(imported.graph, test_dispatch_target()); + if (variant != 0) { + // Without standalone pool/clamp dispatches, these graphs must reject the fusion and the full claim. + std::vector covered(imported.graph.nodes().size(), false); + ggml::hrx::CommandPlan plan; + ggml::hrx::DispatchMatch match; + const size_t root = producer_index_for_tensor(imported.graph, tensors.absolute); + if (match_dispatch_at_index(imported.graph, plan, covered, root, match)) { + for (const auto & dispatch : match.dispatches) { + REQUIRE(kernel_name_for_id(dispatch.kernel.kernel_id) != "loom_libs:ggml_amax_scale_f32"); + } + } + REQUIRE(!scheduled); + ggml_free(ctx); + continue; + } + REQUIRE(scheduled); + size_t fused_count = 0; + for (const auto & dispatch : scheduler.plan().dispatches) { + if (kernel_name_for_id(dispatch.kernel.kernel_id) == "loom_libs:ggml_amax_scale_f32") { + ++fused_count; + REQUIRE(dispatch.bindings.size() == 3); + REQUIRE(dispatch.kernel.integer_parameters.at("row_width") == 896); + REQUIRE(dispatch.kernel.integer_parameters.at("row_count") == 128); + } + } + REQUIRE(fused_count == 1); + REQUIRE(scheduler.plan().dispatches.size() == 2); + ggml_free(ctx); + } +} + +static std::vector run_amax_scale_fusion_case(ggml_backend_t backend, int64_t width, + int64_t tokens, bool nonfinite = false) { + ggml_context * ctx = ggml_init({ 1024 * 1024, nullptr, true }); + REQUIRE(ctx != nullptr); + const auto tensors = build_amax_scale_test_graph(ctx, width, tokens); + ggml_set_output(tensors.scale); + ggml_cgraph * graph = ggml_new_graph(ctx); + ggml_build_forward_expand(graph, tensors.output); + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); + REQUIRE(buffer != nullptr); + const float amplitudes[] = { 0.0f, 0.000125f, 1024.0f, std::nextafter(1024.0f, 0.0f), + std::nextafter(1024.0f, 2048.0f), 65536.0f, 1.0e6f, 1.0e30f }; + std::vector input(width * 8 * tokens); + for (size_t i = 0; i < input.size(); ++i) { + const size_t row = i / width; + const size_t column = i % width; + const float fraction = static_cast(column % 17) / 16.0f; + input[i] = amplitudes[row % 8] * fraction * (column % 2 ? -1.0f : 1.0f); + } + for (size_t row = 0; row < static_cast(8 * tokens); ++row) { + input[(row + 1) * width - 1] = -amplitudes[row % 8]; + } + if (nonfinite) { + std::fill(input.begin(), input.begin() + width, std::numeric_limits::quiet_NaN()); + input[2 * width - 1] = std::numeric_limits::infinity(); + input[3 * width - 1] = -std::numeric_limits::infinity(); + input[4 * width - 1] = std::numeric_limits::quiet_NaN(); + input[5 * width - 1] = std::numeric_limits::max(); + } + ggml_backend_tensor_set(tensors.input, input.data(), 0, input.size() * sizeof(float)); + REQUIRE(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(backend); + const size_t rows = 8 * tokens; + std::vector output(input.size() + rows); + ggml_backend_tensor_get(tensors.output, output.data(), 0, input.size() * sizeof(float)); + ggml_backend_tensor_get(tensors.scale, output.data() + input.size(), 0, rows * sizeof(float)); + const size_t roundtrip_elements = nonfinite ? 0 : input.size(); + for (size_t i = 0; i < roundtrip_elements; ++i) { + const float recovered = output[i] * output[input.size() + i / width]; + REQUIRE(std::isfinite(output[i])); + REQUIRE(std::fabs(recovered - input[i]) <= 3.0e-6f * std::fabs(input[i]) + 1.0e-7f); + } + ggml_backend_buffer_free(buffer); + ggml_free(ctx); + return output; +} + +static void run_amax_scale_fusion_numerics() { + ggml_backend_t cpu = ggml_backend_cpu_init(); + ggml_backend_t hrx = ggml_backend_hrx_init(0); + REQUIRE(cpu != nullptr); + REQUIRE(hrx != nullptr); + for (int64_t width : { 1, 64, 257, 896, 32768 }) { + for (int64_t tokens : { 1, 16 }) { + const auto reference = run_amax_scale_fusion_case(cpu, width, tokens); + const auto fused = run_amax_scale_fusion_case(hrx, width, tokens); + REQUIRE(fused.size() == reference.size()); + for (size_t i = 0; i < fused.size(); ++i) { + REQUIRE(std::isfinite(fused[i])); + const bool close_to_cpu = std::fabs(fused[i] - reference[i]) <= 3.0e-6f * std::fabs(reference[i]) + 1.0e-7f; + if (!close_to_cpu) { + std::fprintf(stderr, "amax width=%lld tokens=%lld i=%zu cpu=%.9g hrx=%.9g\n", + (long long) width, (long long) tokens, i, reference[i], fused[i]); + } + REQUIRE(close_to_cpu); + } + } + } + // HRX maxnum ignores NaNs in the reduction. Division preserves each NaN input. + constexpr size_t width = 257; + constexpr size_t elements = width * 8; + const auto fused = run_amax_scale_fusion_case(hrx, width, 1, true); + for (size_t column = 0; column < width; ++column) { + REQUIRE(std::isnan(fused[column])); + } + REQUIRE(fused[elements] == 1.0f); + REQUIRE(fused[elements + 3] == 1.0f); + REQUIRE(std::isnan(fused[4 * width - 1])); + REQUIRE(fused[3 * width + 1] == -std::nextafter(1024.0f, 0.0f) / 16.0f); + REQUIRE(fused[2 * width - 1] == std::numeric_limits::infinity()); + REQUIRE(fused[3 * width - 1] == -std::numeric_limits::infinity()); + // Infinite maxima saturate the exponent at 127; the resulting scale stays finite. + for (size_t row : { 1, 2 }) { + REQUIRE(std::isfinite(fused[elements + row])); + REQUIRE(std::fabs(fused[elements + row] / std::ldexp(1.0f, 127) - 1.0f) <= 3.0e-6f); + } + REQUIRE(std::isfinite(fused[5 * width - 1])); + REQUIRE(std::fabs(fused[5 * width - 1] - 1024.0f) <= 3.0e-6f * 1024.0f); + ggml_backend_free(hrx); + ggml_backend_free(cpu); +} + +struct RoutedKTailOutputs { + std::vector raw; + std::vector normalized; +}; + +static RoutedKTailOutputs run_routed_k_tail_case(ggml_backend_t backend, + ggml_type weight_type, + int64_t input_size, + int64_t token_count, + int postops) { + ggml_context * ctx = ggml_init({ 4 * 1024 * 1024, nullptr, true }); + REQUIRE(ctx != nullptr); + const bool has_postops = postops != 0; + const bool has_rmsnorm = postops == 2; + const CommonMulMatIdPostOpsTensors tensors = build_common_mul_mat_id_postops_graph( + ctx, has_postops, has_postops, has_rmsnorm, weight_type, token_count, input_size, 8); + // HRX may keep internal values in private storage; both observed values must be external outputs. + ggml_set_output(tensors.residual_output); + ggml_set_output(tensors.normalized_output); + ggml_cgraph * graph = ggml_new_graph(ctx); + REQUIRE(graph != nullptr); + ggml_build_forward_expand(graph, tensors.normalized_output); + + // Require each of the six shared exports, so another dispatch cannot hide a missing tail path. + ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*graph); + REQUIRE(imported.valid()); + std::vector covered(imported.graph.nodes().size(), false); + ggml::hrx::CommandPlan plan; + ggml::hrx::DispatchMatch match; + const size_t root = producer_index_for_tensor(imported.graph, tensors.projection); + REQUIRE(match_dispatch_at_index(imported.graph, plan, covered, root, match)); + REQUIRE(match.dispatches.size() == 3); + const char * expected_kernel = has_postops ? common_mul_mat_id_postops_f32_kernel_name(token_count, has_rmsnorm) : + common_mul_mat_id_f32_kernel_name(token_count); + REQUIRE(kernel_name_for_id(match.dispatches.back().kernel.kernel_id) == expected_kernel); + + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); + REQUIRE(buffer != nullptr); + const int64_t output_size = tensors.weight->ne[1]; + const int64_t expert_count = tensors.weight->ne[2]; + const int64_t route_count = tensors.route_ids->ne[0]; + std::vector weights(ggml_nelements(tensors.weight)); + for (int64_t row = 0; row < output_size * expert_count; ++row) { + for (int64_t column = 0; column < input_size; ++column) { + const int64_t packet = column % 32; + float value; + if (weight_type == GGML_TYPE_Q8_0) { + const int code = packet == 0 ? 127 : packet == 1 ? -127 : packet == 2 ? 0 : + static_cast((row * 3 + column + column / 32) % 31) - 15; + value = static_cast(code) / 256.0f; + } else if (weight_type == GGML_TYPE_Q5_1) { + const int code = packet == 0 ? 0 : packet == 1 ? 31 : packet == 2 ? 0 : + static_cast((row * 3 + column + column / 32) % 32); + value = static_cast(code) / 64.0f; + } else { + REQUIRE(weight_type == GGML_TYPE_Q1_0); + const float scale = static_cast(1 + (row + column / 128) % 3) / 16.0f; + const bool is_anchor = packet < 2; + const bool positive_code = (row + column / 32 + column / 128) % 2 == 0; + const bool positive_weight = is_anchor || positive_code; + value = positive_weight ? scale : -scale; + } + weights[row * input_size + column] = value; + } + } + std::vector packed(ggml_nbytes(tensors.weight)); + REQUIRE(ggml_quantize_chunk(weight_type, weights.data(), packed.data(), 0, + output_size * expert_count, input_size, nullptr) == packed.size()); + ggml_backend_tensor_set(tensors.weight, packed.data(), 0, packed.size()); + + // Anchors fix the CPU's activation quantization scale to an exact binary value. Q1_0 has + // no zero code, so its two anchors cancel against equal weights within each Q8_0 block. + // The remaining sparse products and sums are also exact in F16 WMMA. Two contributions are + // in the final partial block, and weight rows vary across channels and experts. + std::vector input(ggml_nelements(tensors.input), 0.0f); + std::vector ids(ggml_nelements(tensors.route_ids)); + for (int64_t token = 0; token < token_count; ++token) { + for (int64_t route = 0; route < route_count; ++route) { + const int64_t assignment = token * route_count + route; + for (int64_t column = 0; column < input_size; column += 32) { + if (weight_type == GGML_TYPE_Q1_0) { + input[assignment * input_size + column] = 127.0f / 16.0f; + input[assignment * input_size + column + 1] = -127.0f / 16.0f; + } else { + input[assignment * input_size + column + 2] = 127.0f / 16.0f; + } + } + input[assignment * input_size + 3] = static_cast(route + 1) / 16.0f; + input[assignment * input_size + input_size - 5] = static_cast(token % 3 + 1) / 16.0f; + input[assignment * input_size + input_size - 37] = -1.0f / 16.0f; + ids[assignment] = static_cast((token * 3 + route) % expert_count); + } + } + ggml_backend_tensor_set(tensors.input, input.data(), 0, input.size() * sizeof(float)); + ggml_backend_tensor_set(tensors.route_ids, ids.data(), 0, ids.size() * sizeof(int32_t)); + if (has_postops) { + std::vector bias(output_size); + std::vector residual(ggml_nelements(tensors.residual_input)); + for (size_t i = 0; i < bias.size(); ++i) { + bias[i] = static_cast(static_cast(i % 7) - 3) / 128.0f; + } + for (size_t i = 0; i < residual.size(); ++i) { + residual[i] = static_cast(static_cast(i % 13) - 6) / 256.0f; + } + ggml_backend_tensor_set(tensors.bias, bias.data(), 0, bias.size() * sizeof(float)); + ggml_backend_tensor_set(tensors.residual_input, residual.data(), 0, residual.size() * sizeof(float)); + } + if (has_rmsnorm) { + std::vector norm_weight(output_size); + for (size_t i = 0; i < norm_weight.size(); ++i) { + norm_weight[i] = 0.5f + static_cast(i % 5) / 8.0f; + } + ggml_backend_tensor_set(tensors.norm_weight, norm_weight.data(), 0, norm_weight.size() * sizeof(float)); + } + + REQUIRE(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(backend); + RoutedKTailOutputs result; + result.raw.resize(ggml_nelements(tensors.residual_output)); + result.normalized.resize(ggml_nelements(tensors.normalized_output)); + ggml_backend_tensor_get(tensors.residual_output, result.raw.data(), 0, result.raw.size() * sizeof(float)); + ggml_backend_tensor_get(tensors.normalized_output, result.normalized.data(), 0, + result.normalized.size() * sizeof(float)); + ggml_backend_buffer_free(buffer); + ggml_free(ctx); + return result; +} + +static void run_routed_k_tail_numerics(ggml_type type, int64_t input_size, int64_t token_count, int postops) { + ggml_backend_t hrx = ggml_backend_hrx_init(0); + ggml_backend_t cpu = ggml_backend_cpu_init(); + REQUIRE(hrx != nullptr); + REQUIRE(cpu != nullptr); + ggml_backend_cpu_set_n_threads(cpu, 2); + const RoutedKTailOutputs expected = run_routed_k_tail_case(cpu, type, input_size, token_count, postops); + const RoutedKTailOutputs actual = run_routed_k_tail_case(hrx, type, input_size, token_count, postops); + const RoutedKTailOutputs repeated = run_routed_k_tail_case(hrx, type, input_size, token_count, postops); + REQUIRE(std::any_of(expected.raw.begin(), expected.raw.end(), [](float value) { return value != 0.0f; })); + REQUIRE(actual.raw == expected.raw); + REQUIRE(repeated.raw == actual.raw); + REQUIRE(repeated.normalized == actual.normalized); + for (size_t i = 0; i < actual.normalized.size(); ++i) { + REQUIRE(std::isfinite(actual.normalized[i])); + const float tolerance = 3.0e-5f + 3.0e-5f * std::fabs(expected.normalized[i]); + REQUIRE(std::fabs(actual.normalized[i] - expected.normalized[i]) <= tolerance); + } + ggml_backend_free(cpu); + ggml_backend_free(hrx); +} + static void run_scale_f32_inplace() { ggml_backend_t backend = ggml_backend_hrx_init(0); REQUIRE(backend != nullptr); @@ -14641,6 +14937,7 @@ static void register_hrx_backend_host_cases(test_runner::Suite & suite) { run_qwen_weighted_reduce_next_rmsnorm_q8_publication_checks(); }); suite.host_case("common_mul_mat_id_postops_dispatch", [] { run_common_mul_mat_id_postops_dispatch_checks(); }); + suite.host_case("amax_scale_fusion_dispatch", [] { run_amax_scale_fusion_dispatch_checks(); }); suite.host_case("alias_value_import", [] { run_alias_value_import_checks(); }); suite.host_case("multi_dispatch", [] { run_multi_dispatch_checks(); }); suite.host_case("layout_alias_scheduler_elision", [] { run_layout_alias_scheduler_elision_checks(); }); @@ -14812,6 +15109,22 @@ static void register_hrx_backend_device_cases(test_runner::Suite & suite) { suite.device_case("rope_scale_f32_numerics", [] { run_rope_scale_f32_numerics(); }); suite.device_case("gather_add_rmsnorm_f32_numerics", [] { run_gather_add_rmsnorm_f32_numerics(); }); suite.device_case("scale_f32", [] { run_scale_f32(); }); + suite.device_case("amax_scale_fusion_numerics", [] { run_amax_scale_fusion_numerics(); }); + for (const auto & shape : { std::pair{ GGML_TYPE_Q8_0, 896 }, + { GGML_TYPE_Q8_0, 704 }, { GGML_TYPE_Q5_1, 704 }, + { GGML_TYPE_Q8_0, 1024 }, { GGML_TYPE_Q5_1, 768 }, + { GGML_TYPE_Q1_0, 384 }, { GGML_TYPE_Q1_0, 512 } }) { + for (int64_t tokens : { 1, 16 }) { + for (int postops : { 0, 1, 2 }) { + const std::string name = "routed_k_tail." + std::string(ggml_type_name(shape.first)) + + ".k" + std::to_string(shape.second) + ".t" + std::to_string(tokens) + + ".postops" + std::to_string(postops); + suite.device_case(name, [shape, tokens, postops] { + run_routed_k_tail_numerics(shape.first, shape.second, tokens, postops); + }); + } + } + } suite.device_case("scale_f32_inplace", [] { run_scale_f32_inplace(); }); suite.device_case("two_independent_add_f32", [] { run_two_independent_add_f32(); }); suite.device_case("chained_add_f32", [] { run_chained_add_f32(); }); diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index fb3dc56f5f9e..eceafc2165db 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -4847,80 +4847,6 @@ static void run_asymmetric_flash_attention_cpu_reference_case() { restore_environment_value(kDisableQwenDispatchEnv, had_original_env, original_env_value); } -static void run_qwen_full_cache_prefill_flash_attention_scheduling_case(int64_t token_count) { - constexpr int64_t kQueryHeadCount = 32; - constexpr int64_t kKeyValueHeadCount = 4; - constexpr int64_t kFullCacheRowCount = 40960; - const size_t active_mask_byte_count = - static_cast(token_count) * static_cast(token_count) * sizeof(ggml_fp16_t); - - ggml_init_params params = {}; - params.mem_size = 128 * 1024 * 1024; - params.no_alloc = true; - ggml_context * ctx = ggml_init(params); - REQUIRE(ctx != nullptr); - - AttentionPostprocessGraph attention = - build_attention_postprocess_graph(ctx, token_count, kQueryHeadCount, kKeyValueHeadCount, kFullCacheRowCount); - ggml_tensor * flash_output = append_qwen_full_cache_flash_attention_consumer( - ctx, attention, token_count, kQueryHeadCount, kKeyValueHeadCount, kFullCacheRowCount); - - ggml_cgraph * graph = ggml_new_graph(ctx); - REQUIRE(graph != nullptr); - ggml_build_forward_expand(graph, attention.key_output); - ggml_build_forward_expand(graph, attention.value_output); - ggml_build_forward_expand(graph, flash_output); - - ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*graph); - REQUIRE(imported.valid()); - const ggml::hrx::DispatchRegistry * registry = ggml::hrx::find_dispatch_registry({ "gfx1151" }); - REQUIRE(registry != nullptr); - - std::vector covered_nodes(imported.graph.nodes().size(), false); - ggml::hrx::CommandPlan plan; - ggml::hrx::DispatchMatch postprocess_match; - match_dispatch_at_index(imported.graph, *registry, plan, covered_nodes, - producer_index_for_tensor(imported.graph, attention.query_reshape), postprocess_match); - ggml::hrx::DispatchMatch flash_match; - match_dispatch_at_index(imported.graph, *registry, plan, covered_nodes, - producer_index_for_tensor(imported.graph, flash_output), flash_match); - - const ggml::hrx::Value * mask_value = imported.graph.values().find_tensor(attention.attention_mask); - REQUIRE(mask_value != nullptr); - const ggml::hrx::CommandPlanAlternateValue * compact_mask = - ggml::hrx::find_alternate_value(plan, mask_value->id, GGML_TYPE_F16, active_mask_byte_count); - REQUIRE(compact_mask != nullptr); - - const ggml::hrx::Dispatch * metadata_dispatch = nullptr; - for (const ggml::hrx::Dispatch & dispatch : plan.initialization_dispatches) { - if (kernel_name_for_id(dispatch.kernel.kernel_id) == "qwen3_moe:qwen_attention_metadata") { - metadata_dispatch = &dispatch; - } - } - REQUIRE(metadata_dispatch != nullptr); - REQUIRE(metadata_dispatch->kernel.integer_parameters.at("token_count") == token_count); - REQUIRE(metadata_dispatch->kernel.integer_parameters.at("context_capacity") == token_count); - REQUIRE(metadata_dispatch->bindings.size() == 5); - REQUIRE(metadata_dispatch->bindings[4].value == compact_mask->alternate_value); - REQUIRE(metadata_dispatch->bindings[4].length == active_mask_byte_count); - - const ggml::hrx::Dispatch * flash_dispatch = nullptr; - for (const ggml::hrx::Dispatch & dispatch : plan.dispatches) { - if (kernel_name_for_id(dispatch.kernel.kernel_id) == "loom_libs:ggml_flash_attention_f32_f16_wmma") { - flash_dispatch = &dispatch; - } - } - REQUIRE(flash_dispatch != nullptr); - REQUIRE(flash_dispatch->kernel.integer_parameters.at("query_token_count") == token_count); - REQUIRE(flash_dispatch->kernel.integer_parameters.at("key_value_token_count") == token_count); - REQUIRE(flash_dispatch->bindings.size() == 6); - REQUIRE(flash_dispatch->kernel.compile_parameters.at("ggml.flash_attention.apply_gate") == "0"); - REQUIRE(flash_dispatch->bindings[3].value == compact_mask->alternate_value); - REQUIRE(flash_dispatch->bindings[3].length == active_mask_byte_count); - - ggml_free(ctx); -} - static void run_qwen_decode_split_flash_attention_scheduling_case(int64_t query_token_count, int64_t key_value_token_count, bool use_common_dispatch = false, @@ -7658,6 +7584,48 @@ static void run_rope_unit_factors_cpu_reference_case() { ggml_backend_free(cpu_backend); } +static void initialize_attention_postprocess_data(ggml_backend_t cpu_backend, + const AttentionPostprocessGraph & cpu, + ggml_backend_t hrx_backend, + const AttentionPostprocessGraph & hrx) { + const int64_t token_count = cpu.positions->ne[0]; + const int64_t query_size = cpu.query_weight->ne[1]; + const int64_t key_value_size = cpu.key_weight->ne[1]; + const int64_t cache_row_count = cpu.key_cache->ne[1]; + const std::vector input = make_pattern_f32(kQwenHiddenSize * token_count, 10, 0.01f); + const std::vector query_w = make_quantized_rows(GGML_TYPE_Q4_K, kQwenHiddenSize, query_size, 11); + const std::vector key_w = make_quantized_rows(GGML_TYPE_Q4_K, kQwenHiddenSize, key_value_size, 12); + const std::vector value_w = make_quantized_rows(GGML_TYPE_Q6_K, kQwenHiddenSize, key_value_size, 13); + const std::vector query_nw = make_weight(kQwenFlashHeadSize); + const std::vector key_nw = make_pattern_f32(kQwenFlashHeadSize, 14, 0.02f); + const std::vector key_cache(static_cast(key_value_size * cache_row_count), + ggml_fp32_to_fp16(0.0f)); + const std::vector value_cache(static_cast(key_value_size * cache_row_count), + ggml_fp32_to_fp16(0.0f)); + + if (cpu.hidden_state != nullptr) { + const auto attention_weight = make_weight(kQwenHiddenSize); + set_tensor_pair_bytes(cpu_backend, cpu.hidden_state, hrx_backend, hrx.hidden_state, + input.data(), input.size() * sizeof(float)); + set_tensor_pair_bytes(cpu_backend, cpu.attention_weight, hrx_backend, hrx.attention_weight, + attention_weight.data(), attention_weight.size() * sizeof(float)); + } else { + set_tensor_pair_bytes(cpu_backend, cpu.input, hrx_backend, hrx.input, + input.data(), input.size() * sizeof(float)); + } + set_tensor_pair_bytes(cpu_backend, cpu.query_weight, hrx_backend, hrx.query_weight, query_w.data(), query_w.size()); + set_tensor_pair_bytes(cpu_backend, cpu.key_weight, hrx_backend, hrx.key_weight, key_w.data(), key_w.size()); + set_tensor_pair_bytes(cpu_backend, cpu.value_weight, hrx_backend, hrx.value_weight, value_w.data(), value_w.size()); + set_tensor_pair_bytes(cpu_backend, cpu.query_norm_weight, hrx_backend, hrx.query_norm_weight, query_nw.data(), + query_nw.size() * sizeof(float)); + set_tensor_pair_bytes(cpu_backend, cpu.key_norm_weight, hrx_backend, hrx.key_norm_weight, key_nw.data(), + key_nw.size() * sizeof(float)); + set_tensor_pair_bytes(cpu_backend, cpu.key_cache, hrx_backend, hrx.key_cache, key_cache.data(), + key_cache.size() * sizeof(ggml_fp16_t)); + set_tensor_pair_bytes(cpu_backend, cpu.value_cache, hrx_backend, hrx.value_cache, value_cache.data(), + value_cache.size() * sizeof(ggml_fp16_t)); +} + static std::pair, std::vector> run_attention_postprocess_cpu_reference_case( bool check_decode = false, bool implicit_rope = false) { ggml_backend_t cpu_backend = init_cpu_backend(); @@ -7688,8 +7656,6 @@ static std::pair, std::vector> run_attention_postproce build_attention_postprocess_graph(hrx_ctx, token_count, query_head_count, key_value_head_count, cache_row_count, explicit_factors); - - auto hrx_query_buffer = allocate_test_weight(hrx_backend, hrx.query_weight); auto hrx_key_buffer = allocate_test_weight(hrx_backend, hrx.key_weight); auto hrx_value_buffer = allocate_test_weight(hrx_backend, hrx.value_weight); @@ -7707,8 +7673,6 @@ static std::pair, std::vector> run_attention_postproce ggml_build_forward_expand(hrx_graph, hrx.key_output); ggml_build_forward_expand(hrx_graph, hrx.value_output); - - require_kernel_subsequence(scheduled_kernel_sequence(hrx_graph), { check_decode ? "qwen3_moe:qwen3_moe_attention_qkv_postprocess_fused_decode" : "qwen3_moe:qwen3_moe_attention_postprocess_f32_f16" }); @@ -7718,45 +7682,15 @@ static std::pair, std::vector> run_attention_postproce REQUIRE(cpu_buffer != nullptr); REQUIRE(hrx_buffer != nullptr); - const std::vector input = make_pattern_f32(kQwenHiddenSize * token_count, 10, 0.01f); - const std::vector query_w = make_quantized_rows(GGML_TYPE_Q4_K, kQwenHiddenSize, query_size, 11); - const std::vector key_w = make_quantized_rows(GGML_TYPE_Q4_K, kQwenHiddenSize, key_value_size, 12); - const std::vector value_w = make_quantized_rows(GGML_TYPE_Q6_K, kQwenHiddenSize, key_value_size, 13); - const std::vector query_nw = make_weight(kQwenFlashHeadSize); - const std::vector key_nw = make_pattern_f32(kQwenFlashHeadSize, 14, 0.02f); std::vector positions = check_decode ? std::vector{ 8 } : std::vector{ 8, 9 }; std::vector factors(static_cast(kQwenFlashHeadSize / 2), 1.0f); - const std::vector key_cache(static_cast(key_value_size * cache_row_count), - ggml_fp32_to_fp16(0.0f)); - const std::vector value_cache(static_cast(key_value_size * cache_row_count), - ggml_fp32_to_fp16(0.0f)); std::vector cache_indices = make_i64_mod_data(token_count, cache_row_count); - if (check_decode) { - const auto attention_weight = make_weight(kQwenHiddenSize); - set_tensor_pair_bytes(cpu_backend, cpu.hidden_state, hrx_backend, hrx.hidden_state, - input.data(), input.size() * sizeof(float)); - set_tensor_pair_bytes(cpu_backend, cpu.attention_weight, hrx_backend, hrx.attention_weight, - attention_weight.data(), attention_weight.size() * sizeof(float)); - } else { - set_tensor_pair_bytes(cpu_backend, cpu.input, hrx_backend, hrx.input, - input.data(), input.size() * sizeof(float)); - } - set_tensor_pair_bytes(cpu_backend, cpu.query_weight, hrx_backend, hrx.query_weight, query_w.data(), query_w.size()); - set_tensor_pair_bytes(cpu_backend, cpu.key_weight, hrx_backend, hrx.key_weight, key_w.data(), key_w.size()); - set_tensor_pair_bytes(cpu_backend, cpu.value_weight, hrx_backend, hrx.value_weight, value_w.data(), value_w.size()); - set_tensor_pair_bytes(cpu_backend, cpu.query_norm_weight, hrx_backend, hrx.query_norm_weight, query_nw.data(), - query_nw.size() * sizeof(float)); - set_tensor_pair_bytes(cpu_backend, cpu.key_norm_weight, hrx_backend, hrx.key_norm_weight, key_nw.data(), - key_nw.size() * sizeof(float)); + initialize_attention_postprocess_data(cpu_backend, cpu, hrx_backend, hrx); set_tensor_pair_bytes(cpu_backend, cpu.positions, hrx_backend, hrx.positions, positions.data(), positions.size() * sizeof(int32_t)); set_tensor_pair_bytes(cpu_backend, cpu.freq_factors, hrx_backend, hrx.freq_factors, factors.data(), factors.size() * sizeof(float)); - set_tensor_pair_bytes(cpu_backend, cpu.key_cache, hrx_backend, hrx.key_cache, key_cache.data(), - key_cache.size() * sizeof(ggml_fp16_t)); - set_tensor_pair_bytes(cpu_backend, cpu.value_cache, hrx_backend, hrx.value_cache, value_cache.data(), - value_cache.size() * sizeof(ggml_fp16_t)); set_tensor_pair_bytes(cpu_backend, cpu.key_cache_indices, hrx_backend, hrx.key_cache_indices, cache_indices.data(), cache_indices.size() * sizeof(int64_t)); set_tensor_pair_bytes(cpu_backend, cpu.value_cache_indices, hrx_backend, hrx.value_cache_indices, @@ -7767,7 +7701,9 @@ static std::pair, std::vector> run_attention_postproce std::vector unit_query; std::vector unit_key_cache; for (int round = 0; round < rounds; ++round) { - if (round == 1 && explicit_factors) { + const bool factor_change_round = round == 1; + const bool update_factors = factor_change_round && explicit_factors; + if (update_factors) { for (size_t channel = 0; channel < factors.size(); ++channel) { factors[channel] = channel % 3 == 0 ? 0.5f : channel % 3 == 1 ? 2.0f : 4.0f; } @@ -7884,6 +7820,153 @@ static std::pair, std::vector> run_attention_postproce return { unit_query, unit_key_cache }; } +static void run_attention_postprocess_metadata_reuse_case() { + ggml_backend_t cpu_backend = init_cpu_backend(); + ggml_backend_t hrx_backend = ggml_backend_hrx_init(0); + REQUIRE(hrx_backend != nullptr); + + ggml_init_params params = {}; + params.mem_size = 32 * 1024 * 1024; + params.no_alloc = true; + ggml_context * cpu_ctx = ggml_init(params); + ggml_context * hrx_ctx = ggml_init(params); + REQUIRE(cpu_ctx != nullptr); + REQUIRE(hrx_ctx != nullptr); + + constexpr int64_t token_count = 2; + constexpr int64_t query_head_count = 4; + constexpr int64_t key_value_head_count = 2; + constexpr int64_t cache_row_count = 8; + constexpr int64_t query_size = query_head_count * kQwenFlashHeadSize; + constexpr int64_t key_value_size = key_value_head_count * kQwenFlashHeadSize; + AttentionPostprocessGraph cpu = build_attention_postprocess_graph( + cpu_ctx, token_count, query_head_count, key_value_head_count, cache_row_count); + AttentionPostprocessGraph hrx = build_attention_postprocess_graph( + hrx_ctx, token_count, query_head_count, key_value_head_count, cache_row_count); + ggml_tensor * cpu_flash = append_qwen_full_cache_flash_attention_consumer( + cpu_ctx, cpu, token_count, query_head_count, key_value_head_count, cache_row_count); + ggml_tensor * hrx_flash = append_qwen_full_cache_flash_attention_consumer( + hrx_ctx, hrx, token_count, query_head_count, key_value_head_count, cache_row_count); + ggml_set_output(cpu.query_output); + ggml_set_output(hrx.query_output); + + auto hrx_query_buffer = allocate_test_weight(hrx_backend, hrx.query_weight); + auto hrx_key_buffer = allocate_test_weight(hrx_backend, hrx.key_weight); + auto hrx_value_buffer = allocate_test_weight(hrx_backend, hrx.value_weight); + + ggml_cgraph * cpu_graph = ggml_new_graph(cpu_ctx); + ggml_cgraph * hrx_graph = ggml_new_graph(hrx_ctx); + REQUIRE(cpu_graph != nullptr); + REQUIRE(hrx_graph != nullptr); + // A stable graph UID exercises the cached execution contract across changing inputs. + hrx_graph->uid = 4001; + ggml_build_forward_expand(cpu_graph, cpu.query_output); + ggml_build_forward_expand(cpu_graph, cpu.key_output); + ggml_build_forward_expand(cpu_graph, cpu.value_output); + ggml_build_forward_expand(hrx_graph, hrx.query_output); + ggml_build_forward_expand(hrx_graph, hrx.key_output); + ggml_build_forward_expand(hrx_graph, hrx.value_output); + ggml_build_forward_expand(cpu_graph, cpu_flash); + ggml_build_forward_expand(hrx_graph, hrx_flash); + REQUIRE(ggml_backend_supports_op(hrx_backend, hrx_flash)); + require_kernel_subsequence(scheduled_kernel_sequence(hrx_graph), + { "qwen3_moe:qwen3_moe_attention_postprocess_f32_f16" }); + + ggml_backend_buffer_t cpu_buffer = ggml_backend_alloc_ctx_tensors(cpu_ctx, cpu_backend); + ggml_backend_buffer_t hrx_buffer = ggml_backend_alloc_ctx_tensors(hrx_ctx, hrx_backend); + REQUIRE(cpu_buffer != nullptr); + REQUIRE(hrx_buffer != nullptr); + + initialize_attention_postprocess_data(cpu_backend, cpu, hrx_backend, hrx); + // Keep factors fixed so each execution changes only caller attention metadata. + const std::vector factors(static_cast(kQwenFlashHeadSize / 2), 1.0f); + set_tensor_pair_bytes(cpu_backend, cpu.freq_factors, hrx_backend, hrx.freq_factors, + factors.data(), factors.size() * sizeof(float)); + std::vector positions(token_count); + std::vector cache_indices(token_count); + ggml_backend_hrx_cache_stats first_stats = {}; + for (int round = 0; round < 3; ++round) { + std::vector supplied_mask(token_count * cache_row_count, ggml_fp32_to_fp16(-INFINITY)); + for (int64_t token = 0; token < token_count; ++token) { + positions[token] = static_cast(6 + round * token_count + token); + cache_indices[token] = positions[token] % cache_row_count; + // A window of one retains exactly the current physical row, including after wrap. + supplied_mask[token * cache_row_count + cache_indices[token]] = ggml_fp32_to_fp16(0.0f); + } + set_tensor_pair_bytes(cpu_backend, cpu.attention_mask, hrx_backend, hrx.attention_mask, + supplied_mask.data(), supplied_mask.size() * sizeof(ggml_fp16_t)); + set_tensor_pair_bytes(cpu_backend, cpu.positions, hrx_backend, hrx.positions, + positions.data(), positions.size() * sizeof(int32_t)); + set_tensor_pair_bytes(cpu_backend, cpu.key_cache_indices, hrx_backend, hrx.key_cache_indices, + cache_indices.data(), cache_indices.size() * sizeof(int64_t)); + set_tensor_pair_bytes(cpu_backend, cpu.value_cache_indices, hrx_backend, hrx.value_cache_indices, + cache_indices.data(), cache_indices.size() * sizeof(int64_t)); + REQUIRE(ggml_backend_graph_compute(cpu_backend, cpu_graph) == GGML_STATUS_SUCCESS); + REQUIRE(ggml_backend_graph_compute(hrx_backend, hrx_graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(cpu_backend); + ggml_backend_synchronize(hrx_backend); + require_close(get_f32_tensor(hrx_backend, hrx.query_output), get_f32_tensor(cpu_backend, cpu.query_output), 2.0f, + 5.0e-2f); + require_close(get_f32_tensor(hrx_backend, hrx.key_output), get_f32_tensor(cpu_backend, cpu.key_output), 2.0f, + 5.0e-2f); + require_close(get_f32_tensor(hrx_backend, hrx.value_output), get_f32_tensor(cpu_backend, cpu.value_output), 2.0f, + 5.0e-2f); + + std::vector actual_factors(factors.size()); + ggml_backend_tensor_get(hrx.freq_factors, actual_factors.data(), 0, + actual_factors.size() * sizeof(float)); + REQUIRE(actual_factors == factors); + + std::vector actual_positions(positions.size()); + std::vector actual_key_indices(cache_indices.size()); + std::vector actual_value_indices(cache_indices.size()); + ggml_backend_tensor_get(hrx.positions, actual_positions.data(), 0, + actual_positions.size() * sizeof(int32_t)); + ggml_backend_tensor_get(hrx.key_cache_indices, actual_key_indices.data(), 0, + actual_key_indices.size() * sizeof(int64_t)); + ggml_backend_tensor_get(hrx.value_cache_indices, actual_value_indices.data(), 0, + actual_value_indices.size() * sizeof(int64_t)); + REQUIRE(actual_positions == positions); + REQUIRE(actual_key_indices == cache_indices); + REQUIRE(actual_value_indices == cache_indices); + + std::vector actual_mask(supplied_mask.size()); + ggml_backend_tensor_get(hrx.attention_mask, actual_mask.data(), 0, + actual_mask.size() * sizeof(ggml_fp16_t)); + REQUIRE(actual_mask == supplied_mask); + + const auto cached_values = get_f32_tensor(hrx_backend, hrx.value_cache); + std::vector expected(token_count * query_size); + for (int64_t token = 0; token < token_count; ++token) { + for (int64_t head = 0; head < query_head_count; ++head) { + const int64_t value_head = head / (query_head_count / key_value_head_count); + for (int64_t channel = 0; channel < kQwenFlashHeadSize; ++channel) { + expected[token * query_size + head * kQwenFlashHeadSize + channel] = + cached_values[cache_indices[token] * key_value_size + + value_head * kQwenFlashHeadSize + channel]; + } + } + } + require_close(get_f32_tensor(hrx_backend, hrx_flash), expected, 2.0e-3f, 1.0e-3f); + ggml_backend_hrx_cache_stats stats = {}; + REQUIRE(ggml_backend_hrx_get_cache_stats(hrx_backend, &stats)); + if (round == 0) { + first_stats = stats; + } else { + REQUIRE(stats.prepared_program_builds == first_stats.prepared_program_builds); + REQUIRE(stats.graph_program_builds == first_stats.graph_program_builds); + REQUIRE(stats.graph_program_hits > first_stats.graph_program_hits); + } + } + + ggml_backend_buffer_free(cpu_buffer); + ggml_backend_buffer_free(hrx_buffer); + ggml_free(cpu_ctx); + ggml_free(hrx_ctx); + ggml_backend_free(cpu_backend); + ggml_backend_free(hrx_backend); +} + static void run_routed_moe_cpu_reference_case(ggml_type down_weight_type, bool include_next_rmsnorm) { ggml_backend_t cpu_backend = init_cpu_backend(); ggml_backend_t hrx_backend = ggml_backend_hrx_init(0); @@ -8179,15 +8262,22 @@ static void run_decode_attention_qkv_scheduling_case() { REQUIRE(ctx != nullptr); AttentionPostprocessGraph graph = build_decode_attention_qkv_graph(ctx); + ggml_tensor * flash = append_qwen_full_cache_flash_attention_consumer(ctx, graph, 1, 32, 4, 1024); ggml_cgraph * cgraph = ggml_new_graph(ctx); REQUIRE(cgraph != nullptr); ggml_build_forward_expand(cgraph, graph.query_output); ggml_build_forward_expand(cgraph, graph.key_output); ggml_build_forward_expand(cgraph, graph.value_output); + ggml_build_forward_expand(cgraph, flash); require_kernel_subsequence(scheduled_kernel_sequence(cgraph), { "qwen3_moe:qwen3_moe_rmsnorm_f32_quantize_q8_1_x4", "qwen3_moe:qwen3_moe_attention_qkv_postprocess_fused_decode" }); + auto imported = ggml::hrx::import_ggml_graph(*cgraph); + REQUIRE(imported.valid()); + ggml::hrx::DispatchScheduler scheduler; + REQUIRE(scheduler.schedule_graph(imported.graph, { "gfx1151" })); + REQUIRE(scheduler.plan().initialization_dispatches.empty()); ggml_free(ctx); } @@ -8913,6 +9003,8 @@ static void register_rmsnorm_and_scheduling_cases(Suite & suite) { require_close(explicit_unit.second, implicit.second, 1.0e-3f, 1.0e-3f); }); suite.device_case("attention_postprocess.decode_factors", [] { run_attention_postprocess_cpu_reference_case(true); }); + suite.device_case("attention_postprocess.metadata_reuse", + [] { run_attention_postprocess_metadata_reuse_case(); }); suite.device_case("routed_moe.q4_k.next_rmsnorm", [] { run_routed_moe_cpu_reference_case(GGML_TYPE_Q4_K, true); }); suite.host_case("routed_moe.codebook_admission", [] { run_routed_codebook_admission_checks(); }); suite.device_case("routed_pair.iq3_xxs.iq3_xxs.skinny.cpu_reference", [] { @@ -9032,8 +9124,6 @@ static void register_flash_attention_cases(Suite & suite) { [] { run_qwen_decode_attention_output_next_q8_scheduling_case(false); }); suite.device_case("qwen_decode_attention_output_next_q8.selectors", [] { run_qwen_decode_attention_output_next_q8_scheduling_case(true); }); - suite.device_case("qwen_full_cache_prefill_flash_attention.tokens512", - [] { run_qwen_full_cache_prefill_flash_attention_scheduling_case(512); }); } static void register_hrx_ops_cases(Suite & suite) {