From 5ebe0c678e912fff6ae305b25a70d9f21914c7c3 Mon Sep 17 00:00:00 2001 From: Aaron St George Date: Thu, 5 Nov 2026 06:20:49 -0700 Subject: [PATCH 1/5] ggml-hrx: support Gemma routed IDs and expert scales (#180) Bind exact router ID view spans and gather scalar expert scales with runtime ID bounds checks. Validated on gfx1151 with focused operator/backend tests, full paired HRX/Vulkan perplexity, and generation. High absolute perplexity and CPU REPEAT fallback remain documented limitations. --- .../common/dispatch-get-rows.cpp | 66 +++++++++ .../qwen/dispatch-moe-router.cpp | 6 +- .../kernels/loom-libs/manifest.json | 48 +++++++ .../motifs/routing/router_topk_row.loom | 4 +- .../ops/rows/get_rows_scalar_3d_f32.loom | 54 ++++++++ tests/hrx-backend-test.cpp | 67 ++++++++-- tests/hrx-ops-test.cpp | 125 ++++++++++++++++-- 7 files changed, 344 insertions(+), 26 deletions(-) create mode 100644 ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rows/get_rows_scalar_3d_f32.loom diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp index 022d1f05279e..fd7fbfb481cd 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp @@ -19,6 +19,8 @@ namespace ggml::hrx { namespace { static constexpr KernelCatalogRef kGetRowsF32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_get_rows_f32"); +static constexpr KernelCatalogRef kGetRowsScalar3dF32Kernel = + GGML_HRX_KERNEL_REF("loom_libs", "ggml_get_rows_scalar_3d_f32"); static constexpr KernelCatalogRef kGetRowsF32NextKernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_get_rows_f32_next"); static constexpr KernelCatalogRef kGetRowsScaleF32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_get_rows_scale_f32"); @@ -153,6 +155,62 @@ static void add_primary_bindings(Dispatch & dispatch, const GetRowsMatch & match dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count }); } +static bool match_get_rows_scalar_3d_f32_dispatch(const DispatchMatchContext & context, DispatchMatch & match) { + const GraphNode * node = context.root_node; + const bool is_get_rows = node != nullptr && node->op == GGML_OP_GET_ROWS; + const bool has_two_inputs = node != nullptr && node->inputs.size() == 2; + if (!is_get_rows || !has_two_inputs) { + return false; + } + + const Value * scales = graph_value(context.graph, node->inputs[0]); + const Value * ids = graph_value(context.graph, node->inputs[1]); + const Value * output = graph_value(context.graph, node->output); + const bool has_values = scales != nullptr && ids != nullptr && output != nullptr; + if (!has_values) { + return false; + } + + const int64_t token_count = output->ne[2]; + const bool has_supported_tokens = is_supported_token_count(token_count); + const bool has_scale_shape = scales->ne[0] == 1 && scales->ne[1] == 128 && + scales->ne[2] == token_count && scales->ne[3] == 1; + const bool has_id_shape = ids->ne[0] == 8 && ids->ne[1] == token_count && + ids->ne[2] == 1 && ids->ne[3] == 1; + const bool has_output_shape = output->ne[0] == 1 && output->ne[1] == 8 && + output->ne[3] == 1; + const bool has_types = scales->type == GGML_TYPE_F32 && ids->type == GGML_TYPE_I32 && + output->type == GGML_TYPE_F32; + const bool has_contiguous_values = scales->contiguous && output->contiguous; + const bool has_id_columns = ids->nb[0] == sizeof(int32_t) && ids->nb[1] % sizeof(int32_t) == 0; + const bool has_supported_shape = has_supported_tokens && has_scale_shape && has_id_shape && + has_output_shape && has_types && has_contiguous_values && has_id_columns; + if (!has_supported_shape) { + return false; + } + + const size_t route_stride = ids->nb[1] / sizeof(int32_t); + const bool has_supported_stride = route_stride >= 8 && route_stride <= 512; + const size_t expected_id_bytes = (static_cast(token_count) - 1) * ids->nb[1] + 8 * sizeof(int32_t); + const bool has_exact_id_span = ids->byte_count == expected_id_bytes; + if (!has_supported_stride || !has_exact_id_span) { + return false; + } + + Dispatch dispatch; + dispatch.kernel = make_kernel_specialization(kGetRowsScalar3dF32Kernel); + dispatch.kernel.workload_specialization = WorkloadSpecialization::Dynamic; + dispatch.kernel.integer_parameters.emplace("token_count", token_count); + dispatch.kernel.integer_parameters.emplace("route_stride", static_cast(route_stride)); + dispatch.bindings.push_back({ ids->id, 0, ids->byte_count }); + dispatch.bindings.push_back({ scales->id, 0, scales->byte_count }); + dispatch.bindings.push_back({ output->id, 0, output->byte_count }); + + match.covered_nodes.push_back(context.root_index); + match.dispatches.push_back(std::move(dispatch)); + return true; +} + static GetRowsMatch match_get_rows_f32(const Graph & graph, const GraphNode * node) { GetRowsMatch match; if (node == nullptr || node->op != GGML_OP_GET_ROWS || node->inputs.size() != 2) { @@ -472,6 +530,14 @@ static bool match_get_rows_rmsnorm_binary_dispatch(const DispatchMatchContext & } // namespace void register_get_rows_dispatches(DispatchRegistryBuilder & registry) { + registry.add({ + "common.get_rows.scalar_3d_f32", + GGML_OP_GET_ROWS, + DispatchMatchKind::SingleOp, + 125, + DispatchSource::Common, + match_get_rows_scalar_3d_f32_dispatch, + }); registry.add({ "common.get_rows_scale.f32", GGML_OP_GET_ROWS, diff --git a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-moe-router.cpp b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-moe-router.cpp index 44de4f943b78..440f706220f6 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-moe-router.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/qwen/dispatch-moe-router.cpp @@ -536,8 +536,7 @@ static bool match_moe_router_projection_top8_fused_decode_dispatch(const Dispatc dispatch.kernel.compile_parameters.emplace("qwen3_moe.workload.token_capacity", to_config_value(match.top8.token_count)); - const size_t route_id_length = - static_cast(match.top8.token_count * match.top8.route_stride) * sizeof(int32_t); + const size_t route_id_length = match.top8.route_ids->byte_count; dispatch.bindings.push_back({ match.input->id, 0, match.input->byte_count }); dispatch.bindings.push_back({ match.weight->id, 0, match.weight->byte_count }); dispatch.bindings.push_back({ match.logits->id, 0, match.logits->byte_count }); @@ -576,8 +575,7 @@ static bool match_moe_router_top8_dispatch(const DispatchMatchContext & context, dispatch.kernel.compile_parameters.emplace("qwen3_moe.workload.token_capacity", to_config_value(router_match.token_count)); - const size_t route_id_length = - static_cast(router_match.token_count * router_match.route_stride) * sizeof(int32_t); + const size_t route_id_length = router_match.route_ids->byte_count; dispatch.bindings.push_back({ router_match.logits->id, 0, router_match.logits->byte_count }); dispatch.bindings.push_back({ router_match.route_ids->id, 0, route_id_length }); dispatch.bindings.push_back({ router_match.route_weights->id, 0, router_match.route_weights->byte_count }); diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json index 4e1acc579461..5c3a44febca5 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json @@ -314,6 +314,9 @@ { "path": "ops/rows/get_rows_f32.loom" }, + { + "path": "ops/rows/get_rows_scalar_3d_f32.loom" + }, { "path": "ops/rows/get_rows_scale_f32.loom" }, @@ -6660,6 +6663,51 @@ "motifs/quantize_q8_1_x4.loom" ] }, + { + "name": "ggml_get_rows_scalar_3d_f32", + "family": "loom_libs", + "symbol": "ggml_get_rows_scalar_3d_f32", + "source": "ops/rows/get_rows_scalar_3d_f32.loom", + "workload_parameters": [ + { + "name": "token_count", + "type": "index" + }, + { + "name": "route_stride", + "type": "index" + } + ], + "launch_parameters": [ + { + "name": "token_count", + "type": "index" + }, + { + "name": "route_stride", + "type": "index" + } + ], + "bindings": [ + "route_ids", + "scales", + "output" + ], + "binding_access": [ + "read", + "read", + "read_write" + ], + "target_selector": "", + "compile_recipe": { + "mode": "direct", + "primary_sources": [ + "ops/rows/get_rows_scalar_3d_f32.loom" + ], + "library_sources": [] + }, + "compile_dependencies": [] + }, { "name": "ggml_get_rows_f32", "family": "loom_libs", diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/routing/router_topk_row.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/routing/router_topk_row.loom index b028425c98e4..0b6551572ed1 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/routing/router_topk_row.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/routing/router_topk_row.loom @@ -28,7 +28,9 @@ template.def<@qwen3_moe.router.top8.row> device requires [#target.subgroup.size< %experts_per_lane0 = index.div %expert_count, %wave_size : index %experts_per_lane = index.assume %experts_per_lane0 [range(%experts_per_lane0, 1, 16)] : index %lane_expert_base = index.mul %lane, %experts_per_lane : index - %route_id_storage_count = index.mul %launch_token_count, %bounded_route_id_stride : index + %last_token = index.sub %launch_token_count, %c1 : index + %last_token_base = index.mul %last_token, %bounded_route_id_stride : index + %route_id_storage_count = index.add %last_token_base, %route_count : index %route_weight_storage_count = index.mul %launch_token_count, %route_count : index %logits_noalias, %route_ids_noalias, %route_weights_noalias = buffer.assume.noalias %logits, %route_ids, %route_weights : buffer, buffer, buffer %logits_view = buffer.view %logits_noalias[%c0_offset] : buffer -> view<[%launch_token_count]x[%expert_count]xf32> diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rows/get_rows_scalar_3d_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rows/get_rows_scalar_3d_f32.loom new file mode 100644 index 000000000000..0c0133b55079 --- /dev/null +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rows/get_rows_scalar_3d_f32.loom @@ -0,0 +1,54 @@ +// Copyright 2026 The HRX Authors +// SPDX-License-Identifier: Apache-2.0 + +// Gathers one F32 expert scale for each route and token. The IDs may be a +// narrow view of wider argsort rows; only the selected IDs are in bounds. +amdgpu.target @ggml_get_rows_scalar_3d_gfx11_wave64 {subgroup_size = 64} + +kernel.def target(@ggml_get_rows_scalar_3d_gfx11_wave64) @ggml_get_rows_scalar_3d_f32(%token_count: index, %route_stride: index) { + %c1 = index.constant 1 : index + %c64 = index.constant 64 : index + kernel.launch.config workgroups(%token_count, %c1, %c1) workgroup_size(%c64, %c1, %c1) : index +} launch(%token_count: index, %route_stride: index, %route_ids: buffer, %scales: buffer, %output: buffer) where [range(%token_count, 1, 2048), range(%route_stride, 8, 512)] { + %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048)] : index + %bounded_route_stride = index.assume %route_stride [range(%route_stride, 8, 512)] : index + %token0 = kernel.workgroup.id : index + %route0 = kernel.workitem.id : index + %c1 = index.constant 1 : index + %c8 = index.constant 8 : index + %c0_i32 = scalar.constant 0 : i32 + %c128_i32 = scalar.constant 128 : i32 + %c0_f32 = scalar.constant 0.0 : f32 + %c0_offset = index.constant 0 : offset + %valid_token = index.cmp ult, %token0, %bounded_token_count : index + %valid_route = index.cmp ult, %route0, %c8 : index + %publish = scalar.andi %valid_token, %valid_route : i1 + %last_token = index.sub %bounded_token_count, %c1 : index + %last_row_base = index.mul %last_token, %bounded_route_stride : index + %route_id_span = index.add %last_row_base, %c8 : index + %ids_view = buffer.view %route_ids[%c0_offset] : buffer -> view<[%route_id_span]xi32> + %scales_view = buffer.view %scales[%c0_offset] : buffer -> view<[%bounded_token_count]x128xf32> + %output_view = buffer.view %output[%c0_offset] : buffer -> view<[%bounded_token_count]x8xf32> + scf.if %publish { + %token = index.assume %token0 [lt(%token0, %bounded_token_count)] : index + %route = index.assume %route0 [range(%route0, 0, 7)] : index + %row_base = index.mul %token, %bounded_route_stride : index + %id_index = index.add %row_base, %route : index + %bounded_id_index = index.assume %id_index [lt(%id_index, %route_id_span)] : index + %expert_i32 = view.load %ids_view[%bounded_id_index] : view<[%route_id_span]xi32> -> i32 + %expert_nonnegative = scalar.cmpi sge, %expert_i32, %c0_i32 : i32 + %expert_below_limit = scalar.cmpi slt, %expert_i32, %c128_i32 : i32 + %valid_expert = scalar.andi %expert_nonnegative, %expert_below_limit : i1 + // Invalid IDs have no CPU GET_ROWS value; publish zero without reading a scale row. + %scale = scf.if %valid_expert -> (f32) { + %expert0 = index.cast %expert_i32 : i32 to index + %expert = index.assume %expert0 [range(%expert0, 0, 127)] : index + %selected_scale = view.load %scales_view[%token, %expert] : view<[%bounded_token_count]x128xf32> -> f32 + scf.yield %selected_scale : f32 + } else { + scf.yield %c0_f32 : f32 + } + view.store %scale, %output_view[%token, %route] : f32, view<[%bounded_token_count]x8xf32> + } + kernel.return +} diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index a90b8e9ea567..530b449d7406 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -4457,6 +4457,54 @@ static void schedule_token_embedding_command(ggml_context * ctx, REQUIRE(commands.commands.front().bindings[2].name == "output"); } +static void run_scalar_3d_get_rows_dispatch_checks() { + auto check_case = [](int64_t expert_count, int64_t route_count, int64_t token_count, + int64_t route_stride, bool expected_match) { + ggml_init_params params = {}; + params.mem_size = 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + + ggml_tensor * scales = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1, expert_count, token_count); + ggml_tensor * id_storage = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, route_stride, token_count); + REQUIRE(scales != nullptr); + REQUIRE(id_storage != nullptr); + ggml_tensor * ids = route_stride == route_count ? id_storage : + ggml_view_2d(ctx, id_storage, route_count, token_count, id_storage->nb[1], 0); + REQUIRE(ids != nullptr); + ggml_tensor * output = ggml_get_rows(ctx, scales, ids); + REQUIRE(output != nullptr); + + ggml_cgraph * graph = ggml_new_graph(ctx); + REQUIRE(graph != nullptr); + ggml_build_forward_expand(graph, output); + ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*graph); + REQUIRE(imported.valid()); + + ggml::hrx::DispatchScheduler scheduler; + const bool scheduled = scheduler.schedule_graph(imported.graph, test_dispatch_target()); + REQUIRE(scheduled == expected_match); + if (expected_match) { + REQUIRE(scheduler.plan().dispatches.size() == 1); + const ggml::hrx::Dispatch & dispatch = scheduler.plan().dispatches.front(); + REQUIRE(kernel_name_for_id(dispatch.kernel.kernel_id) == "loom_libs:ggml_get_rows_scalar_3d_f32"); + REQUIRE(dispatch.bindings.size() == 3); + REQUIRE(dispatch.bindings[0].length == ggml_nbytes(ids)); + REQUIRE(dispatch.kernel.integer_parameters.at("token_count") == token_count); + REQUIRE(dispatch.kernel.integer_parameters.at("route_stride") == route_stride); + } + ggml_free(ctx); + }; + + check_case(128, 8, 1, 8, true); + check_case(128, 8, 2, 128, true); + check_case(128, 8, 8, 128, true); + check_case(64, 8, 2, 128, false); + check_case(128, 4, 2, 128, false); + check_case(128, 8, 2, 513, false); +} + static void run_qwen_token_embedding_dispatch_checks() { ggml_init_params params = {}; params.mem_size = 4 * 1024 * 1024; @@ -9312,7 +9360,9 @@ static void schedule_qwen_router_top8_command(ggml_context * ctx, REQUIRE(scheduler.plan().valid()); const int64_t token_count = output->ne[2]; - const size_t route_id_length = static_cast(token_count) * route_ids->nb[1]; + const size_t route_id_length = static_cast(token_count - 1) * route_ids->nb[1] + + static_cast(route_ids->ne[0]) * sizeof(int32_t); + REQUIRE(route_ids_value->byte_count == route_id_length); const size_t expert_table_bytes = qwen_expert_table_size(token_count, expected_expert_count); const size_t partition_table_bytes = qwen_partition_table_size(token_count, expected_route_count, expected_expert_count); @@ -9463,7 +9513,8 @@ static void schedule_qwen_router_top8_command(ggml_context * ctx, const ggml::hrx::TransientAllocation * route_ids_allocation = ggml::hrx::find_transient_allocation(commands.transients, route_ids_value->storage_root); REQUIRE(route_ids_allocation != nullptr); - REQUIRE(route_ids_allocation->size == dispatch.bindings[1].length); + REQUIRE(route_ids_allocation->size == static_cast(token_count * expected_expert_count) * sizeof(int32_t)); + REQUIRE(dispatch.bindings[1].length <= route_ids_allocation->size); const ggml::hrx::TransientAllocation * expert_table_allocation = ggml::hrx::find_transient_allocation(commands.transients, expert_table_transient.value); REQUIRE(expert_table_allocation != nullptr); @@ -11015,15 +11066,8 @@ static void run_qwen_router_top8_dispatch_checks() { ggml_context * ctx = ggml_init(params); REQUIRE(ctx != nullptr); - { - ggml_tensor * logits = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, kQwenRouterExpertCount, 4); - ggml_tensor * route_ids = nullptr; - REQUIRE(logits != nullptr); - ggml_tensor * output = build_qwen_router_top8_graph(ctx, logits, &route_ids); - schedule_qwen_router_top8_command(ctx, output, route_ids); - } - { - ggml_tensor * logits = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, kQwenRouterExpertCount, 1); + for (int64_t token_count : { 1, 2, 4, 8 }) { + ggml_tensor * logits = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, kQwenRouterExpertCount, token_count); ggml_tensor * route_ids = nullptr; REQUIRE(logits != nullptr); ggml_tensor * output = build_qwen_router_top8_graph(ctx, logits, &route_ids); @@ -14880,6 +14924,7 @@ static void register_hrx_backend_host_cases(test_runner::Suite & suite) { suite.host_case("rope_set_rows_dispatch", [] { run_rope_set_rows_dispatch_checks(); }); suite.host_case("graph_traversal", [] { run_graph_traversal_checks(); }); suite.host_case("qwen_token_embedding_dispatch", [] { run_qwen_token_embedding_dispatch_checks(); }); + suite.host_case("scalar_3d_get_rows_dispatch", [] { run_scalar_3d_get_rows_dispatch_checks(); }); suite.host_case("get_rows_rmsnorm_binary_dispatch", [] { run_get_rows_rmsnorm_binary_dispatch_checks(); }); suite.host_case("get_rows_scale_dispatch", [] { run_get_rows_scale_dispatch_checks(); }); suite.host_case("gather_add_dispatch", [] { run_gather_add_dispatch_checks(); }); diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index eceafc2165db..491321e2f555 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -231,8 +231,15 @@ static std::vector router_projection_reference(const std::vector & return output; } -static std::vector router_top8_weights_reference(const std::vector & logits, int64_t token_count) { - std::vector output(kQwenRouterRouteCount * token_count); +struct RouterTop8Reference { + std::vector ids; + std::vector weights; +}; + +static RouterTop8Reference router_top8_reference(const std::vector & logits, int64_t token_count) { + RouterTop8Reference reference; + reference.ids.resize(kQwenRouterRouteCount * token_count); + reference.weights.resize(kQwenRouterRouteCount * token_count); for (int64_t token = 0; token < token_count; ++token) { bool used[kQwenRouterExpertCount] = {}; int64_t selected[kQwenRouterRouteCount] = {}; @@ -249,6 +256,7 @@ static std::vector router_top8_weights_reference(const std::vector } selected[route] = best_expert; used[best_expert] = true; + reference.ids[token * kQwenRouterRouteCount + route] = static_cast(best_expert); } float max_selected = -std::numeric_limits::infinity(); @@ -258,14 +266,14 @@ static std::vector router_top8_weights_reference(const std::vector float sum = 0.0f; for (int64_t route = 0; route < kQwenRouterRouteCount; ++route) { const float value = std::exp(logits[token * kQwenRouterExpertCount + selected[route]] - max_selected); - output[token * kQwenRouterRouteCount + route] = value; + reference.weights[token * kQwenRouterRouteCount + route] = value; sum += value; } for (int64_t route = 0; route < kQwenRouterRouteCount; ++route) { - output[token * kQwenRouterRouteCount + route] /= sum; + reference.weights[token * kQwenRouterRouteCount + route] /= sum; } } - return output; + return reference; } static std::vector flash_attention_reference(const std::vector & query, @@ -4498,6 +4506,72 @@ static void run_q4_prefill_source_eligibility_case() { ggml_backend_free(backend); } +static void run_scalar_3d_get_rows_case(int64_t token_count, int64_t route_stride, bool include_invalid_ids = false) { + ggml_backend_t backend = ggml_backend_hrx_init(0); + REQUIRE(backend != nullptr); + + ggml_init_params params = {}; + params.mem_size = 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + + ggml_tensor * scales = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1, 128, token_count); + ggml_tensor * ids_storage = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, route_stride, token_count); + REQUIRE(scales != nullptr); + REQUIRE(ids_storage != nullptr); + ggml_tensor * ids = route_stride == 8 ? ids_storage : + ggml_view_2d(ctx, ids_storage, 8, token_count, ids_storage->nb[1], 0); + REQUIRE(ids != nullptr); + ggml_tensor * output = ggml_get_rows(ctx, scales, ids); + REQUIRE(output != nullptr); + ggml_set_output(output); + + ggml_cgraph * graph = ggml_new_graph(ctx); + REQUIRE(graph != nullptr); + ggml_build_forward_expand(graph, output); + require_kernel_subsequence(scheduled_kernel_sequence(graph), { "loom_libs:ggml_get_rows_scalar_3d_f32" }); + + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); + REQUIRE(buffer != nullptr); + std::vector scales_data(128 * token_count); + std::vector ids_data(route_stride * token_count, -1); + std::vector expected(8 * token_count); + for (int64_t token = 0; token < token_count; ++token) { + for (int64_t expert = 0; expert < 128; ++expert) { + scales_data[token * 128 + expert] = static_cast(token * 1024) + expert * 0.25f; + } + for (int64_t route = 0; route < 8; ++route) { + int32_t expert = route == 0 ? 0 : route == 7 ? 127 : + static_cast((token * 17 + route * 19) % 128); + if (include_invalid_ids) { + switch (route) { + case 1: expert = -1; break; + case 2: expert = 128; break; + case 3: expert = std::numeric_limits::min(); break; + case 4: expert = std::numeric_limits::max(); break; + } + } + ids_data[token * route_stride + route] = expert; + const bool valid_expert = expert >= 0 && expert < 128; + expected[token * 8 + route] = valid_expert ? scales_data[token * 128 + expert] : 0.0f; + } + } + + ggml_backend_tensor_set(scales, scales_data.data(), 0, scales_data.size() * sizeof(float)); + ggml_backend_tensor_set(ids_storage, ids_data.data(), 0, ids_data.size() * sizeof(int32_t)); + REQUIRE(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(backend); + + std::vector actual(expected.size()); + ggml_backend_tensor_get(output, actual.data(), 0, actual.size() * sizeof(float)); + REQUIRE(actual == expected); + + ggml_backend_buffer_free(buffer); + ggml_free(ctx); + ggml_backend_free(backend); +} + static void run_router_top8_case(int64_t token_count) { ggml_backend_t backend = ggml_backend_hrx_init(0); REQUIRE(backend != nullptr); @@ -4510,7 +4584,9 @@ static void run_router_top8_case(int64_t token_count) { ggml_tensor * logits = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, kQwenRouterExpertCount, token_count); REQUIRE(logits != nullptr); - ggml_tensor * output = build_qwen_router_top8_graph(ctx, logits); + ggml_tensor * route_ids = nullptr; + ggml_tensor * output = build_qwen_router_top8_graph(ctx, logits, &route_ids); + ggml_set_output(route_ids); ggml_cgraph * graph = ggml_new_graph(ctx); REQUIRE(graph != nullptr); @@ -4522,20 +4598,36 @@ static void run_router_top8_case(int64_t token_count) { REQUIRE(buffer != nullptr); const std::vector logits_data = make_router_logits(token_count); - const std::vector expected = router_top8_weights_reference(logits_data, token_count); + const RouterTop8Reference expected = router_top8_reference(logits_data, token_count); ggml_backend_tensor_set(logits, logits_data.data(), 0, logits_data.size() * sizeof(float)); REQUIRE(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS); ggml_backend_synchronize(backend); - std::vector actual(expected.size()); + std::vector actual(expected.weights.size()); ggml_backend_tensor_get(output, actual.data(), 0, actual.size() * sizeof(float)); for (size_t i = 0; i < actual.size(); ++i) { - const float diff = std::fabs(actual[i] - expected[i]); + const float diff = std::fabs(actual[i] - expected.weights[i]); REQUIRE(diff <= 1.0e-5f); } + const size_t row_stride = route_ids->nb[1] / sizeof(int32_t); + const size_t span = (static_cast(token_count) - 1) * row_stride + kQwenRouterRouteCount; + std::vector actual_ids(span); + ggml_backend_tensor_get(route_ids, actual_ids.data(), 0, span * sizeof(int32_t)); + for (int64_t token = 0; token < token_count; ++token) { + for (int64_t route = 0; route < kQwenRouterRouteCount; ++route) { + const size_t output_index = token * kQwenRouterRouteCount + route; + const size_t storage_index = token * row_stride + route; + if (actual_ids[storage_index] != expected.ids[output_index]) { + std::fprintf(stderr, "router IDs token=%" PRId64 " route=%" PRId64 " stride=%zu actual=%d expected=%d\n", + token, route, row_stride, actual_ids[storage_index], expected.ids[output_index]); + } + REQUIRE(actual_ids[storage_index] == expected.ids[output_index]); + } + } + ggml_backend_buffer_free(buffer); ggml_free(ctx); ggml_backend_free(backend); @@ -9064,7 +9156,20 @@ static void register_rmsnorm_and_scheduling_cases(Suite & suite) { GGML_GLU_OP_SWIGLU, 2048, true); }); suite.device_case("mixed_symmetric_source_eligibility", [] { run_mixed_symmetric_source_eligibility_case(); }); - suite.device_case("router_top8.tokens4", [] { run_router_top8_case(4); }); + for (int64_t tokens : { 1, 2, 8 }) { + for (int64_t stride : { 8, 128 }) { + suite.device_case("scalar_3d_get_rows.tokens" + std::to_string(tokens) + ".stride" + + std::to_string(stride), + [=] { run_scalar_3d_get_rows_case(tokens, stride); }); + } + } + for (int64_t stride : { 8, 128 }) { + suite.device_case("scalar_3d_get_rows.invalid_ids.stride" + std::to_string(stride), + [=] { run_scalar_3d_get_rows_case(2, stride, true); }); + } + for (int64_t tokens : { 1, 2, 4, 8 }) { + suite.device_case("router_top8.tokens" + std::to_string(tokens), [=] { run_router_top8_case(tokens); }); + } for (int64_t experts : { 128, 256, 512 }) { for (int64_t tokens : { 1, 2, 512 }) { suite.device_case("router_common_matmul.experts" + std::to_string(experts) + ".tokens" + std::to_string(tokens), From db39fcb2d64bf126f84347602516835e5699df0d Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Sun, 8 Nov 2026 18:50:54 +0000 Subject: [PATCH 2/5] ggml-hrx: fuse expert scale repeat with routed gather (lemonade-ws-enb9) --- .../common/dispatch-get-rows.cpp | 148 ++++++++++++++++++ ggml/src/ggml-hrx/ggml-hrx.cpp | 20 +++ .../kernels/loom-libs/manifest.json | 46 ++++++ .../ops/rows/get_rows_scalar_3d_f32.loom | 15 +- .../ops/rows/repeat_scalar_3d_f32.loom | 23 +++ tests/hrx-backend-test.cpp | 54 +++++++ tests/hrx-ops-test.cpp | 60 ++++++- 7 files changed, 355 insertions(+), 11 deletions(-) create mode 100644 ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rows/repeat_scalar_3d_f32.loom diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp index fd7fbfb481cd..59f9418cd930 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp @@ -21,6 +21,8 @@ namespace { static constexpr KernelCatalogRef kGetRowsF32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_get_rows_f32"); static constexpr KernelCatalogRef kGetRowsScalar3dF32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_get_rows_scalar_3d_f32"); +static constexpr KernelCatalogRef kRepeatScalar3dF32Kernel = + GGML_HRX_KERNEL_REF("loom_libs", "ggml_repeat_scalar_3d_f32"); static constexpr KernelCatalogRef kGetRowsF32NextKernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_get_rows_f32_next"); static constexpr KernelCatalogRef kGetRowsScaleF32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_get_rows_scale_f32"); @@ -155,6 +157,135 @@ static void add_primary_bindings(Dispatch & dispatch, const GetRowsMatch & match dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count }); } +static bool match_repeat_scalar_3d_f32_input(const Graph & graph, const GraphNode * node, + const Value *& source, const Value *& repeated) { + const bool is_repeat = node != nullptr && node->op == GGML_OP_REPEAT; + const bool has_one_input = node != nullptr && node->inputs.size() == 1; + if (!is_repeat || !has_one_input) { + return false; + } + + source = graph_value(graph, node->inputs[0]); + repeated = graph_value(graph, node->output); + const bool has_values = source != nullptr && repeated != nullptr; + if (!has_values) { + return false; + } + + const bool has_types = source->type == GGML_TYPE_F32 && repeated->type == GGML_TYPE_F32; + const bool has_source_shape = source->ne[0] == 1 && source->ne[1] == 128 && + source->ne[2] == 1 && source->ne[3] == 1; + const bool has_repeat_shape = repeated->ne[0] == 1 && repeated->ne[1] == 128 && + repeated->ne[3] == 1 && is_supported_token_count(repeated->ne[2]); + const bool has_layout = source->contiguous && repeated->contiguous; + return has_types && has_source_shape && has_repeat_shape && has_layout; +} + +static bool match_repeat_scalar_3d_f32_dispatch(const DispatchMatchContext & context, DispatchMatch & match) { + const Value * source = nullptr; + const Value * repeated = nullptr; + if (!match_repeat_scalar_3d_f32_input(context.graph, context.root_node, source, repeated)) { + return false; + } + + Dispatch dispatch; + dispatch.kernel = make_kernel_specialization(kRepeatScalar3dF32Kernel); + dispatch.kernel.workload_specialization = WorkloadSpecialization::Dynamic; + dispatch.kernel.integer_parameters.emplace("token_count", repeated->ne[2]); + dispatch.bindings.push_back({ source->id, 0, source->byte_count }); + dispatch.bindings.push_back({ repeated->id, 0, repeated->byte_count }); + + match.covered_nodes.push_back(context.root_index); + match.dispatches.push_back(std::move(dispatch)); + return true; +} + +static bool match_repeat_get_rows_scalar_3d_f32_dispatch(const DispatchMatchContext & context, DispatchMatch & match) { + const Value * source = nullptr; + const Value * repeated = nullptr; + if (!match_repeat_scalar_3d_f32_input(context.graph, context.root_node, source, repeated)) { + return false; + } + if (repeated->kind != ValueKind::Transient) { + return false; + } + + const std::vector & consumers = context.graph.index().consumers(repeated->id); + const bool has_one_consumer = consumers.size() == 1; + if (!has_one_consumer) { + return false; + } + const GraphNode * rows = consumers.front(); + const bool is_get_rows = rows != nullptr && rows->op == GGML_OP_GET_ROWS; + const bool has_two_inputs = rows != nullptr && rows->inputs.size() == 2; + if (!is_get_rows || !has_two_inputs) { + return false; + } + const bool has_repeat_input = rows->inputs[0] == repeated->id; + if (!has_repeat_input) { + return false; + } + + const Value * ids = graph_value(context.graph, rows->inputs[1]); + const Value * output = graph_value(context.graph, rows->output); + const bool has_values = ids != nullptr && output != nullptr; + if (!has_values) { + return false; + } + const GraphNode * ids_producer = context.graph.index().producer(ids->id); + size_t ids_producer_index = 0; + const bool has_producer_index = ids_producer != nullptr && + context.graph.index().node_index(ids_producer, ids_producer_index); + const bool ids_ready = ids_producer == nullptr || + (has_producer_index && ids_producer_index < context.covered_nodes.size() && + context.covered_nodes[ids_producer_index]); + if (!ids_ready) { + return false; + } + const int64_t token_count = repeated->ne[2]; + const bool has_id_shape = ids->ne[0] == 8 && ids->ne[1] == token_count && + ids->ne[2] == 1 && ids->ne[3] == 1; + const bool has_output_shape = output->ne[0] == 1 && output->ne[1] == 8 && + output->ne[2] == token_count && output->ne[3] == 1; + const bool has_types = ids->type == GGML_TYPE_I32 && output->type == GGML_TYPE_F32; + const bool has_layout = output->contiguous && ids->nb[0] == sizeof(int32_t) && + ids->nb[1] % sizeof(int32_t) == 0; + if (!(has_id_shape && has_output_shape && has_types && has_layout)) { + return false; + } + + const size_t route_stride = ids->nb[1] / sizeof(int32_t); + const bool has_supported_stride = route_stride >= 8 && route_stride <= 512; + const size_t expected_id_bytes = (static_cast(token_count) - 1) * ids->nb[1] + 8 * sizeof(int32_t); + const bool has_exact_id_span = ids->byte_count == expected_id_bytes; + if (!has_supported_stride || !has_exact_id_span) { + return false; + } + + size_t rows_index = 0; + const bool has_rows_index = context.graph.index().node_index(rows, rows_index); + const bool rows_available = has_rows_index && rows_index < context.covered_nodes.size() && + !context.covered_nodes[rows_index]; + if (!rows_available) { + return false; + } + + Dispatch dispatch; + dispatch.kernel = make_kernel_specialization(kGetRowsScalar3dF32Kernel); + dispatch.kernel.workload_specialization = WorkloadSpecialization::Dynamic; + dispatch.kernel.integer_parameters.emplace("token_count", token_count); + dispatch.kernel.integer_parameters.emplace("route_stride", static_cast(route_stride)); + dispatch.kernel.integer_parameters.emplace("scale_token_stride", 0); + dispatch.bindings.push_back({ ids->id, 0, ids->byte_count }); + dispatch.bindings.push_back({ source->id, 0, source->byte_count }); + dispatch.bindings.push_back({ output->id, 0, output->byte_count }); + + match.covered_nodes.push_back(context.root_index); + match.covered_nodes.push_back(rows_index); + match.dispatches.push_back(std::move(dispatch)); + return true; +} + static bool match_get_rows_scalar_3d_f32_dispatch(const DispatchMatchContext & context, DispatchMatch & match) { const GraphNode * node = context.root_node; const bool is_get_rows = node != nullptr && node->op == GGML_OP_GET_ROWS; @@ -202,6 +333,7 @@ static bool match_get_rows_scalar_3d_f32_dispatch(const DispatchMatchContext & c dispatch.kernel.workload_specialization = WorkloadSpecialization::Dynamic; dispatch.kernel.integer_parameters.emplace("token_count", token_count); dispatch.kernel.integer_parameters.emplace("route_stride", static_cast(route_stride)); + dispatch.kernel.integer_parameters.emplace("scale_token_stride", 128); dispatch.bindings.push_back({ ids->id, 0, ids->byte_count }); dispatch.bindings.push_back({ scales->id, 0, scales->byte_count }); dispatch.bindings.push_back({ output->id, 0, output->byte_count }); @@ -530,6 +662,22 @@ static bool match_get_rows_rmsnorm_binary_dispatch(const DispatchMatchContext & } // namespace void register_get_rows_dispatches(DispatchRegistryBuilder & registry) { + registry.add({ + "common.repeat_get_rows.scalar_3d_f32", + GGML_OP_REPEAT, + DispatchMatchKind::Fused, + 300, + DispatchSource::Common, + match_repeat_get_rows_scalar_3d_f32_dispatch, + }); + registry.add({ + "common.repeat.scalar_3d_f32", + GGML_OP_REPEAT, + DispatchMatchKind::SingleOp, + 125, + DispatchSource::Common, + match_repeat_scalar_3d_f32_dispatch, + }); registry.add({ "common.get_rows.scalar_3d_f32", GGML_OP_GET_ROWS, diff --git a/ggml/src/ggml-hrx/ggml-hrx.cpp b/ggml/src/ggml-hrx/ggml-hrx.cpp index c0620720a058..ca6314ffe387 100644 --- a/ggml/src/ggml-hrx/ggml-hrx.cpp +++ b/ggml/src/ggml-hrx/ggml-hrx.cpp @@ -1089,6 +1089,23 @@ static bool supported_scale_f32_tensor(const ggml_tensor * op) { static_cast(ggml_nelements(op)) <= std::numeric_limits::max(); } +static bool supported_repeat_scalar_3d_f32_tensor(const ggml_tensor * op) { + const bool is_repeat = op != nullptr && op->op == GGML_OP_REPEAT; + const bool has_source = is_repeat && op->src[0] != nullptr; + if (!has_source) { + return false; + } + + const ggml_tensor * source = op->src[0]; + const bool has_types = source->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + const bool has_source_shape = source->ne[0] == 1 && source->ne[1] == 128 && + source->ne[2] == 1 && source->ne[3] == 1; + const bool has_repeat_shape = op->ne[0] == 1 && op->ne[1] == 128 && op->ne[2] >= 1 && + op->ne[2] <= 2048 && op->ne[3] == 1; + const bool has_layout = ggml_is_contiguous(source) && ggml_is_contiguous(op); + return has_types && has_source_shape && has_repeat_shape && has_layout; +} + static bool supported_qwen_attention_projection_get_rows_tensor(const ggml_tensor * op) { if (op == nullptr || op->op != GGML_OP_GET_ROWS || op->src[0] == nullptr || op->src[1] == nullptr || op->src[0]->op != GGML_OP_MUL_MAT || op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 || @@ -1198,6 +1215,9 @@ static bool device_supports_op(ggml_backend_dev_t device, const ggml_tensor * op if (supported_scale_f32_tensor(op)) { return true; } + if (supported_repeat_scalar_3d_f32_tensor(op)) { + return true; + } // Admit pooling for the amax-scale fusion; graph claim validates the complete chain. if (supported_full_max_pool_1d_f32_tensor(op)) { return true; diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json index 5c3a44febca5..134cecc4297e 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json @@ -317,6 +317,9 @@ { "path": "ops/rows/get_rows_scalar_3d_f32.loom" }, + { + "path": "ops/rows/repeat_scalar_3d_f32.loom" + }, { "path": "ops/rows/get_rows_scale_f32.loom" }, @@ -6676,6 +6679,10 @@ { "name": "route_stride", "type": "index" + }, + { + "name": "scale_token_stride", + "type": "index" } ], "launch_parameters": [ @@ -6686,6 +6693,10 @@ { "name": "route_stride", "type": "index" + }, + { + "name": "scale_token_stride", + "type": "index" } ], "bindings": [ @@ -6708,6 +6719,41 @@ }, "compile_dependencies": [] }, + { + "name": "ggml_repeat_scalar_3d_f32", + "family": "loom_libs", + "symbol": "ggml_repeat_scalar_3d_f32", + "source": "ops/rows/repeat_scalar_3d_f32.loom", + "workload_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "launch_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "bindings": [ + "scales", + "output" + ], + "binding_access": [ + "read", + "read_write" + ], + "target_selector": "", + "compile_recipe": { + "mode": "direct", + "primary_sources": [ + "ops/rows/repeat_scalar_3d_f32.loom" + ], + "library_sources": [] + }, + "compile_dependencies": [] + }, { "name": "ggml_get_rows_f32", "family": "loom_libs", diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rows/get_rows_scalar_3d_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rows/get_rows_scalar_3d_f32.loom index 0c0133b55079..068d566b3586 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rows/get_rows_scalar_3d_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rows/get_rows_scalar_3d_f32.loom @@ -5,17 +5,19 @@ // narrow view of wider argsort rows; only the selected IDs are in bounds. amdgpu.target @ggml_get_rows_scalar_3d_gfx11_wave64 {subgroup_size = 64} -kernel.def target(@ggml_get_rows_scalar_3d_gfx11_wave64) @ggml_get_rows_scalar_3d_f32(%token_count: index, %route_stride: index) { +kernel.def target(@ggml_get_rows_scalar_3d_gfx11_wave64) @ggml_get_rows_scalar_3d_f32(%token_count: index, %route_stride: index, %scale_token_stride: index) { %c1 = index.constant 1 : index %c64 = index.constant 64 : index kernel.launch.config workgroups(%token_count, %c1, %c1) workgroup_size(%c64, %c1, %c1) : index -} launch(%token_count: index, %route_stride: index, %route_ids: buffer, %scales: buffer, %output: buffer) where [range(%token_count, 1, 2048), range(%route_stride, 8, 512)] { +} launch(%token_count: index, %route_stride: index, %scale_token_stride: index, %route_ids: buffer, %scales: buffer, %output: buffer) where [range(%token_count, 1, 2048), range(%route_stride, 8, 512), range(%scale_token_stride, 0, 128)] { %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048)] : index %bounded_route_stride = index.assume %route_stride [range(%route_stride, 8, 512)] : index + %bounded_scale_token_stride = index.assume %scale_token_stride [range(%scale_token_stride, 0, 128)] : index %token0 = kernel.workgroup.id : index %route0 = kernel.workitem.id : index %c1 = index.constant 1 : index %c8 = index.constant 8 : index + %c128 = index.constant 128 : index %c0_i32 = scalar.constant 0 : i32 %c128_i32 = scalar.constant 128 : i32 %c0_f32 = scalar.constant 0.0 : f32 @@ -27,7 +29,9 @@ kernel.def target(@ggml_get_rows_scalar_3d_gfx11_wave64) @ggml_get_rows_scalar_3 %last_row_base = index.mul %last_token, %bounded_route_stride : index %route_id_span = index.add %last_row_base, %c8 : index %ids_view = buffer.view %route_ids[%c0_offset] : buffer -> view<[%route_id_span]xi32> - %scales_view = buffer.view %scales[%c0_offset] : buffer -> view<[%bounded_token_count]x128xf32> + %last_scale_row = index.mul %last_token, %bounded_scale_token_stride : index + %scale_span = index.add %last_scale_row, %c128 : index + %scales_view = buffer.view %scales[%c0_offset] : buffer -> view<[%scale_span]xf32> %output_view = buffer.view %output[%c0_offset] : buffer -> view<[%bounded_token_count]x8xf32> scf.if %publish { %token = index.assume %token0 [lt(%token0, %bounded_token_count)] : index @@ -43,7 +47,10 @@ kernel.def target(@ggml_get_rows_scalar_3d_gfx11_wave64) @ggml_get_rows_scalar_3 %scale = scf.if %valid_expert -> (f32) { %expert0 = index.cast %expert_i32 : i32 to index %expert = index.assume %expert0 [range(%expert0, 0, 127)] : index - %selected_scale = view.load %scales_view[%token, %expert] : view<[%bounded_token_count]x128xf32> -> f32 + %scale_row = index.mul %token, %bounded_scale_token_stride : index + %scale_index = index.add %scale_row, %expert : index + %bounded_scale_index = index.assume %scale_index [lt(%scale_index, %scale_span)] : index + %selected_scale = view.load %scales_view[%bounded_scale_index] : view<[%scale_span]xf32> -> f32 scf.yield %selected_scale : f32 } else { scf.yield %c0_f32 : f32 diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rows/repeat_scalar_3d_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rows/repeat_scalar_3d_f32.loom new file mode 100644 index 000000000000..ef507f0d95da --- /dev/null +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rows/repeat_scalar_3d_f32.loom @@ -0,0 +1,23 @@ +// Copyright 2026 The HRX Authors +// SPDX-License-Identifier: Apache-2.0 + +// Materialize the exact 128-scale token broadcast when a consumer needs the repeat itself. +amdgpu.target @ggml_repeat_scalar_3d_gfx11_wave64 {subgroup_size = 64} + +kernel.def target(@ggml_repeat_scalar_3d_gfx11_wave64) @ggml_repeat_scalar_3d_f32(%token_count: index) { + %c1 = index.constant 1 : index + %c128 = index.constant 128 : index + kernel.launch.config workgroups(%token_count, %c1, %c1) workgroup_size(%c128, %c1, %c1) : index +} launch(%token_count: index, %scales: buffer, %output: buffer) where [range(%token_count, 1, 2048)] { + %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048)] : index + %token0 = kernel.workgroup.id : index + %expert0 = kernel.workitem.id : index + %token = index.assume %token0 [lt(%token0, %bounded_token_count)] : index + %expert = index.assume %expert0 [range(%expert0, 0, 127)] : index + %c0_offset = index.constant 0 : offset + %scales_view = buffer.view %scales[%c0_offset] : buffer -> view<128xf32> + %output_view = buffer.view %output[%c0_offset] : buffer -> view<[%bounded_token_count]x128xf32> + %scale = view.load %scales_view[%expert] : view<128xf32> -> f32 + view.store %scale, %output_view[%token, %expert] : f32, view<[%bounded_token_count]x128xf32> + kernel.return +} diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index 530b449d7406..e0fafcfe1b6a 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -4505,6 +4505,59 @@ static void run_scalar_3d_get_rows_dispatch_checks() { check_case(128, 8, 2, 513, false); } +static void run_repeat_get_rows_scalar_3d_dispatch_checks() { + auto check_case = [](int64_t token_count, int64_t route_stride, bool expose_repeat, + bool ids_ready_before_repeat = true) { + ggml_init_params params = {}; + params.mem_size = 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + + ggml_tensor * scales = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 128); + ggml_tensor * ids_storage = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, route_stride, token_count); + ggml_tensor * ids = route_stride == 8 ? ids_storage : + ggml_view_2d(ctx, ids_storage, 8, token_count, ids_storage->nb[1], 0); + ggml_tensor * reshaped = ggml_reshape_3d(ctx, scales, 1, 128, 1); + ggml_tensor * repeated = ggml_repeat_4d(ctx, reshaped, 1, 128, token_count, 1); + ggml_tensor * output = ggml_get_rows(ctx, repeated, ids); + REQUIRE(output != nullptr); + if (expose_repeat) { + ggml_set_output(repeated); + } + ggml_set_output(output); + + ggml_cgraph * graph = ggml_new_graph(ctx); + REQUIRE(graph != nullptr); + if (ids_ready_before_repeat) { + ggml_build_forward_expand(graph, ids); + } + ggml_build_forward_expand(graph, output); + ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*graph); + REQUIRE(imported.valid()); + + ggml::hrx::DispatchScheduler scheduler; + REQUIRE(scheduler.schedule_graph(imported.graph, test_dispatch_target())); + const auto & dispatches = scheduler.plan().dispatches; + const bool expect_fusion = !expose_repeat && ids_ready_before_repeat; + REQUIRE(dispatches.size() == (expect_fusion ? 1 : 2)); + if (!expect_fusion) { + REQUIRE(kernel_name_for_id(dispatches[0].kernel.kernel_id) == "loom_libs:ggml_repeat_scalar_3d_f32"); + } + const ggml::hrx::Dispatch & gather = dispatches.back(); + REQUIRE(kernel_name_for_id(gather.kernel.kernel_id) == "loom_libs:ggml_get_rows_scalar_3d_f32"); + REQUIRE(gather.kernel.integer_parameters.at("scale_token_stride") == (expect_fusion ? 0 : 128)); + REQUIRE(gather.bindings[1].length == (expect_fusion ? ggml_nbytes(scales) : ggml_nbytes(repeated))); + ggml_free(ctx); + }; + + check_case(1, 8, false); + check_case(2, 128, false); + check_case(128, 8, false); + check_case(2, 128, true); + check_case(2, 128, false, false); +} + static void run_qwen_token_embedding_dispatch_checks() { ggml_init_params params = {}; params.mem_size = 4 * 1024 * 1024; @@ -14925,6 +14978,7 @@ static void register_hrx_backend_host_cases(test_runner::Suite & suite) { suite.host_case("graph_traversal", [] { run_graph_traversal_checks(); }); suite.host_case("qwen_token_embedding_dispatch", [] { run_qwen_token_embedding_dispatch_checks(); }); suite.host_case("scalar_3d_get_rows_dispatch", [] { run_scalar_3d_get_rows_dispatch_checks(); }); + suite.host_case("repeat_get_rows_scalar_3d_dispatch", [] { run_repeat_get_rows_scalar_3d_dispatch_checks(); }); suite.host_case("get_rows_rmsnorm_binary_dispatch", [] { run_get_rows_rmsnorm_binary_dispatch_checks(); }); suite.host_case("get_rows_scale_dispatch", [] { run_get_rows_scale_dispatch_checks(); }); suite.host_case("gather_add_dispatch", [] { run_gather_add_dispatch_checks(); }); diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index 491321e2f555..9db56a021b93 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -4506,7 +4506,8 @@ static void run_q4_prefill_source_eligibility_case() { ggml_backend_free(backend); } -static void run_scalar_3d_get_rows_case(int64_t token_count, int64_t route_stride, bool include_invalid_ids = false) { +static void run_scalar_3d_get_rows_case(int64_t token_count, int64_t route_stride, bool include_invalid_ids = false, + bool broadcast_scales = false, bool expose_repeat = false) { ggml_backend_t backend = ggml_backend_hrx_init(0); REQUIRE(backend != nullptr); @@ -4516,30 +4517,52 @@ static void run_scalar_3d_get_rows_case(int64_t token_count, int64_t route_strid ggml_context * ctx = ggml_init(params); REQUIRE(ctx != nullptr); - ggml_tensor * scales = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1, 128, token_count); + ggml_tensor * scales = broadcast_scales ? ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 128) : + ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1, 128, token_count); ggml_tensor * ids_storage = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, route_stride, token_count); REQUIRE(scales != nullptr); REQUIRE(ids_storage != nullptr); ggml_tensor * ids = route_stride == 8 ? ids_storage : ggml_view_2d(ctx, ids_storage, 8, token_count, ids_storage->nb[1], 0); REQUIRE(ids != nullptr); - ggml_tensor * output = ggml_get_rows(ctx, scales, ids); + ggml_tensor * repeated = nullptr; + ggml_tensor * gather_scales = scales; + if (broadcast_scales) { + ggml_tensor * reshaped = ggml_reshape_3d(ctx, scales, 1, 128, 1); + repeated = ggml_repeat_4d(ctx, reshaped, 1, 128, token_count, 1); + gather_scales = repeated; + if (expose_repeat) { + ggml_set_output(repeated); + } + } + ggml_tensor * output = ggml_get_rows(ctx, gather_scales, ids); REQUIRE(output != nullptr); ggml_set_output(output); ggml_cgraph * graph = ggml_new_graph(ctx); REQUIRE(graph != nullptr); + if (broadcast_scales) { + ggml_build_forward_expand(graph, ids); + } ggml_build_forward_expand(graph, output); - require_kernel_subsequence(scheduled_kernel_sequence(graph), { "loom_libs:ggml_get_rows_scalar_3d_f32" }); + if (expose_repeat) { + require_kernel_subsequence(scheduled_kernel_sequence(graph), + { "loom_libs:ggml_repeat_scalar_3d_f32", "loom_libs:ggml_get_rows_scalar_3d_f32" }); + } else { + require_kernel_subsequence(scheduled_kernel_sequence(graph), { "loom_libs:ggml_get_rows_scalar_3d_f32" }); + } ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); REQUIRE(buffer != nullptr); - std::vector scales_data(128 * token_count); + std::vector scales_data(broadcast_scales ? 128 : 128 * token_count); std::vector ids_data(route_stride * token_count, -1); std::vector expected(8 * token_count); for (int64_t token = 0; token < token_count; ++token) { for (int64_t expert = 0; expert < 128; ++expert) { - scales_data[token * 128 + expert] = static_cast(token * 1024) + expert * 0.25f; + if (!broadcast_scales || token == 0) { + const size_t scale_index = static_cast(broadcast_scales ? expert : token * 128 + expert); + scales_data[scale_index] = static_cast(broadcast_scales ? 0 : token * 1024) + expert * 0.25f; + } } for (int64_t route = 0; route < 8; ++route) { int32_t expert = route == 0 ? 0 : route == 7 ? 127 : @@ -4554,7 +4577,8 @@ static void run_scalar_3d_get_rows_case(int64_t token_count, int64_t route_strid } ids_data[token * route_stride + route] = expert; const bool valid_expert = expert >= 0 && expert < 128; - expected[token * 8 + route] = valid_expert ? scales_data[token * 128 + expert] : 0.0f; + const size_t scale_index = static_cast((broadcast_scales ? 0 : token * 128) + expert); + expected[token * 8 + route] = valid_expert ? scales_data[scale_index] : 0.0f; } } @@ -4566,6 +4590,15 @@ static void run_scalar_3d_get_rows_case(int64_t token_count, int64_t route_strid std::vector actual(expected.size()); ggml_backend_tensor_get(output, actual.data(), 0, actual.size() * sizeof(float)); REQUIRE(actual == expected); + if (expose_repeat) { + std::vector repeated_data(128 * token_count); + ggml_backend_tensor_get(repeated, repeated_data.data(), 0, repeated_data.size() * sizeof(float)); + for (int64_t token = 0; token < token_count; ++token) { + for (int64_t expert = 0; expert < 128; ++expert) { + REQUIRE(repeated_data[token * 128 + expert] == scales_data[expert]); + } + } + } ggml_backend_buffer_free(buffer); ggml_free(ctx); @@ -9167,6 +9200,19 @@ static void register_rmsnorm_and_scheduling_cases(Suite & suite) { suite.device_case("scalar_3d_get_rows.invalid_ids.stride" + std::to_string(stride), [=] { run_scalar_3d_get_rows_case(2, stride, true); }); } + for (int64_t tokens : { 1, 2, 8, 128 }) { + for (int64_t stride : { 8, 128 }) { + suite.device_case("repeat_get_rows_scalar_3d.tokens" + std::to_string(tokens) + ".stride" + + std::to_string(stride), + [=] { run_scalar_3d_get_rows_case(tokens, stride, false, true); }); + } + } + suite.device_case("repeat_get_rows_scalar_3d.invalid_ids", [] { + run_scalar_3d_get_rows_case(2, 128, true, true); + }); + suite.device_case("repeat_get_rows_scalar_3d.exposed_repeat", [] { + run_scalar_3d_get_rows_case(2, 128, false, true, true); + }); for (int64_t tokens : { 1, 2, 4, 8 }) { suite.device_case("router_top8.tokens" + std::to_string(tokens), [=] { run_router_top8_case(tokens); }); } From f4f4f8216613226341c8bea092e93e17e7b4cdd0 Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Sun, 8 Nov 2026 23:33:54 +0000 Subject: [PATCH 3/5] ggml-hrx: fuse Gemma expert scale into routed matmul (lemonade-ws-enb9) --- .../common/dispatch-get-rows.cpp | 95 --------- .../common/dispatch-mul-mat-id.cpp | 181 ++++++++++++++++++ .../kernels/loom-libs/manifest.json | 144 +++++++++++++- .../skinny_input_f32_scaled_publish_f32.loom | 113 +++++++++++ .../tiled_input_f32_scaled_publish_f32.loom | 113 +++++++++++ .../ops/rows/get_rows_scalar_3d_f32.loom | 9 +- tests/hrx-backend-test.cpp | 16 +- tests/hrx-ops-test.cpp | 131 ++++++++++++- 8 files changed, 681 insertions(+), 121 deletions(-) create mode 100644 ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_scaled_publish_f32.loom create mode 100644 ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_scaled_publish_f32.loom diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp index 59f9418cd930..9ef56a6d0d15 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp @@ -200,92 +200,6 @@ static bool match_repeat_scalar_3d_f32_dispatch(const DispatchMatchContext & con return true; } -static bool match_repeat_get_rows_scalar_3d_f32_dispatch(const DispatchMatchContext & context, DispatchMatch & match) { - const Value * source = nullptr; - const Value * repeated = nullptr; - if (!match_repeat_scalar_3d_f32_input(context.graph, context.root_node, source, repeated)) { - return false; - } - if (repeated->kind != ValueKind::Transient) { - return false; - } - - const std::vector & consumers = context.graph.index().consumers(repeated->id); - const bool has_one_consumer = consumers.size() == 1; - if (!has_one_consumer) { - return false; - } - const GraphNode * rows = consumers.front(); - const bool is_get_rows = rows != nullptr && rows->op == GGML_OP_GET_ROWS; - const bool has_two_inputs = rows != nullptr && rows->inputs.size() == 2; - if (!is_get_rows || !has_two_inputs) { - return false; - } - const bool has_repeat_input = rows->inputs[0] == repeated->id; - if (!has_repeat_input) { - return false; - } - - const Value * ids = graph_value(context.graph, rows->inputs[1]); - const Value * output = graph_value(context.graph, rows->output); - const bool has_values = ids != nullptr && output != nullptr; - if (!has_values) { - return false; - } - const GraphNode * ids_producer = context.graph.index().producer(ids->id); - size_t ids_producer_index = 0; - const bool has_producer_index = ids_producer != nullptr && - context.graph.index().node_index(ids_producer, ids_producer_index); - const bool ids_ready = ids_producer == nullptr || - (has_producer_index && ids_producer_index < context.covered_nodes.size() && - context.covered_nodes[ids_producer_index]); - if (!ids_ready) { - return false; - } - const int64_t token_count = repeated->ne[2]; - const bool has_id_shape = ids->ne[0] == 8 && ids->ne[1] == token_count && - ids->ne[2] == 1 && ids->ne[3] == 1; - const bool has_output_shape = output->ne[0] == 1 && output->ne[1] == 8 && - output->ne[2] == token_count && output->ne[3] == 1; - const bool has_types = ids->type == GGML_TYPE_I32 && output->type == GGML_TYPE_F32; - const bool has_layout = output->contiguous && ids->nb[0] == sizeof(int32_t) && - ids->nb[1] % sizeof(int32_t) == 0; - if (!(has_id_shape && has_output_shape && has_types && has_layout)) { - return false; - } - - const size_t route_stride = ids->nb[1] / sizeof(int32_t); - const bool has_supported_stride = route_stride >= 8 && route_stride <= 512; - const size_t expected_id_bytes = (static_cast(token_count) - 1) * ids->nb[1] + 8 * sizeof(int32_t); - const bool has_exact_id_span = ids->byte_count == expected_id_bytes; - if (!has_supported_stride || !has_exact_id_span) { - return false; - } - - size_t rows_index = 0; - const bool has_rows_index = context.graph.index().node_index(rows, rows_index); - const bool rows_available = has_rows_index && rows_index < context.covered_nodes.size() && - !context.covered_nodes[rows_index]; - if (!rows_available) { - return false; - } - - Dispatch dispatch; - dispatch.kernel = make_kernel_specialization(kGetRowsScalar3dF32Kernel); - dispatch.kernel.workload_specialization = WorkloadSpecialization::Dynamic; - dispatch.kernel.integer_parameters.emplace("token_count", token_count); - dispatch.kernel.integer_parameters.emplace("route_stride", static_cast(route_stride)); - dispatch.kernel.integer_parameters.emplace("scale_token_stride", 0); - dispatch.bindings.push_back({ ids->id, 0, ids->byte_count }); - dispatch.bindings.push_back({ source->id, 0, source->byte_count }); - dispatch.bindings.push_back({ output->id, 0, output->byte_count }); - - match.covered_nodes.push_back(context.root_index); - match.covered_nodes.push_back(rows_index); - match.dispatches.push_back(std::move(dispatch)); - return true; -} - static bool match_get_rows_scalar_3d_f32_dispatch(const DispatchMatchContext & context, DispatchMatch & match) { const GraphNode * node = context.root_node; const bool is_get_rows = node != nullptr && node->op == GGML_OP_GET_ROWS; @@ -333,7 +247,6 @@ static bool match_get_rows_scalar_3d_f32_dispatch(const DispatchMatchContext & c dispatch.kernel.workload_specialization = WorkloadSpecialization::Dynamic; dispatch.kernel.integer_parameters.emplace("token_count", token_count); dispatch.kernel.integer_parameters.emplace("route_stride", static_cast(route_stride)); - dispatch.kernel.integer_parameters.emplace("scale_token_stride", 128); dispatch.bindings.push_back({ ids->id, 0, ids->byte_count }); dispatch.bindings.push_back({ scales->id, 0, scales->byte_count }); dispatch.bindings.push_back({ output->id, 0, output->byte_count }); @@ -662,14 +575,6 @@ static bool match_get_rows_rmsnorm_binary_dispatch(const DispatchMatchContext & } // namespace void register_get_rows_dispatches(DispatchRegistryBuilder & registry) { - registry.add({ - "common.repeat_get_rows.scalar_3d_f32", - GGML_OP_REPEAT, - DispatchMatchKind::Fused, - 300, - DispatchSource::Common, - match_repeat_get_rows_scalar_3d_f32_dispatch, - }); registry.add({ "common.repeat.scalar_3d_f32", GGML_OP_REPEAT, diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat-id.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat-id.cpp index 629742b25c70..898a04e16592 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat-id.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat-id.cpp @@ -18,6 +18,10 @@ static constexpr KernelCatalogRef kMulMatIdTiledF32F32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_id_tiled_input_f32_publish_f32"); static constexpr KernelCatalogRef kMulMatIdSkinnyF32F32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_id_skinny_input_f32_publish_f32"); +static constexpr KernelCatalogRef kMulMatIdTiledScaledF32F32Kernel = + GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_id_tiled_input_f32_scaled_publish_f32"); +static constexpr KernelCatalogRef kMulMatIdSkinnyScaledF32F32Kernel = + GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_id_skinny_input_f32_scaled_publish_f32"); static constexpr KernelCatalogRef kMulMatIdTiledPostOpsF32F32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_id_tiled_input_f32_postops_publish_f32"); static constexpr KernelCatalogRef kMulMatIdSkinnyPostOpsF32F32Kernel = @@ -62,6 +66,21 @@ struct MulMatIdPostOpsMatch { } }; +struct MulMatIdExpertScaleMatch { + CommonMulMatIdMatch root; + const Value * scale = nullptr; + const Value * output = nullptr; + const GraphNode * repeat = nullptr; + const GraphNode * gather = nullptr; + const GraphNode * multiply = nullptr; + int64_t route_stride = 0; + + bool matched() const { + return root.matched() && scale != nullptr && output != nullptr && repeat != nullptr && gather != nullptr && + multiply != nullptr; + } +}; + static bool is_bias_shape(const Value & value, int64_t output_size) { if (value.type != GGML_TYPE_F32 || !value.contiguous || value.ne[0] != output_size) { return false; @@ -236,6 +255,160 @@ static bool match_mul_mat_id_dispatch(const DispatchMatchContext & context, Disp return true; } +static MulMatIdExpertScaleMatch match_mul_mat_id_expert_scale(const DispatchMatchContext & context) { + MulMatIdExpertScaleMatch match; + const CommonMulMatIdMatch root = + common_match_mul_mat_id_any_format(context.graph, context.root_node, context.plan); + if (!root.matched()) { + return match; + } + + const bool has_gemma_experts = root.expert_count == 128; + const bool has_eight_routes = root.route_count == 8; + const bool has_gemma_routing = has_gemma_experts && has_eight_routes; + const bool has_supported_format = root.weight_format == CommonMulMatWeightFormat::Q4K || + root.weight_format == CommonMulMatWeightFormat::Q6K || + root.weight_format == CommonMulMatWeightFormat::Q5_1 || + root.weight_format == CommonMulMatWeightFormat::Q8_0; + const bool has_transient_matmul_output = root.output->kind == ValueKind::Transient; + const bool has_eligible_root = has_gemma_routing && has_supported_format && has_transient_matmul_output; + if (!has_eligible_root) { + return {}; + } + + const GraphNode * multiply = + common_mul_mat_id_find_only_consumer_with_op(context.graph, root.output->id, GGML_OP_MUL); + const bool has_multiply = multiply != nullptr && common_mul_mat_id_binary_node_is_mul(*multiply) && + multiply->inputs[0] == root.output->id; + if (!has_multiply) { + return {}; + } + + const Value * selected = common_mul_mat_id_graph_value(context.graph, multiply->inputs[1]); + const Value * output = common_mul_mat_id_graph_value(context.graph, multiply->output); + const bool has_values = selected != nullptr && output != nullptr; + if (!has_values) { + return {}; + } + const GraphNode * gather = context.graph.index().producer(selected->id); + const bool has_gather = gather != nullptr && gather->op == GGML_OP_GET_ROWS && gather->inputs.size() == 2; + const bool has_selected_shape = selected->type == GGML_TYPE_F32 && + common_mul_mat_id_is_shape(*selected, 1, 8, root.token_count, 1); + const bool has_output_shape = output->type == GGML_TYPE_F32 && output->contiguous && + common_mul_mat_id_same_shape(*output, *root.output); + const bool has_selected_consumer = + context.graph.index().consumers(selected->id).size() == 1 && + context.graph.index().consumers(selected->id).front() == multiply; + const bool has_scale_gather = has_gather && has_selected_shape && has_output_shape && has_selected_consumer; + if (!has_scale_gather) { + return {}; + } + const bool uses_same_ids = gather->inputs[1] == root.route_ids->id; + if (!uses_same_ids) { + return {}; + } + + const Value * repeated = common_mul_mat_id_graph_value(context.graph, gather->inputs[0]); + const GraphNode * repeat = repeated == nullptr ? nullptr : context.graph.index().producer(repeated->id); + const bool has_repeat = repeat != nullptr && repeat->op == GGML_OP_REPEAT && repeat->inputs.size() == 1; + const bool has_repeat_shape = repeated != nullptr && repeated->type == GGML_TYPE_F32 && + common_mul_mat_id_is_shape(*repeated, 1, 128, root.token_count, 1) && + repeated->kind == ValueKind::Transient; + const bool has_repeat_consumer = repeated != nullptr && + context.graph.index().consumers(repeated->id).size() == 1 && + context.graph.index().consumers(repeated->id).front() == gather; + const bool has_scale_repeat = has_repeat && has_repeat_shape && has_repeat_consumer; + if (!has_scale_repeat) { + return {}; + } + + const Value * scale = common_mul_mat_id_graph_value(context.graph, repeat->inputs[0]); + const bool has_scale_shape = scale != nullptr && scale->type == GGML_TYPE_F32 && scale->contiguous && + common_mul_mat_id_is_shape(*scale, 1, 128, 1, 1) && + scale->byte_count == 128 * sizeof(float); + const bool has_contiguous_repeat = repeated->contiguous; + const bool has_scale_source = has_scale_shape && has_contiguous_repeat; + if (!has_scale_source) { + return {}; + } + + const bool has_id_columns = root.route_ids->nb[0] == sizeof(int32_t) && + root.route_ids->nb[1] % sizeof(int32_t) == 0; + if (!has_id_columns) { + return {}; + } + const int64_t route_stride = static_cast(root.route_ids->nb[1] / sizeof(int32_t)); + const bool has_supported_stride = route_stride >= 8 && route_stride <= 512; + const size_t expected_id_bytes = (static_cast(root.token_count) - 1) * root.route_ids->nb[1] + + 8 * sizeof(int32_t); + const bool has_exact_id_span = root.route_ids->byte_count == expected_id_bytes; + const bool has_supported_ids = has_supported_stride && has_exact_id_span; + if (!has_supported_ids) { + return {}; + } + + match.root = root; + match.scale = scale; + match.output = output; + match.repeat = repeat; + match.gather = gather; + match.multiply = multiply; + match.route_stride = route_stride; + return match; +} + +static bool match_mul_mat_id_expert_scale_dispatch(const DispatchMatchContext & context, + DispatchMatch & dispatch_match) { + const MulMatIdExpertScaleMatch match = match_mul_mat_id_expert_scale(context); + if (!match.matched()) { + return false; + } + + CommandPlanMoeRoutingBundle routing_bundle; + if (!common_mul_mat_id_ensure_moe_routing_bundle(context, match.root, dispatch_match, routing_bundle)) { + return false; + } + + Dispatch dispatch; + const bool skinny_route = common_mul_mat_id_is_skinny_route(match.root); + dispatch.kernel = make_kernel_specialization(skinny_route ? kMulMatIdSkinnyScaledF32F32Kernel : + kMulMatIdTiledScaledF32F32Kernel); + dispatch.kernel.integer_parameters.emplace("token_count", match.root.token_count); + dispatch.kernel.compile_parameters.emplace("ggml.workload.token_capacity", + common_mul_mat_id_to_config_value(match.root.token_count)); + dispatch.kernel.compile_parameters.emplace("ggml.mul_mat_id.input_size", + common_mul_mat_id_to_config_value(match.root.input_size)); + dispatch.kernel.compile_parameters.emplace("ggml.mul_mat_id.output_size", + common_mul_mat_id_to_config_value(match.root.output_size)); + dispatch.kernel.compile_parameters.emplace("ggml.mul_mat_id.expert_count", + common_mul_mat_id_to_config_value(match.root.expert_count)); + dispatch.kernel.compile_parameters.emplace("ggml.mul_mat_id.route_count", + common_mul_mat_id_to_config_value(match.root.route_count)); + dispatch.kernel.compile_parameters.emplace("ggml.mul_mat_id.input_route_count", + common_mul_mat_id_to_config_value(match.root.input_route_count)); + dispatch.kernel.compile_parameters.emplace( + "ggml.mul_mat_id.weight_format", + common_mul_mat_id_to_config_value(common_mul_mat_format_config_value(match.root.weight_format))); + dispatch.kernel.compile_parameters.emplace("ggml.mul_mat_id.scale.route_stride", + common_mul_mat_id_to_config_value(match.route_stride)); + dispatch.bindings.push_back({ match.root.input->id, 0, match.root.input->byte_count }); + dispatch.bindings.push_back({ routing_bundle.expert_table, 0, routing_bundle.expert_table_byte_count }); + dispatch.bindings.push_back({ routing_bundle.partition_table, 0, routing_bundle.partition_table_byte_count }); + dispatch.bindings.push_back({ match.root.weight->id, 0, match.root.weight->byte_count }); + dispatch.bindings.push_back({ match.scale->id, 0, match.scale->byte_count }); + dispatch.bindings.push_back({ match.root.route_ids->id, 0, match.root.route_ids->byte_count }); + dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count }); + + for (const GraphNode * covered_node : { context.root_node, match.repeat, match.gather, match.multiply }) { + if (!append_covered_node_index_once(context.graph, context.covered_nodes, covered_node, + dispatch_match.covered_nodes)) { + return false; + } + } + dispatch_match.dispatches.push_back(std::move(dispatch)); + return true; +} + static bool match_mul_mat_id_postops_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) { const MulMatIdPostOpsMatch match = match_mul_mat_id_postops(context); if (!match.matched()) { @@ -331,6 +504,14 @@ static bool match_mul_mat_id_postops_dispatch(const DispatchMatchContext & conte } void register_mul_mat_id_dispatches(DispatchRegistryBuilder & registry) { + registry.add({ + "common.mul_mat_id.expert_scale.f32_f32_wmma", + GGML_OP_MUL_MAT_ID, + DispatchMatchKind::Fused, + 190, + DispatchSource::Common, + match_mul_mat_id_expert_scale_dispatch, + }); registry.add({ "common.mul_mat_id_postops.f32_f32_wmma", GGML_OP_MUL_MAT_ID, diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json index 134cecc4297e..6f1e49137a5a 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json @@ -164,9 +164,15 @@ { "path": "ops/matmul_id/tiled_input_f32_publish_f32.loom" }, + { + "path": "ops/matmul_id/tiled_input_f32_scaled_publish_f32.loom" + }, { "path": "ops/matmul_id/skinny_input_f32_publish_f32.loom" }, + { + "path": "ops/matmul_id/skinny_input_f32_scaled_publish_f32.loom" + }, { "path": "ops/matmul_id/tiled_input_f32_postops_publish_f32.loom" }, @@ -3620,6 +3626,71 @@ "motifs/f32_f16.loom" ] }, + { + "name": "ggml_mul_mat_id_tiled_input_f32_scaled_publish_f32", + "family": "loom_libs", + "symbol": "ggml_mul_mat_id_tiled_input_f32_scaled_publish_f32", + "source": "ops/matmul_id/tiled_input_f32_scaled_publish_f32.loom", + "workload_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "launch_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "bindings": [ + "input", + "expert_table", + "partition_table", + "weight", + "scales", + "route_ids", + "output" + ], + "binding_access": [ + "read", + "read", + "read", + "read", + "read", + "read", + "read_write" + ], + "target_selector": "", + "compile_recipe": { + "mode": "direct", + "primary_sources": [ + "ops/matmul_id/tiled_input_f32_scaled_publish_f32.loom" + ], + "library_sources": [ + "motifs/matmul_id/tiled_input_f32_dequant_accumulate_f32.loom", + "motifs/dequant.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ] + }, + "compile_dependencies": [ + "motifs/matmul_id/tiled_input_f32_dequant_accumulate_f32.loom", + "motifs/dequant.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ] + }, { "name": "ggml_mul_mat_id_skinny_input_f32_publish_f32", "family": "loom_libs", @@ -3681,6 +3752,71 @@ "motifs/f32_f16.loom" ] }, + { + "name": "ggml_mul_mat_id_skinny_input_f32_scaled_publish_f32", + "family": "loom_libs", + "symbol": "ggml_mul_mat_id_skinny_input_f32_scaled_publish_f32", + "source": "ops/matmul_id/skinny_input_f32_scaled_publish_f32.loom", + "workload_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "launch_parameters": [ + { + "name": "token_count", + "type": "index" + } + ], + "bindings": [ + "input", + "expert_table", + "partition_table", + "weight", + "scales", + "route_ids", + "output" + ], + "binding_access": [ + "read", + "read", + "read", + "read", + "read", + "read", + "read_write" + ], + "target_selector": "", + "compile_recipe": { + "mode": "direct", + "primary_sources": [ + "ops/matmul_id/skinny_input_f32_scaled_publish_f32.loom" + ], + "library_sources": [ + "motifs/matmul_id/skinny_input_f32_dequant_accumulate_f32.loom", + "motifs/dequant.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ] + }, + "compile_dependencies": [ + "motifs/matmul_id/skinny_input_f32_dequant_accumulate_f32.loom", + "motifs/dequant.loom", + "motifs/q4_k_f16.loom", + "motifs/q6_k_f16.loom", + "motifs/q8_0_f16.loom", + "motifs/q8_1_f16.loom", + "motifs/f16_f16.loom", + "motifs/bf16_f16.loom", + "motifs/f32_f16.loom" + ] + }, { "name": "ggml_mul_mat_id_tiled_input_f32_postops_publish_f32", "family": "loom_libs", @@ -6679,10 +6815,6 @@ { "name": "route_stride", "type": "index" - }, - { - "name": "scale_token_stride", - "type": "index" } ], "launch_parameters": [ @@ -6693,10 +6825,6 @@ { "name": "route_stride", "type": "index" - }, - { - "name": "scale_token_stride", - "type": "index" } ], "bindings": [ diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_scaled_publish_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_scaled_publish_f32.loom new file mode 100644 index 000000000000..dadd41d06c80 --- /dev/null +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/skinny_input_f32_scaled_publish_f32.loom @@ -0,0 +1,113 @@ +// Routed MUL_MAT_ID with an F32 per-expert scale applied before publication. +// +// The kernel consumes a precomputed expert table. Each table row stores compact +// assignment indices encoded as token * route_count + route. Input can either +// be broadcast across all routes or contain one input row per route. +template.decl @ggml.matmul_id.skinny_f32_f32.core(%arg0: index, %arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index, %arg6: index, %arg7: f32, %arg8: index, %arg9: index, %arg10: buffer, %arg11: buffer, %arg12: buffer, %arg13: buffer, %arg14: buffer, %arg15: buffer, %arg16: buffer, %arg17: buffer, %arg18: buffer, %arg19: buffer, %arg20: buffer) + +template.decl @ggml.matmul_id.skinny_f32_f32.finish_tile(%token_count: index, %route_count: index, %output_size: index, %expert_count: index, %output_tile_count: index, %maximum_partition_count: index, %epsilon: f32, %channel_tile: index, %descriptor_ordinal: index, %expert: index, %route_tile_base: index, %partition_row_count: index, %expert_table: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer) + +template.decl @ggml.matmul_id.skinny_f32_f32.publish_vector4(%publish_word: i1, %assignment: index, %token: index, %route: index, %channel: index, %token_count0: index, %route_count0: index, %output_size0: index, %values: vector<4xf32>, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer) + +amdgpu.target @ggml_mul_mat_id_skinny_input_f32_scaled_publish_f32_gfx11_wave64 {subgroup_size = 64} + +config.decl @ggml.mul_mat_id.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 32)] + +config.decl @ggml.mul_mat_id.output_size : %value: index where [range(%value, 1, 262144)] + +config.decl @ggml.mul_mat_id.expert_count : %value: index where [range(%value, 1, 512)] + +config.decl @ggml.mul_mat_id.route_count : %value: index where [range(%value, 1, 32)] + +config.decl @ggml.mul_mat_id.input_route_count : %value: index where [range(%value, 1, 32)] + +config.decl @ggml.mul_mat_id.weight_format : %value: index where [range(%value, 4, 81)] + +config.decl @ggml.mul_mat_id.scale.route_stride : %value: index where [range(%value, 8, 512)] + +config.decl @ggml.workload.token_capacity : %value: index where [range(%value, 1, 2048)] + +template.def<@ggml.matmul_id.skinny_f32_f32.publish_vector4> device @ggml_mul_mat_id_skinny_f32_f32_scaled_publish_vector4(%publish_word: i1, %assignment: index, %token: index, %route: index, %channel: index, %token_count0: index, %route_count0: index, %output_size0: index, %values: vector<4xf32>, %scales: buffer, %output: buffer, %route_ids: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer) { + %token_count = index.assume %token_count0 [range(%token_count0, 1, 2048)] : index + %route_count = index.assume %route_count0 [range(%route_count0, 1, 32)] : index + %output_size = index.assume %output_size0 [range(%output_size0, 1, 262144)] : index + %route_stride0 = config.get @ggml.mul_mat_id.scale.route_stride : index + %route_stride = index.assume %route_stride0 [range(%route_stride0, 8, 512)] : index + %c0_offset = index.constant 0 : offset + %c1 = index.constant 1 : index + %c8 = index.constant 8 : index + %c0_i32 = scalar.constant 0 : i32 + %c128_i32 = scalar.constant 128 : i32 + %c0_f32 = scalar.constant 0.0 : f32 + %last_token = index.sub %token_count, %c1 : index + %last_row_base = index.mul %last_token, %route_stride : index + %route_id_span = index.add %last_row_base, %c8 : index + %output_row_count = index.mul %token_count, %route_count : index + %ids_view = buffer.view %route_ids[%c0_offset] : buffer -> view<[%route_id_span]xi32> + %scales_view = buffer.view %scales[%c0_offset] : buffer -> view<128xf32> + %output_noalias = buffer.assume.noalias %output : buffer + %output_view = buffer.view %output_noalias[%c0_offset] : buffer -> view<[%output_row_count]x[%output_size]xf32> + scf.if %publish_word { + %id_row = index.mul %token, %route_stride : index + %id_index0 = index.add %id_row, %route : index + %id_index = index.assume %id_index0 [lt(%id_index0, %route_id_span)] : index + %expert_i32 = view.load %ids_view[%id_index] : view<[%route_id_span]xi32> -> i32 + %nonnegative = scalar.cmpi sge, %expert_i32, %c0_i32 : i32 + %below_limit = scalar.cmpi slt, %expert_i32, %c128_i32 : i32 + %valid_expert = scalar.andi %nonnegative, %below_limit : i1 + %scale = scf.if %valid_expert -> (f32) { + %expert0 = index.cast %expert_i32 : i32 to index + %expert = index.assume %expert0 [range(%expert0, 0, 127)] : index + %value = view.load %scales_view[%expert] : view<128xf32> -> f32 + scf.yield %value : f32 + } else { + scf.yield %c0_f32 : f32 + } + %scale_vector = vector.splat %scale : vector<4xf32> + %scaled = vector.mulf %values, %scale_vector : vector<4xf32> + %bounded_assignment, %assignment_count = index.assume %assignment, %output_row_count [lt(%assignment, %output_row_count)] : index, index + %mask = vector.mask.range [%channel to %output_size step %c1] : index -> vector<4xi1> + vector.store.mask %scaled, %output_view[%bounded_assignment, %channel], %mask : vector<4xf32>, view<[%output_row_count]x[%output_size]xf32>, vector<4xi1> + } + template.return +} + +template.def<@ggml.matmul_id.skinny_f32_f32.finish_tile> device @ggml_mul_mat_id_skinny_f32_f32_finish_tile(%token_count: index, %route_count: index, %output_size: index, %expert_count: index, %output_tile_count: index, %maximum_partition_count: index, %epsilon: f32, %channel_tile: index, %descriptor_ordinal: index, %expert: index, %route_tile_base: index, %partition_row_count: index, %expert_table: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer) { + template.return +} + +kernel.def target(@ggml_mul_mat_id_skinny_input_f32_scaled_publish_f32_gfx11_wave64) @ggml_mul_mat_id_skinny_input_f32_scaled_publish_f32(%token_count: index) { + %expert_count = config.get @ggml.mul_mat_id.expert_count : index + %route_count = config.get @ggml.mul_mat_id.route_count : index + %output_size = config.get @ggml.mul_mat_id.output_size : index + %c1 = index.constant 1 : index + %c31 = index.constant 31 : index + %c32 = index.constant 32 : index + %c63 = index.constant 63 : index + %c64 = index.constant 64 : index + %c128 = index.constant 128 : index + %padded_output_size = index.add %output_size, %c63 : index + %output_tiles = index.div %padded_output_size, %c64 : index + %assignment_count = index.mul %token_count, %route_count : index + %rounded_assignment_count = index.add %assignment_count, %c31 : index + %assignment_partition_count = index.div %rounded_assignment_count, %c32 : index + %has_more_assignment_partitions = index.cmp ugt, %assignment_partition_count, %expert_count : index + %launch_partition_count = scf.select %has_more_assignment_partitions, %assignment_partition_count, %expert_count : index + kernel.launch.config workgroups(%output_tiles, %launch_partition_count, %c1) workgroup_size(%c128, %c1, %c1) : index +} launch(%token_count: index, %input: buffer, %expert_table: buffer, %partition_table: buffer, %weight: buffer, %scales: buffer, %route_ids: buffer, %output: buffer) where [range(%token_count, 1, 2048)] { + %input_size = config.get @ggml.mul_mat_id.input_size : index + %route_count = config.get @ggml.mul_mat_id.route_count : index + %input_route_count = config.get @ggml.mul_mat_id.input_route_count : index + %expert_count = config.get @ggml.mul_mat_id.expert_count : index + %output_size = config.get @ggml.mul_mat_id.output_size : index + %weight_format = config.get @ggml.mul_mat_id.weight_format : index + %token_capacity = config.get @ggml.workload.token_capacity : index + %c0_f32 = scalar.constant 0.0 : f32 + %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048), le(%token_count, %token_capacity)] : index + %channel_tile = kernel.workgroup.id : index + %partition_ordinal = kernel.workgroup.id : index + // The core forwards its bias and residual slots to publish_vector4. + // Here those slots carry scales and route IDs; other post-op slots are unused. + template.apply<@ggml.matmul_id.skinny_f32_f32.core>(%weight_format, %bounded_token_count, %input_size, %output_size, %route_count, %input_route_count, %expert_count, %c0_f32, %channel_tile, %partition_ordinal, %input, %expert_table, %partition_table, %weight, %scales, %output, %route_ids, %output, %output, %output, %output) : (index, index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) + kernel.return +} diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_scaled_publish_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_scaled_publish_f32.loom new file mode 100644 index 000000000000..e41a885462b2 --- /dev/null +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/matmul_id/tiled_input_f32_scaled_publish_f32.loom @@ -0,0 +1,113 @@ +// Routed MUL_MAT_ID with an F32 per-expert scale applied before publication. +// +// The kernel consumes a precomputed expert table. Each table row stores compact +// assignment indices encoded as token * route_count + route. Input can either +// be broadcast across all routes or contain one input row per route. +template.decl @ggml.matmul_id.tiled_f32_f32.core(%arg0: index, %arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index, %arg6: index, %arg7: f32, %arg8: index, %arg9: index, %arg10: buffer, %arg11: buffer, %arg12: buffer, %arg13: buffer, %arg14: buffer, %arg15: buffer, %arg16: buffer, %arg17: buffer, %arg18: buffer, %arg19: buffer, %arg20: buffer) + +template.decl @ggml.matmul_id.tiled_f32_f32.finish_tile(%token_count: index, %route_count: index, %output_size: index, %expert_count: index, %output_tile_count: index, %maximum_partition_count: index, %epsilon: f32, %channel_tile: index, %descriptor_ordinal: index, %expert: index, %route_tile_base: index, %partition_row_count: index, %expert_table: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer) + +template.decl @ggml.matmul_id.tiled_f32_f32.publish_vector4(%publish_word: i1, %assignment: index, %token: index, %route: index, %channel: index, %token_count0: index, %route_count0: index, %output_size0: index, %values: vector<4xf32>, %bias: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer) + +amdgpu.target @ggml_mul_mat_id_tiled_input_f32_scaled_publish_f32_gfx11_wave64 {subgroup_size = 64} + +config.decl @ggml.mul_mat_id.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 32)] + +config.decl @ggml.mul_mat_id.output_size : %value: index where [range(%value, 1, 262144)] + +config.decl @ggml.mul_mat_id.expert_count : %value: index where [range(%value, 1, 512)] + +config.decl @ggml.mul_mat_id.route_count : %value: index where [range(%value, 1, 32)] + +config.decl @ggml.mul_mat_id.input_route_count : %value: index where [range(%value, 1, 32)] + +config.decl @ggml.mul_mat_id.weight_format : %value: index where [range(%value, 4, 81)] + +config.decl @ggml.mul_mat_id.scale.route_stride : %value: index where [range(%value, 8, 512)] + +config.decl @ggml.workload.token_capacity : %value: index where [range(%value, 1, 2048)] + +template.def<@ggml.matmul_id.tiled_f32_f32.publish_vector4> device @ggml_mul_mat_id_tiled_f32_f32_scaled_publish_vector4(%publish_word: i1, %assignment: index, %token: index, %route: index, %channel: index, %token_count0: index, %route_count0: index, %output_size0: index, %values: vector<4xf32>, %scales: buffer, %output: buffer, %route_ids: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer) { + %token_count = index.assume %token_count0 [range(%token_count0, 1, 2048)] : index + %route_count = index.assume %route_count0 [range(%route_count0, 1, 32)] : index + %output_size = index.assume %output_size0 [range(%output_size0, 1, 262144)] : index + %route_stride0 = config.get @ggml.mul_mat_id.scale.route_stride : index + %route_stride = index.assume %route_stride0 [range(%route_stride0, 8, 512)] : index + %c0_offset = index.constant 0 : offset + %c1 = index.constant 1 : index + %c8 = index.constant 8 : index + %c0_i32 = scalar.constant 0 : i32 + %c128_i32 = scalar.constant 128 : i32 + %c0_f32 = scalar.constant 0.0 : f32 + %last_token = index.sub %token_count, %c1 : index + %last_row_base = index.mul %last_token, %route_stride : index + %route_id_span = index.add %last_row_base, %c8 : index + %output_row_count = index.mul %token_count, %route_count : index + %ids_view = buffer.view %route_ids[%c0_offset] : buffer -> view<[%route_id_span]xi32> + %scales_view = buffer.view %scales[%c0_offset] : buffer -> view<128xf32> + %output_noalias = buffer.assume.noalias %output : buffer + %output_view = buffer.view %output_noalias[%c0_offset] : buffer -> view<[%output_row_count]x[%output_size]xf32> + scf.if %publish_word { + %id_row = index.mul %token, %route_stride : index + %id_index0 = index.add %id_row, %route : index + %id_index = index.assume %id_index0 [lt(%id_index0, %route_id_span)] : index + %expert_i32 = view.load %ids_view[%id_index] : view<[%route_id_span]xi32> -> i32 + %nonnegative = scalar.cmpi sge, %expert_i32, %c0_i32 : i32 + %below_limit = scalar.cmpi slt, %expert_i32, %c128_i32 : i32 + %valid_expert = scalar.andi %nonnegative, %below_limit : i1 + %scale = scf.if %valid_expert -> (f32) { + %expert0 = index.cast %expert_i32 : i32 to index + %expert = index.assume %expert0 [range(%expert0, 0, 127)] : index + %value = view.load %scales_view[%expert] : view<128xf32> -> f32 + scf.yield %value : f32 + } else { + scf.yield %c0_f32 : f32 + } + %scale_vector = vector.splat %scale : vector<4xf32> + %scaled = vector.mulf %values, %scale_vector : vector<4xf32> + %bounded_assignment, %assignment_count = index.assume %assignment, %output_row_count [lt(%assignment, %output_row_count)] : index, index + %mask = vector.mask.range [%channel to %output_size step %c1] : index -> vector<4xi1> + vector.store.mask %scaled, %output_view[%bounded_assignment, %channel], %mask : vector<4xf32>, view<[%output_row_count]x[%output_size]xf32>, vector<4xi1> + } + template.return +} + +template.def<@ggml.matmul_id.tiled_f32_f32.finish_tile> device @ggml_mul_mat_id_tiled_f32_f32_finish_tile(%token_count: index, %route_count: index, %output_size: index, %expert_count: index, %output_tile_count: index, %maximum_partition_count: index, %epsilon: f32, %channel_tile: index, %descriptor_ordinal: index, %expert: index, %route_tile_base: index, %partition_row_count: index, %expert_table: buffer, %output: buffer, %residual_input: buffer, %residual_output: buffer, %norm_weight: buffer, %normalized_output: buffer, %completion_counters: buffer) { + template.return +} + +kernel.def target(@ggml_mul_mat_id_tiled_input_f32_scaled_publish_f32_gfx11_wave64) @ggml_mul_mat_id_tiled_input_f32_scaled_publish_f32(%token_count: index) { + %expert_count = config.get @ggml.mul_mat_id.expert_count : index + %route_count = config.get @ggml.mul_mat_id.route_count : index + %output_size = config.get @ggml.mul_mat_id.output_size : index + %c1 = index.constant 1 : index + %c31 = index.constant 31 : index + %c32 = index.constant 32 : index + %c63 = index.constant 63 : index + %c64 = index.constant 64 : index + %c128 = index.constant 128 : index + %padded_output_size = index.add %output_size, %c63 : index + %output_tiles = index.div %padded_output_size, %c64 : index + %assignment_count = index.mul %token_count, %route_count : index + %rounded_assignment_count = index.add %assignment_count, %c31 : index + %assignment_partition_count = index.div %rounded_assignment_count, %c32 : index + %has_more_assignment_partitions = index.cmp ugt, %assignment_partition_count, %expert_count : index + %launch_partition_count = scf.select %has_more_assignment_partitions, %assignment_partition_count, %expert_count : index + kernel.launch.config workgroups(%output_tiles, %launch_partition_count, %c1) workgroup_size(%c128, %c1, %c1) : index +} launch(%token_count: index, %input: buffer, %expert_table: buffer, %partition_table: buffer, %weight: buffer, %scales: buffer, %route_ids: buffer, %output: buffer) where [range(%token_count, 1, 2048)] { + %input_size = config.get @ggml.mul_mat_id.input_size : index + %route_count = config.get @ggml.mul_mat_id.route_count : index + %input_route_count = config.get @ggml.mul_mat_id.input_route_count : index + %expert_count = config.get @ggml.mul_mat_id.expert_count : index + %output_size = config.get @ggml.mul_mat_id.output_size : index + %weight_format = config.get @ggml.mul_mat_id.weight_format : index + %token_capacity = config.get @ggml.workload.token_capacity : index + %c0_f32 = scalar.constant 0.0 : f32 + %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048), le(%token_count, %token_capacity)] : index + %channel_tile = kernel.workgroup.id : index + %partition_ordinal = kernel.workgroup.id : index + // The core forwards its bias and residual slots to publish_vector4. + // Here those slots carry scales and route IDs; other post-op slots are unused. + template.apply<@ggml.matmul_id.tiled_f32_f32.core>(%weight_format, %bounded_token_count, %input_size, %output_size, %route_count, %input_route_count, %expert_count, %c0_f32, %channel_tile, %partition_ordinal, %input, %expert_table, %partition_table, %weight, %scales, %output, %route_ids, %output, %output, %output, %output) : (index, index, index, index, index, index, index, f32, index, index, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer, buffer) + kernel.return +} diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rows/get_rows_scalar_3d_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rows/get_rows_scalar_3d_f32.loom index 068d566b3586..a9b1e4bf0027 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rows/get_rows_scalar_3d_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rows/get_rows_scalar_3d_f32.loom @@ -5,14 +5,13 @@ // narrow view of wider argsort rows; only the selected IDs are in bounds. amdgpu.target @ggml_get_rows_scalar_3d_gfx11_wave64 {subgroup_size = 64} -kernel.def target(@ggml_get_rows_scalar_3d_gfx11_wave64) @ggml_get_rows_scalar_3d_f32(%token_count: index, %route_stride: index, %scale_token_stride: index) { +kernel.def target(@ggml_get_rows_scalar_3d_gfx11_wave64) @ggml_get_rows_scalar_3d_f32(%token_count: index, %route_stride: index) { %c1 = index.constant 1 : index %c64 = index.constant 64 : index kernel.launch.config workgroups(%token_count, %c1, %c1) workgroup_size(%c64, %c1, %c1) : index -} launch(%token_count: index, %route_stride: index, %scale_token_stride: index, %route_ids: buffer, %scales: buffer, %output: buffer) where [range(%token_count, 1, 2048), range(%route_stride, 8, 512), range(%scale_token_stride, 0, 128)] { +} launch(%token_count: index, %route_stride: index, %route_ids: buffer, %scales: buffer, %output: buffer) where [range(%token_count, 1, 2048), range(%route_stride, 8, 512)] { %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048)] : index %bounded_route_stride = index.assume %route_stride [range(%route_stride, 8, 512)] : index - %bounded_scale_token_stride = index.assume %scale_token_stride [range(%scale_token_stride, 0, 128)] : index %token0 = kernel.workgroup.id : index %route0 = kernel.workitem.id : index %c1 = index.constant 1 : index @@ -29,7 +28,7 @@ kernel.def target(@ggml_get_rows_scalar_3d_gfx11_wave64) @ggml_get_rows_scalar_3 %last_row_base = index.mul %last_token, %bounded_route_stride : index %route_id_span = index.add %last_row_base, %c8 : index %ids_view = buffer.view %route_ids[%c0_offset] : buffer -> view<[%route_id_span]xi32> - %last_scale_row = index.mul %last_token, %bounded_scale_token_stride : index + %last_scale_row = index.mul %last_token, %c128 : index %scale_span = index.add %last_scale_row, %c128 : index %scales_view = buffer.view %scales[%c0_offset] : buffer -> view<[%scale_span]xf32> %output_view = buffer.view %output[%c0_offset] : buffer -> view<[%bounded_token_count]x8xf32> @@ -47,7 +46,7 @@ kernel.def target(@ggml_get_rows_scalar_3d_gfx11_wave64) @ggml_get_rows_scalar_3 %scale = scf.if %valid_expert -> (f32) { %expert0 = index.cast %expert_i32 : i32 to index %expert = index.assume %expert0 [range(%expert0, 0, 127)] : index - %scale_row = index.mul %token, %bounded_scale_token_stride : index + %scale_row = index.mul %token, %c128 : index %scale_index = index.add %scale_row, %expert : index %bounded_scale_index = index.assume %scale_index [lt(%scale_index, %scale_span)] : index %selected_scale = view.load %scales_view[%bounded_scale_index] : view<[%scale_span]xf32> -> f32 diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index e0fafcfe1b6a..5d4dd3e2ac34 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -4505,7 +4505,7 @@ static void run_scalar_3d_get_rows_dispatch_checks() { check_case(128, 8, 2, 513, false); } -static void run_repeat_get_rows_scalar_3d_dispatch_checks() { +static void run_standalone_repeat_get_rows_scalar_3d_dispatch_checks() { auto check_case = [](int64_t token_count, int64_t route_stride, bool expose_repeat, bool ids_ready_before_repeat = true) { ggml_init_params params = {}; @@ -4539,15 +4539,11 @@ static void run_repeat_get_rows_scalar_3d_dispatch_checks() { ggml::hrx::DispatchScheduler scheduler; REQUIRE(scheduler.schedule_graph(imported.graph, test_dispatch_target())); const auto & dispatches = scheduler.plan().dispatches; - const bool expect_fusion = !expose_repeat && ids_ready_before_repeat; - REQUIRE(dispatches.size() == (expect_fusion ? 1 : 2)); - if (!expect_fusion) { - REQUIRE(kernel_name_for_id(dispatches[0].kernel.kernel_id) == "loom_libs:ggml_repeat_scalar_3d_f32"); - } + REQUIRE(dispatches.size() == 2); + REQUIRE(kernel_name_for_id(dispatches[0].kernel.kernel_id) == "loom_libs:ggml_repeat_scalar_3d_f32"); const ggml::hrx::Dispatch & gather = dispatches.back(); REQUIRE(kernel_name_for_id(gather.kernel.kernel_id) == "loom_libs:ggml_get_rows_scalar_3d_f32"); - REQUIRE(gather.kernel.integer_parameters.at("scale_token_stride") == (expect_fusion ? 0 : 128)); - REQUIRE(gather.bindings[1].length == (expect_fusion ? ggml_nbytes(scales) : ggml_nbytes(repeated))); + REQUIRE(gather.bindings[1].length == ggml_nbytes(repeated)); ggml_free(ctx); }; @@ -14978,7 +14974,9 @@ static void register_hrx_backend_host_cases(test_runner::Suite & suite) { suite.host_case("graph_traversal", [] { run_graph_traversal_checks(); }); suite.host_case("qwen_token_embedding_dispatch", [] { run_qwen_token_embedding_dispatch_checks(); }); suite.host_case("scalar_3d_get_rows_dispatch", [] { run_scalar_3d_get_rows_dispatch_checks(); }); - suite.host_case("repeat_get_rows_scalar_3d_dispatch", [] { run_repeat_get_rows_scalar_3d_dispatch_checks(); }); + suite.host_case("standalone_repeat_get_rows_scalar_3d_dispatch", [] { + run_standalone_repeat_get_rows_scalar_3d_dispatch_checks(); + }); suite.host_case("get_rows_rmsnorm_binary_dispatch", [] { run_get_rows_rmsnorm_binary_dispatch_checks(); }); suite.host_case("get_rows_scale_dispatch", [] { run_get_rows_scale_dispatch_checks(); }); suite.host_case("gather_add_dispatch", [] { run_gather_add_dispatch_checks(); }); diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index 9db56a021b93..f2c57ad90cb1 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -4506,6 +4506,116 @@ static void run_q4_prefill_source_eligibility_case() { ggml_backend_free(backend); } +static void run_mul_mat_id_expert_scale_case(ggml_type weight_type, int64_t token_count, int64_t route_stride, + bool include_invalid_ids = false, bool expose_matmul = false) { + constexpr int64_t input_size = 256; + constexpr int64_t output_size = 64; + constexpr int64_t expert_count = 128; + constexpr int64_t route_count = 8; + ggml_backend_t backend = ggml_backend_hrx_init(0); + REQUIRE(backend != nullptr); + + ggml_init_params params = {}; + params.mem_size = 4 * 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + + ggml_tensor * weight = ggml_new_tensor_3d(ctx, weight_type, input_size, output_size, expert_count); + ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, input_size, 1, token_count); + ggml_tensor * ids_storage = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, route_stride, token_count); + ggml_tensor * ids = route_stride == route_count ? ids_storage : + ggml_view_2d(ctx, ids_storage, route_count, token_count, ids_storage->nb[1], 0); + ggml_tensor * scales = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, expert_count); + ggml_tensor * matmul = ggml_mul_mat_id(ctx, weight, input, ids); + ggml_tensor * scale_rows = ggml_reshape_3d(ctx, scales, 1, expert_count, 1); + ggml_tensor * repeated = ggml_repeat_4d(ctx, scale_rows, 1, expert_count, token_count, 1); + ggml_tensor * selected = ggml_get_rows(ctx, repeated, ids); + ggml_tensor * output = ggml_mul(ctx, matmul, selected); + if (expose_matmul) { + ggml_set_output(matmul); + } + ggml_cgraph * graph = ggml_new_graph(ctx); + REQUIRE(graph != nullptr); + ggml_build_forward_expand(graph, output); + + const std::vector kernels = scheduled_kernel_sequence(graph); + const char * expected_kernel = expose_matmul ? + "loom_libs:ggml_mul_mat_id_skinny_input_f32_publish_f32" : + token_count <= 5 ? "loom_libs:ggml_mul_mat_id_skinny_input_f32_scaled_publish_f32" : + "loom_libs:ggml_mul_mat_id_tiled_input_f32_scaled_publish_f32"; + require_kernel_subsequence(kernels, { expected_kernel }); + if (!expose_matmul) { + REQUIRE(std::find(kernels.begin(), kernels.end(), "loom_libs:ggml_get_rows_scalar_3d_f32") == kernels.end()); + REQUIRE(std::find(kernels.begin(), kernels.end(), "loom_libs:ggml_binary_bc_f32") == kernels.end()); + } else { + require_kernel_subsequence(kernels, { + "loom_libs:ggml_repeat_scalar_3d_f32", "loom_libs:ggml_get_rows_scalar_3d_f32" + }); + } + + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); + REQUIRE(buffer != nullptr); + const std::vector weight_data = + make_quantized_rows(weight_type, input_size, output_size * expert_count, 57); + const std::vector input_data = make_pattern_f32(input_size * token_count, 11, 0.01f); + std::vector ids_data(route_stride * token_count, -1); + std::vector scales_data(expert_count); + for (int64_t expert = 0; expert < expert_count; ++expert) { + scales_data[expert] = 0.25f + 0.01f * expert; + } + for (int64_t token = 0; token < token_count; ++token) { + for (int64_t route = 0; route < route_count; ++route) { + int32_t expert = static_cast((token * 17 + route * 19) % expert_count); + const bool use_invalid_ids = include_invalid_ids && token == 0; + if (use_invalid_ids) { + if (route == 1) expert = -1; + if (route == 2) expert = expert_count; + if (route == 3) expert = std::numeric_limits::min(); + if (route == 4) expert = std::numeric_limits::max(); + } + ids_data[token * route_stride + route] = expert; + } + } + ggml_backend_tensor_set(weight, weight_data.data(), 0, weight_data.size()); + ggml_backend_tensor_set(input, input_data.data(), 0, input_data.size() * sizeof(float)); + ggml_backend_tensor_set(ids_storage, ids_data.data(), 0, ids_data.size() * sizeof(int32_t)); + ggml_backend_tensor_set(scales, scales_data.data(), 0, scales_data.size() * sizeof(float)); + + REQUIRE(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(backend); + + const ggml_type_traits * traits = ggml_get_type_traits(weight_type); + REQUIRE(traits != nullptr); + REQUIRE(traits->to_float != nullptr); + const size_t row_size = ggml_row_size(weight_type, input_size); + std::vector dequantized(input_size); + std::vector expected(output_size * route_count * token_count, 0.0f); + for (int64_t token = 0; token < token_count; ++token) { + for (int64_t route = 0; route < route_count; ++route) { + const int32_t expert = ids_data[token * route_stride + route]; + const bool valid_expert = expert >= 0 && expert < expert_count; + if (!valid_expert) { + continue; + } + for (int64_t channel = 0; channel < output_size; ++channel) { + const size_t row = static_cast(expert * output_size + channel); + traits->to_float(weight_data.data() + row * row_size, dequantized.data(), input_size); + float dot = 0.0f; + for (int64_t index = 0; index < input_size; ++index) { + dot += dequantized[index] * input_data[token * input_size + index]; + } + expected[(token * route_count + route) * output_size + channel] = dot * scales_data[expert]; + } + } + } + require_close(get_f32_tensor(backend, output), expected, 3.0e-2f, 3.0e-2f); + + ggml_backend_buffer_free(buffer); + ggml_free(ctx); + ggml_backend_free(backend); +} + static void run_scalar_3d_get_rows_case(int64_t token_count, int64_t route_stride, bool include_invalid_ids = false, bool broadcast_scales = false, bool expose_repeat = false) { ggml_backend_t backend = ggml_backend_hrx_init(0); @@ -4545,7 +4655,7 @@ static void run_scalar_3d_get_rows_case(int64_t token_count, int64_t route_strid ggml_build_forward_expand(graph, ids); } ggml_build_forward_expand(graph, output); - if (expose_repeat) { + if (broadcast_scales) { require_kernel_subsequence(scheduled_kernel_sequence(graph), { "loom_libs:ggml_repeat_scalar_3d_f32", "loom_libs:ggml_get_rows_scalar_3d_f32" }); } else { @@ -9202,17 +9312,30 @@ static void register_rmsnorm_and_scheduling_cases(Suite & suite) { } for (int64_t tokens : { 1, 2, 8, 128 }) { for (int64_t stride : { 8, 128 }) { - suite.device_case("repeat_get_rows_scalar_3d.tokens" + std::to_string(tokens) + ".stride" + + suite.device_case("standalone_repeat_get_rows_scalar_3d.tokens" + std::to_string(tokens) + ".stride" + std::to_string(stride), [=] { run_scalar_3d_get_rows_case(tokens, stride, false, true); }); } } - suite.device_case("repeat_get_rows_scalar_3d.invalid_ids", [] { + suite.device_case("standalone_repeat_get_rows_scalar_3d.invalid_ids", [] { run_scalar_3d_get_rows_case(2, 128, true, true); }); - suite.device_case("repeat_get_rows_scalar_3d.exposed_repeat", [] { + suite.device_case("standalone_repeat_get_rows_scalar_3d.exposed_repeat", [] { run_scalar_3d_get_rows_case(2, 128, false, true, true); }); + for (ggml_type weight_type : { GGML_TYPE_Q5_1, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K }) { + for (int64_t token_count : { 2, 8 }) { + suite.device_case("mul_mat_id_expert_scale." + std::string(ggml_type_name(weight_type)) + ".tokens" + + std::to_string(token_count), + [=] { run_mul_mat_id_expert_scale_case(weight_type, token_count, 8); }); + } + } + suite.device_case("mul_mat_id_expert_scale.invalid_ids.compact", [] { + run_mul_mat_id_expert_scale_case(GGML_TYPE_Q5_1, 2, 8, true); + }); + suite.device_case("mul_mat_id_expert_scale.exposed_matmul", [] { + run_mul_mat_id_expert_scale_case(GGML_TYPE_Q5_1, 2, 8, false, true); + }); for (int64_t tokens : { 1, 2, 4, 8 }) { suite.device_case("router_top8.tokens" + std::to_string(tokens), [=] { run_router_top8_case(tokens); }); } From 5e00038aade34d58e3218b5d90b6ba6ffa361a66 Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Mon, 9 Nov 2026 01:02:44 +0000 Subject: [PATCH 4/5] test: name scalar gather scenarios (lemonade-ws-enb9) --- tests/hrx-ops-test.cpp | 47 ++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/tests/hrx-ops-test.cpp b/tests/hrx-ops-test.cpp index f2c57ad90cb1..27cdcdd147c0 100644 --- a/tests/hrx-ops-test.cpp +++ b/tests/hrx-ops-test.cpp @@ -4616,8 +4616,22 @@ static void run_mul_mat_id_expert_scale_case(ggml_type weight_type, int64_t toke ggml_backend_free(backend); } -static void run_scalar_3d_get_rows_case(int64_t token_count, int64_t route_stride, bool include_invalid_ids = false, - bool broadcast_scales = false, bool expose_repeat = false) { +enum class ScalarGetRowsScenario { + Expanded, + ExpandedInvalidIds, + Repeated, + RepeatedInvalidIds, + RepeatedExposed, +}; + +static void run_scalar_3d_get_rows_case(int64_t token_count, int64_t route_stride, + ScalarGetRowsScenario scenario) { + const bool include_invalid_ids = scenario == ScalarGetRowsScenario::ExpandedInvalidIds || + scenario == ScalarGetRowsScenario::RepeatedInvalidIds; + const bool repeat_scales = scenario == ScalarGetRowsScenario::Repeated || + scenario == ScalarGetRowsScenario::RepeatedInvalidIds || + scenario == ScalarGetRowsScenario::RepeatedExposed; + const bool expose_repeat = scenario == ScalarGetRowsScenario::RepeatedExposed; ggml_backend_t backend = ggml_backend_hrx_init(0); REQUIRE(backend != nullptr); @@ -4627,7 +4641,7 @@ static void run_scalar_3d_get_rows_case(int64_t token_count, int64_t route_strid ggml_context * ctx = ggml_init(params); REQUIRE(ctx != nullptr); - ggml_tensor * scales = broadcast_scales ? ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 128) : + ggml_tensor * scales = repeat_scales ? ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 128) : ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1, 128, token_count); ggml_tensor * ids_storage = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, route_stride, token_count); REQUIRE(scales != nullptr); @@ -4637,7 +4651,7 @@ static void run_scalar_3d_get_rows_case(int64_t token_count, int64_t route_strid REQUIRE(ids != nullptr); ggml_tensor * repeated = nullptr; ggml_tensor * gather_scales = scales; - if (broadcast_scales) { + if (repeat_scales) { ggml_tensor * reshaped = ggml_reshape_3d(ctx, scales, 1, 128, 1); repeated = ggml_repeat_4d(ctx, reshaped, 1, 128, token_count, 1); gather_scales = repeated; @@ -4651,11 +4665,11 @@ static void run_scalar_3d_get_rows_case(int64_t token_count, int64_t route_strid ggml_cgraph * graph = ggml_new_graph(ctx); REQUIRE(graph != nullptr); - if (broadcast_scales) { + if (repeat_scales) { ggml_build_forward_expand(graph, ids); } ggml_build_forward_expand(graph, output); - if (broadcast_scales) { + if (repeat_scales) { require_kernel_subsequence(scheduled_kernel_sequence(graph), { "loom_libs:ggml_repeat_scalar_3d_f32", "loom_libs:ggml_get_rows_scalar_3d_f32" }); } else { @@ -4664,14 +4678,15 @@ static void run_scalar_3d_get_rows_case(int64_t token_count, int64_t route_strid ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); REQUIRE(buffer != nullptr); - std::vector scales_data(broadcast_scales ? 128 : 128 * token_count); + std::vector scales_data(repeat_scales ? 128 : 128 * token_count); std::vector ids_data(route_stride * token_count, -1); std::vector expected(8 * token_count); for (int64_t token = 0; token < token_count; ++token) { for (int64_t expert = 0; expert < 128; ++expert) { - if (!broadcast_scales || token == 0) { - const size_t scale_index = static_cast(broadcast_scales ? expert : token * 128 + expert); - scales_data[scale_index] = static_cast(broadcast_scales ? 0 : token * 1024) + expert * 0.25f; + const bool initialize_scales = !repeat_scales || token == 0; + if (initialize_scales) { + const size_t scale_index = static_cast(repeat_scales ? expert : token * 128 + expert); + scales_data[scale_index] = static_cast(repeat_scales ? 0 : token * 1024) + expert * 0.25f; } } for (int64_t route = 0; route < 8; ++route) { @@ -4687,7 +4702,7 @@ static void run_scalar_3d_get_rows_case(int64_t token_count, int64_t route_strid } ids_data[token * route_stride + route] = expert; const bool valid_expert = expert >= 0 && expert < 128; - const size_t scale_index = static_cast((broadcast_scales ? 0 : token * 128) + expert); + const size_t scale_index = static_cast((repeat_scales ? 0 : token * 128) + expert); expected[token * 8 + route] = valid_expert ? scales_data[scale_index] : 0.0f; } } @@ -9303,25 +9318,25 @@ static void register_rmsnorm_and_scheduling_cases(Suite & suite) { for (int64_t stride : { 8, 128 }) { suite.device_case("scalar_3d_get_rows.tokens" + std::to_string(tokens) + ".stride" + std::to_string(stride), - [=] { run_scalar_3d_get_rows_case(tokens, stride); }); + [=] { run_scalar_3d_get_rows_case(tokens, stride, ScalarGetRowsScenario::Expanded); }); } } for (int64_t stride : { 8, 128 }) { suite.device_case("scalar_3d_get_rows.invalid_ids.stride" + std::to_string(stride), - [=] { run_scalar_3d_get_rows_case(2, stride, true); }); + [=] { run_scalar_3d_get_rows_case(2, stride, ScalarGetRowsScenario::ExpandedInvalidIds); }); } for (int64_t tokens : { 1, 2, 8, 128 }) { for (int64_t stride : { 8, 128 }) { suite.device_case("standalone_repeat_get_rows_scalar_3d.tokens" + std::to_string(tokens) + ".stride" + std::to_string(stride), - [=] { run_scalar_3d_get_rows_case(tokens, stride, false, true); }); + [=] { run_scalar_3d_get_rows_case(tokens, stride, ScalarGetRowsScenario::Repeated); }); } } suite.device_case("standalone_repeat_get_rows_scalar_3d.invalid_ids", [] { - run_scalar_3d_get_rows_case(2, 128, true, true); + run_scalar_3d_get_rows_case(2, 128, ScalarGetRowsScenario::RepeatedInvalidIds); }); suite.device_case("standalone_repeat_get_rows_scalar_3d.exposed_repeat", [] { - run_scalar_3d_get_rows_case(2, 128, false, true, true); + run_scalar_3d_get_rows_case(2, 128, ScalarGetRowsScenario::RepeatedExposed); }); for (ggml_type weight_type : { GGML_TYPE_Q5_1, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K }) { for (int64_t token_count : { 2, 8 }) { From 92f3fe648a353fed154f3b65939100c62cbc8d9e Mon Sep 17 00:00:00 2001 From: AaronStGeorge Date: Mon, 9 Nov 2026 02:00:13 +0000 Subject: [PATCH 5/5] test: cover Gemma expert-scale dispatch selection (lemonade-ws-enb9) --- tests/hrx-backend-test.cpp | 104 +++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index 5d4dd3e2ac34..92816a9269ff 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -4554,6 +4554,109 @@ static void run_standalone_repeat_get_rows_scalar_3d_dispatch_checks() { check_case(2, 128, false, false); } +enum class ExpertScaleDispatchScenario { + Fused, + ExposedMatmul, + DifferentGatherIds, +}; + +static void run_mul_mat_id_expert_scale_dispatch_checks() { + auto check_case = [](ggml_type weight_type, int64_t token_count, int64_t route_stride, + ExpertScaleDispatchScenario scenario) { + constexpr int64_t input_size = 256; + constexpr int64_t output_size = 64; + constexpr int64_t expert_count = 128; + constexpr int64_t route_count = 8; + ggml_init_params params = {}; + params.mem_size = 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + + ggml_tensor * weight = ggml_new_tensor_3d(ctx, weight_type, input_size, output_size, expert_count); + ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, input_size, 1, token_count); + ggml_tensor * ids_storage = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, route_stride, token_count); + ggml_tensor * ids = route_stride == route_count ? ids_storage : + ggml_view_2d(ctx, ids_storage, route_count, token_count, ids_storage->nb[1], 0); + ggml_tensor * scales = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, expert_count); + REQUIRE(weight != nullptr); + REQUIRE(input != nullptr); + REQUIRE(ids != nullptr); + REQUIRE(scales != nullptr); + + ggml_tensor * gather_ids = ids; + if (scenario == ExpertScaleDispatchScenario::DifferentGatherIds) { + gather_ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, route_count, token_count); + REQUIRE(gather_ids != nullptr); + } + ggml_tensor * matmul = ggml_mul_mat_id(ctx, weight, input, ids); + ggml_tensor * scale_rows = ggml_reshape_3d(ctx, scales, 1, expert_count, 1); + ggml_tensor * repeated = ggml_repeat_4d(ctx, scale_rows, 1, expert_count, token_count, 1); + ggml_tensor * selected = ggml_get_rows(ctx, repeated, gather_ids); + ggml_tensor * output = ggml_mul(ctx, matmul, selected); + REQUIRE(output != nullptr); + if (scenario == ExpertScaleDispatchScenario::ExposedMatmul) { + ggml_set_output(matmul); + } + + ggml_cgraph * graph = ggml_new_graph(ctx); + REQUIRE(graph != nullptr); + ggml_build_forward_expand(graph, output); + ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*graph); + REQUIRE(imported.valid()); + ggml::hrx::DispatchScheduler scheduler; + REQUIRE(scheduler.schedule_graph(imported.graph, test_dispatch_target())); + const auto & dispatches = scheduler.plan().dispatches; + auto find_kernel = [&](const char * name) -> const ggml::hrx::Dispatch * { + for (const ggml::hrx::Dispatch & dispatch : dispatches) { + if (kernel_name_for_id(dispatch.kernel.kernel_id) == name) { + return &dispatch; + } + } + return nullptr; + }; + + const char * scaled_kernel = token_count <= 5 ? + "loom_libs:ggml_mul_mat_id_skinny_input_f32_scaled_publish_f32" : + "loom_libs:ggml_mul_mat_id_tiled_input_f32_scaled_publish_f32"; + if (scenario == ExpertScaleDispatchScenario::Fused) { + const ggml::hrx::Dispatch * dispatch = find_kernel(scaled_kernel); + REQUIRE(dispatch != nullptr); + REQUIRE(dispatch->bindings.size() == 7); + const ggml::hrx::Value * scale_value = imported.graph.values().find_tensor(scale_rows); + const ggml::hrx::Value * ids_value = imported.graph.values().find_tensor(ids); + const ggml::hrx::Value * output_value = imported.graph.values().find_tensor(output); + REQUIRE(scale_value != nullptr); + REQUIRE(ids_value != nullptr); + REQUIRE(output_value != nullptr); + REQUIRE(dispatch->bindings[4].value == scale_value->id); + REQUIRE(dispatch->bindings[4].length == ggml_nbytes(scale_rows)); + REQUIRE(dispatch->bindings[5].value == ids_value->id); + REQUIRE(dispatch->bindings[5].length == ggml_nbytes(ids)); + REQUIRE(dispatch->bindings[6].value == output_value->id); + require_compile_parameter(*dispatch, "ggml.mul_mat_id.scale.route_stride", std::to_string(route_stride)); + REQUIRE(find_kernel("loom_libs:ggml_repeat_scalar_3d_f32") == nullptr); + REQUIRE(find_kernel("loom_libs:ggml_get_rows_scalar_3d_f32") == nullptr); + REQUIRE(find_kernel("loom_libs:ggml_binary_bc_f32") == nullptr); + } else { + const char * ordinary_kernel = token_count <= 5 ? + "loom_libs:ggml_mul_mat_id_skinny_input_f32_publish_f32" : + "loom_libs:ggml_mul_mat_id_tiled_input_f32_publish_f32"; + REQUIRE(find_kernel(scaled_kernel) == nullptr); + REQUIRE(find_kernel(ordinary_kernel) != nullptr); + REQUIRE(find_kernel("loom_libs:ggml_repeat_scalar_3d_f32") != nullptr); + REQUIRE(find_kernel("loom_libs:ggml_get_rows_scalar_3d_f32") != nullptr); + REQUIRE(find_kernel("loom_libs:ggml_binary_bc_f32") != nullptr); + } + ggml_free(ctx); + }; + + check_case(GGML_TYPE_Q5_1, 2, 8, ExpertScaleDispatchScenario::Fused); + check_case(GGML_TYPE_Q8_0, 8, 128, ExpertScaleDispatchScenario::Fused); + check_case(GGML_TYPE_Q5_1, 2, 128, ExpertScaleDispatchScenario::ExposedMatmul); + check_case(GGML_TYPE_Q5_1, 2, 128, ExpertScaleDispatchScenario::DifferentGatherIds); +} + static void run_qwen_token_embedding_dispatch_checks() { ggml_init_params params = {}; params.mem_size = 4 * 1024 * 1024; @@ -14977,6 +15080,7 @@ static void register_hrx_backend_host_cases(test_runner::Suite & suite) { suite.host_case("standalone_repeat_get_rows_scalar_3d_dispatch", [] { run_standalone_repeat_get_rows_scalar_3d_dispatch_checks(); }); + suite.host_case("mul_mat_id_expert_scale_dispatch", [] { run_mul_mat_id_expert_scale_dispatch_checks(); }); suite.host_case("get_rows_rmsnorm_binary_dispatch", [] { run_get_rows_rmsnorm_binary_dispatch_checks(); }); suite.host_case("get_rows_scale_dispatch", [] { run_get_rows_scale_dispatch_checks(); }); suite.host_case("gather_add_dispatch", [] { run_gather_add_dispatch_checks(); });